.gitignore

 __pycache__/
 bin/
 lib/
+lib64
 data/
 pyvenv.cfg
 output_*.log
+*.patch
+CACHEDIR.TAG

.gitlab-ci.yml

@@ -33,7 +33,7 @@ docker:
 
 install:
   stage: install
-  image: registry.git.laquadrature.net/la-quadrature-du-net/raaspotter/base:latest
+  image: registry.git.laquadrature.net/la-quadrature-du-net/attrap/base:latest
   tags:
   - unprivileged
   script:
@@ -51,7 +51,7 @@ install:
 
 pep8:
   stage: lint
-  image: registry.git.laquadrature.net/la-quadrature-du-net/raaspotter/base:latest
+  image: registry.git.laquadrature.net/la-quadrature-du-net/attrap/base:latest
   needs: [install]
   tags:
     - unprivileged
@@ -62,7 +62,6 @@ pep8:
     - lib/
     - pyvenv.cfg
   script:
-    - source bin/activate
     - make lint
   allow_failure: true
   rules:
@@ -70,26 +69,22 @@ pep8:
 
 .default_pref:
   stage: test
-  image: registry.git.laquadrature.net/la-quadrature-du-net/raaspotter/base:latest
+  image: registry.git.laquadrature.net/la-quadrature-du-net/attrap/base:latest
   tags:
   - unprivileged
   needs: [install]
   script:
+  - misc/download-from-s3.sh "${PREF}" "${S3_KEY}" "${S3_SECRET}" "${S3_HOST}" "${S3_BUCKET}" data/ || true
   - source bin/activate
-  - pip install --upgrade -r requirements.txt
   - /etc/init.d/tor start
-  - python ./cli.py --pref "${PREF}"
-  retry: 2
+  - make "${PREF}"
+  - misc/upload-to-s3.sh "${PREF}" "${S3_KEY}" "${S3_SECRET}" "${S3_HOST}" "${S3_BUCKET}" data/ || true
   cache:
-    key: $CI_COMMIT_REF_SLUG-$CI_JOB_NAME_SLUG
-    fallback_keys:
-    - $CI_COMMIT_REF_SLUG
+    key: $CI_COMMIT_REF_SLUG
     paths:
     - bin/
     - lib/
     - pyvenv.cfg
-    - data/${PREF}/raa/*.txt
-    - data/${PREF}/raa/*.json
   artifacts:
     paths:
     - data/${PREF}/raa/*.txt
@@ -104,6 +99,21 @@ test_ppparis:
     PREF: "ppparis"
   extends: .default_pref
 
+test_pref01:
+  variables:
+    PREF: "pref01"
+  extends: .default_pref
+
+test_pref02:
+  variables:
+    PREF: "pref02"
+  extends: .default_pref
+
+test_pref03:
+  variables:
+    PREF: "pref03"
+  extends: .default_pref
+
 test_pref04:
   variables:
     PREF: "pref04"
@@ -124,16 +134,56 @@ test_pref09:
     PREF: "pref09"
   extends: .default_pref
 
+test_pref10:
+  variables:
+    PREF: "pref10"
+  extends: .default_pref
+
+test_pref11:
+  variables:
+    PREF: "pref11"
+  extends: .default_pref
+
 test_pref13:
   variables:
     PREF: "pref13"
   extends: .default_pref
 
+test_pref2a:
+  variables:
+    PREF: "pref2a"
+  extends: .default_pref
+
+test_pref2b:
+  variables:
+    PREF: "pref2b"
+  extends: .default_pref
+
+test_pref25:
+  variables:
+    PREF: "pref25"
+  extends: .default_pref
+
+test_pref29:
+  variables:
+    PREF: "pref29"
+  extends: .default_pref
+
+test_pref30:
+  variables:
+    PREF: "pref30"
+  extends: .default_pref
+
 test_pref31:
   variables:
     PREF: "pref31"
   extends: .default_pref
 
+test_pref33:
+  variables:
+    PREF: "pref33"
+  extends: .default_pref
+
 test_pref34:
   variables:
     PREF: "pref34"
@@ -149,21 +199,66 @@ test_pref38:
     PREF: "pref38"
   extends: .default_pref
 
+test_pref39:
+  variables:
+    PREF: "pref39"
+  extends: .default_pref
+
 test_pref42:
   variables:
     PREF: "pref42"
   extends: .default_pref
 
+test_pref44:
+  variables:
+    PREF: "pref44"
+  extends: .default_pref
+
+test_pref49:
+  variables:
+    PREF: "pref49"
+  extends: .default_pref
+
+test_pref50:
+  variables:
+    PREF: "pref50"
+  extends: .default_pref
+
+test_pref52:
+  variables:
+    PREF: "pref52"
+  extends: .default_pref
+
+test_pref54:
+  variables:
+    PREF: "pref54"
+  extends: .default_pref
+
+test_pref55:
+  variables:
+    PREF: "pref55"
+  extends: .default_pref
+
 test_pref59:
   variables:
     PREF: "pref59"
   extends: .default_pref
 
+test_pref61:
+  variables:
+    PREF: "pref61"
+  extends: .default_pref
+
 test_pref62:
   variables:
     PREF: "pref62"
   extends: .default_pref
 
+test_pref63:
+  variables:
+    PREF: "pref63"
+  extends: .default_pref
+
 test_pref64:
   variables:
     PREF: "pref64"
@@ -184,6 +279,26 @@ test_pref69:
     PREF: "pref69"
   extends: .default_pref
 
+test_pref73:
+  variables:
+    PREF: "pref73"
+  extends: .default_pref
+
+test_pref75:
+  variables:
+    PREF: "pref75"
+  extends: .default_pref
+
+test_pref76:
+  variables:
+    PREF: "pref76"
+  extends: .default_pref
+
+test_pref77:
+  variables:
+    PREF: "pref77"
+  extends: .default_pref
+
 test_pref80:
   variables:
     PREF: "pref80"
@@ -199,7 +314,47 @@ test_pref83:
     PREF: "pref83"
   extends: .default_pref
 
+test_pref87:
+  variables:
+    PREF: "pref87"
+  extends: .default_pref
+
+test_pref91:
+  variables:
+    PREF: "pref91"
+  extends: .default_pref
+
+test_pref92:
+  variables:
+    PREF: "pref92"
+  extends: .default_pref
+
+test_pref93:
+  variables:
+    PREF: "pref93"
+  extends: .default_pref
+
+test_pref94:
+  variables:
+    PREF: "pref94"
+  extends: .default_pref
+
 test_pref976:
   variables:
     PREF: "pref976"
   extends: .default_pref
+
+test_prefbretagne:
+  variables:
+    PREF: "prefbretagne"
+  extends: .default_pref
+
+test_prefidf:
+  variables:
+    PREF: "prefidf"
+  extends: .default_pref
+
+test_prefpaca:
+  variables:
+    PREF: "prefpaca"
+  extends: .default_pref

RAAspotter_ppparis.py → Attrap_ppparis.py

@@ -3,30 +3,27 @@ import datetime
 from bs4 import BeautifulSoup
 from urllib.parse import unquote
 
-from RAAspotter import RAAspotter
+from Attrap import Attrap
 
 
-class RAAspotter_ppparis(RAAspotter):
+class Attrap_ppparis(Attrap):
 
     # Config
-    __HOST = 'https://www.prefecturedepolice.interieur.gouv.fr'
-    __RAA_PAGE = f'{__HOST}/actualites-et-presse/arretes/accueil-arretes'
+    hostname = 'https://www.prefecturedepolice.interieur.gouv.fr'
+    raa_page = f'{hostname}/actualites-et-presse/arretes/accueil-arretes'
     __WAIT_ELEMENT = 'block-decree-list-block'
-    __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
+    user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
     full_name = 'Préfecture de police de Paris'
     short_code = 'ppparis'
+    timezone = 'Europe/Paris'
 
     def __init__(self, data_dir):
-        super().__init__(data_dir, self.__USER_AGENT)
+        super().__init__(data_dir, self.user_agent)
 
     def get_raa(self, keywords):
-        self.print_output('RAAspotter_ppparis')
-        self.print_output(f'Termes recherchés: {keywords}')
-        self.print_output('')
-
-        page_content = self.get_session(self.__RAA_PAGE, self.__WAIT_ELEMENT, 3)
+        page_content = self.get_session(self.raa_page, self.__WAIT_ELEMENT, 6)
         raa_elements = self.get_raa_elements(page_content)
-        self.parse_raa(raa_elements, keywords.split(','))
+        self.parse_raa(raa_elements, keywords)
         self.mailer()
 
     def get_raa_elements(self, page_content):
@@ -47,6 +44,6 @@ class RAAspotter_ppparis(RAAspotter):
                 name = a.find('span').get_text()
                 date = datetime.datetime.strptime(a.find('div', class_="field--type-datetime").get_text().strip(), '%d/%m/%Y')
 
-                raa = RAAspotter.RAA(url, date, name)
+                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
                 elements.append(raa)
         return elements

Attrap_pref01.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref01(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.ain.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs-RAA'
+    full_name = 'Préfecture de l\'Ain'
+    short_code = 'pref01'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '(?:Recueil|Recueils) (?:des actes administratifs)(?:[ -])*([0-9]{4})'

Attrap_pref02.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref02(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.aisne.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs'
+    full_name = 'Préfecture de l\'Aisne'
+    short_code = 'pref02'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'RAA [Aa]nnée ([0-9]{4})'

Attrap_pref03.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref03(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.allier.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs-arretes'
+    full_name = 'Préfecture de l\'Allier'
+    short_code = 'pref03'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'

Attrap_pref04.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref04(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.alpes-de-haute-provence.gouv.fr'
+    raa_page = f'{hostname}/Publications/Publications-administratives-et-legales/Recueil-des-Actes-Administratifs'
+    full_name = 'Préfecture des Alpes-de-Haute-Provence'
+    short_code = 'pref04'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'

Attrap_pref05.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref05(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.hautes-alpes.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs'
+    full_name = 'Préfecture des Hautes-Alpes'
+    short_code = 'pref05'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Année *([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* *[0-9]{4})'

Attrap_pref06.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref06(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.alpes-maritimes.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs-RAA'
+    full_name = 'Préfecture des Alpes-Maritimes'
+    short_code = 'pref06'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Année *([0-9]{4})'

Attrap_pref09.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref09(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.ariege.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs/Recueils-des-Actes-Administratifs-de-l-Ariege-a-partir-du-28-avril-2015'
+    full_name = 'Préfecture de l\'Ariège'
+    short_code = 'pref09'
+    timezone = 'Europe/Paris'

Attrap_pref10.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref10(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.aube.gouv.fr'
+    raa_page = [
+        f'{hostname}/Publications/RAA-Recueil-des-Actes-Administratifs',
+        f'{hostname}/Publications/RAA-Recueil-des-Actes-Administratifs/RAA-Archives'
+    ]
+
+    full_name = 'Préfecture de l\'Aube'
+    short_code = 'pref10'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'RAA *([0-9]{4})'
+
+    # On ajoute un widget custom représentant les liens sur la page d'accueil
+    Attrap_prefdpt.widgets.append(
+        Attrap_prefdpt.DptWidget(
+            'homepage_links',
+            regex={'year': 'Année *([0-9]{4})'},
+            css_path={'title': 'div.fr-text--lead p a.fr-link'}
+        )
+    )

Attrap_pref11.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref11(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.aude.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs-RAA'
+    full_name = 'Préfecture de l\'Aude'
+    short_code = 'pref11'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Année *([0-9]{4})'

Attrap_pref13.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref13(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.bouches-du-rhone.gouv.fr'
+    raa_page = [
+        f'{hostname}/Publications/RAA-et-Archives',
+        f'{hostname}/Publications/RAA-et-Archives/Archives-RAA-des-Bouches-du-Rhone'
+    ]
+    full_name = 'Préfecture des Bouches-du-Rhône'
+    short_code = 'pref13'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'RAA[- ]*([0-9]{4})'
+    Attrap_prefdpt.grey_card['follow_link_on_unrecognised_date'] = False

Attrap_pref25.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref25(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.doubs.gouv.fr'
+    raa_page = f'{hostname}/Publications/Publications-Legales/Recueil-des-Actes-Administratifs-RAA'
+    full_name = 'Préfecture du Doubs'
+    short_code = 'pref25'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'
+    Attrap_prefdpt.grey_card['follow_link_on_unrecognised_date'] = False

Attrap_pref29.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref29(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.finistere.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs'
+    full_name = 'Préfecture du Finistère'
+    short_code = 'pref29'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '(?:Recueils publiés en ).*([0-9]{4})'

Attrap_pref2a.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref2a(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.corse-du-sud.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-de-la-prefecture-de-la-Corse-du-Sud'
+    full_name = 'Préfecture de la Corse-du-Sud'
+    short_code = 'pref2a'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.white_card['regex']['year'] = '([0-9]{4})'

Attrap_pref2b.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref2b(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.haute-corse.gouv.fr'
+    raa_page = f'{hostname}/Publications/Publications-administratives-et-legales/Recueils-des-actes-administratifs'
+    full_name = 'Préfecture de Haute-Corse'
+    short_code = 'pref2b'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Recueils des actes administratifs ([0-9]{4})'
+    Attrap_prefdpt.white_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'

Attrap_pref30.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref30(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.gard.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs'
+    full_name = 'Préfecture du Gard'
+    short_code = 'pref30'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'

Attrap_pref31.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref31(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.haute-garonne.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs/Recueil-des-Actes-Administratifs-Haute-Garonne'
+    full_name = 'Préfecture de la Haute-Garonne'
+    short_code = 'pref31'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'