.gitignore

@@ -6,3 +6,4 @@ data/
 pyvenv.cfg
 output_*.log
 *.patch
+CACHEDIR.TAG

.gitlab-ci.yml

@@ -169,6 +169,11 @@ test_pref29:
     PREF: "pref29"
   extends: .default_pref
 
+test_pref30:
+  variables:
+    PREF: "pref30"
+  extends: .default_pref
+
 test_pref31:
   variables:
     PREF: "pref31"
@@ -219,11 +224,31 @@ test_pref50:
     PREF: "pref50"
   extends: .default_pref
 
+test_pref52:
+  variables:
+    PREF: "pref52"
+  extends: .default_pref
+
+test_pref54:
+  variables:
+    PREF: "pref54"
+  extends: .default_pref
+
+test_pref55:
+  variables:
+    PREF: "pref55"
+  extends: .default_pref
+
 test_pref59:
   variables:
     PREF: "pref59"
   extends: .default_pref
 
+test_pref61:
+  variables:
+    PREF: "pref61"
+  extends: .default_pref
+
 test_pref62:
   variables:
     PREF: "pref62"
@@ -264,6 +289,16 @@ test_pref75:
     PREF: "pref75"
   extends: .default_pref
 
+test_pref76:
+  variables:
+    PREF: "pref76"
+  extends: .default_pref
+
+test_pref77:
+  variables:
+    PREF: "pref77"
+  extends: .default_pref
+
 test_pref80:
   variables:
     PREF: "pref80"
@@ -309,6 +344,11 @@ test_pref976:
     PREF: "pref976"
   extends: .default_pref
 
+test_prefbretagne:
+  variables:
+    PREF: "prefbretagne"
+  extends: .default_pref
+
 test_prefidf:
   variables:
     PREF: "prefidf"

Attrap.py

 import os
+import sys
 import re
 import random
 import ssl
@@ -8,6 +9,7 @@ import string
 import logging
 import requests
 import time
+from types import SimpleNamespace
 import datetime
 import json
 from urllib.parse import quote
@@ -168,7 +170,6 @@ class Attrap:
         self.mastodon_prefix = ''
         self.mastodon_suffix = ''
         self.safe_mode = False
-        self.timezone = datetime.datetime.now(datetime.timezone.utc).astimezone().tzname()
 
         self.update_user_agent(user_agent)
 
@@ -232,7 +233,7 @@ class Attrap:
             self.session.proxies.update(proxies)
             self.tor_requests = 0
 
-    def get_sub_pages(self, page_content, element, host, recursive_until_pdf):
+    def get_sub_pages(self, page_content, element, host, recursive_until_pdf, selenium=False):
         """
         Récupère, à partir d'un chemin CSS, les sous-pages d'une page.
 
@@ -240,6 +241,7 @@ class Attrap:
         element -- Le chemin CSS vers l'objet renvoyant vers la sous-page recherchée
         host -- Le nom d'hôte du site
         recursive_until_pdf -- Un booléen pour savoir s'il faut rechercher un fichier PDF dans le chemin CSS. Le cas échéant, relance la recherche sur la sous-page si le lien n'est pas un PDF.
+        selenium -- lance un navigateur avec Selenium pour contourner les protections anti-robots
         """
         soup = BeautifulSoup(page_content, 'html.parser')
         sub_pages = []
@@ -256,7 +258,8 @@ class Attrap:
                             sub_page_content,
                             element,
                             host,
-                            recursive_until_pdf
+                            recursive_until_pdf,
+                            selenium=selenium
                         ):
                             sub_pages.append(sub_sub_page)
                     else:
@@ -273,7 +276,7 @@ class Attrap:
                     sub_pages.append(sub_page)
         return sub_pages
 
-    def get_sub_pages_with_pager(self, page, sub_page_element, pager_element, details_element, host):
+    def get_sub_pages_with_pager(self, page, sub_page_element, pager_element, details_element, host, selenium=False):
         """
         Récupère, à partir d'un chemin CSS, les sous-pages d'une page contenant un pager.
 
@@ -282,12 +285,13 @@ class Attrap:
         pager_element -- Le chemin CSS vers le lien de page suivante du pager
         details_element -- Le chemin CSS vers l'objet contenant les détails de la sous-page recherchée
         host -- Le nom d'hôte du site
+        selenium -- lance un navigateur avec Selenium pour contourner les protections anti-robots
         """
         pages = []
         if isinstance(page, bytes):
             page = page.decode('utf-8')
         if page.startswith('https://') or page.startswith('http://'):
-            page_content = self.get_page(page, 'get').content
+            page_content = self.get_page(page, 'get', selenium=selenium).content
         else:
             page_content = page
 
@@ -320,13 +324,14 @@ class Attrap:
                 sub_page_element,
                 pager_element,
                 details_element,
-                host
+                host,
+                selenium=selenium
             ):
                 pages.append(sub_page)
 
         return pages
 
-    def get_raa_with_pager(self, pages_list, pager_element, host, filter_from_last_element_date=False):
+    def get_raa_with_pager(self, pages_list, pager_element, host, filter_from_last_element_date=False, selenium=False):
         """
         Récupère et analyse les RAA d'une page contenant un pager.
 
@@ -336,11 +341,12 @@ class Attrap:
         filter_from_last_element_date -- (Optionnel) Si la date du dernier élément de la dernière page parsée
         n'est pas dans la plage temporelle voulue, ne charge pas les pages suivantes. Par défaut à False. Ne doit
         être activé que si l'ordre des éléments est chronologique.
+        selenium -- lance un navigateur avec Selenium pour contourner les protections anti-robots
         """
         elements = []
         # On parse chaque page passée en paramètre
         for page in pages_list:
-            page_content = self.get_page(page, 'get').content
+            page_content = self.get_page(page, 'get', selenium=selenium).content
 
             # Pour chaque page, on récupère les PDF
             for raa in self.get_raa_elements(page_content):
@@ -399,12 +405,17 @@ class Attrap:
         webdriver_options.add_argument("--disable-dev-shm-usage")
         webdriver_options.add_argument("--use_subprocess")
         webdriver_options.add_argument("--disable-blink-features=AutomationControlled")
+        webdriver_options.add_experimental_option("excludeSwitches", ["enable-automation"])
+        webdriver_options.add_experimental_option('useAutomationExtension', False)
 
         if not self.user_agent == "":
             webdriver_options.add_argument(f"--user-agent={self.user_agent}")
 
-        webdriver_options.add_argument("--headless")
-        webdriver_options.add_argument("--window-size=1024,768")
+        if self.tor_enabled:
+            webdriver_options.add_argument(f'--proxy-server=socks5://127.0.0.1:9050')
+
+        webdriver_options.add_argument("--headless=new")
+        webdriver_options.add_argument("--start-maximized")
         display = Display(visible=False, size=(1024, 768))
         display.start()
 
@@ -429,6 +440,8 @@ class Attrap:
                 logger.warning(f'TimeoutException: {exc}')
                 if remaining_retries > 0:
                     time.sleep(5)
+                    if self.tor_enabled:
+                        self.tor_get_new_id()
                     return self.get_session(url, wait_element, (remaining_retries - 1))
                 else:
                     raise TimeoutException(exc)
@@ -455,13 +468,14 @@ class Attrap:
         f.write(data + "\n")
         f.close()
 
-    def get_page(self, url, method, data={}):
+    def get_page(self, url, method, data={}, selenium=False):
         """
         Récupère le contenu HTML d'une page web
 
         url -- L'URL de la page demandée
         method -- 'post' ou 'get', selon le type de requête
         data -- Un dictionnaire contenant les données à envoyer au site
+        selenium -- lance un navigateur avec Selenium pour contourner les protections anti-robots
         """
         try:
             logger.debug(f'Chargement de la page {url}')
@@ -476,10 +490,15 @@ class Attrap:
                 self.last_http_request = int(time.mktime(datetime.datetime.today().timetuple()))
 
             page = None
-            if method == 'get':
-                page = self.session.get(url, timeout=(10, 120))
-            if method == 'post':
-                page = self.session.post(url, data=data, timeout=(10, 120))
+            if selenium and method == 'get':
+                page_content = self.get_session(url, None, 6)
+                page = {'content': page_content, 'status_code': 200}
+                page = SimpleNamespace(**page)
+            else:
+                if method == 'get':
+                    page = self.session.get(url, timeout=(10, 120))
+                if method == 'post':
+                    page = self.session.post(url, data=data, timeout=(10, 120))
 
             if page.status_code == 429:
                 logger.warning('Erreur 429 Too Many Requests reçue, temporisation...')
@@ -623,7 +642,12 @@ class Attrap:
                     raa.parse_metadata(self.data_dir)
                     # Lorsque la date du RAA n'est pas connue, on a dû télécharger le PDF pour récupérer la date de ses métadonnées.
                     # Donc on vérifie à nouveau ici si la date correspond à ce qu'on veut analyser
-                    if (raa.date and raa.date >= Attrap.get_aware_datetime(self.not_before, timezone=self.timezone)):
+                    if not raa.date:
+                        os.remove(f'{self.data_dir}/raa/{raa.get_sha256()}.pdf')
+                        os.remove(f'{self.data_dir}/raa/{raa.get_sha256()}.json')
+                        logger.error(f'ERREUR: le RAA {raa.name} n\'a pas de date !')
+                        sys.exit(1)
+                    if raa.date >= Attrap.get_aware_datetime(self.not_before, timezone=self.timezone):
                         date_str = raa.date.strftime("%d/%m/%Y")
                         logger.info(f'Nouveau fichier : {raa.name} ({date_str}). URL : {url}')
                         self.flatten_pdf(raa)
@@ -631,9 +655,9 @@ class Attrap:
                         raa.extract_content(self.data_dir)
                         self.search_keywords(raa, keywords)
                     else:
+                        # On supprime le fichier de metadonnées puisqu'on ne le parsera pas
                         os.remove(f'{self.data_dir}/raa/{raa.get_sha256()}.pdf')
-                        logger.error(f'ERREUR: le RAA {raa.name} n\'a pas de date !')
-                        sys.exit(1)
+                        os.remove(f'{self.data_dir}/raa/{raa.get_sha256()}.json')
                 except PdfStreamError as exc:
                     logger.warning(f'ATTENTION: le RAA à l\'adresse {raa.url} n\'est pas valide ! On l\'ignore...')
                 except EmptyFileError as exc:
@@ -725,7 +749,7 @@ class Attrap:
         """
         try:
             search = re.search(regex, string, re.IGNORECASE)
-            guessed_date = dateparser.parse(search.group(1), languages=['fr'])
+            guessed_date = dateparser.parse(search.group(1), languages=['fr'], settings={'PREFER_DAY_OF_MONTH': 'last', 'PREFER_MONTH_OF_YEAR': 'last'})
             if guessed_date is None:
                 raise Exception('La date est un objet None')
             else:

Attrap_pref25.py

-import os
-import datetime
-import logging
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref25(Attrap_prefdpt):
 
-logger = logging.getLogger(__name__)
-
-
-class Attrap_pref25(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.doubs.gouv.fr'
     raa_page = f'{hostname}/Publications/Publications-Legales/Recueil-des-Actes-Administratifs-RAA'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture du Doubs'
     short_code = 'pref25'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        sub_pages = self.get_sub_pages_with_pager(
-            self.raa_page,
-            'a.fr-card__link',
-            'a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-            None,
-            self.hostname,
-        )
-
-        pages_to_parse = []
-
-        # TODO : détecter la date de la page à partir du parsing de la page principale et non à partir de son URL
-        for sub_page in sub_pages:
-            url = sub_page['url']
-            last_word = url.split('-')[-1]
-            year = 0
-            try:
-                year = int(last_word)
-
-                if self.not_before.year <= year:
-                    pages_to_parse.append(url)
-            except Exception as e:
-                logger.warning(f"Impossible de déterminer l'année de l'URL {url}")
-
-        elements = []
-        for raa_page in pages_to_parse:
-            page_content = self.get_page(raa_page, 'get').content
-            elements.extend(self.get_raa_elements(page_content))
-
-        self.parse_raa(elements[::-1], keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        elements = []
-        # On récupère chaque balise a
-        for a in soup.select('div.fr-downloads-group.fr-downloads-group--bordered ul li a'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').text.split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'
+    Attrap_prefdpt.grey_card['follow_link_on_unrecognised_date'] = False

Attrap_pref2b.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref2b(Attrap_prefdpt):
 
-
-class Attrap_pref2b(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.haute-corse.gouv.fr'
     raa_page = f'{hostname}/Publications/Publications-administratives-et-legales/Recueils-des-actes-administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de Haute-Corse'
     short_code = 'pref2b'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        # La préfecture de Haute-Corse organise son site avec une page dédiée à l'année N, une autre dédiée à l'année N-1,
-        # puis les années d'avant sont regroupées ensemble sur deux pages. On doit donc parser les pages jusqu'à ce qu'on ne
-        # tombe plus sur des cartes d'années.
-
-        pages_to_parse = []
-
-        page_content = self.get_page(self.raa_page, 'get').content
-        for card in self.get_sub_pages(
-            page_content,
-            'div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        ):
-            if Attrap.guess_date(card['name'], '[a-z]* ([0-9]{4})').year >= self.not_before.year:
-                pages_to_parse.append(card['url'])
-            else:
-                # Si on n'a pas trouvé une page d'année, on tente de parser la page à la rechercge
-                # de sous-pages (et sinon on ignore la page)
-                page_content = self.get_page(card['url'], 'get').content
-                for card in self.get_sub_pages(
-                    page_content,
-                    'div.fr-card__body div.fr-card__content h2.fr-card__title a',
-                    self.hostname,
-                    False
-                ):
-                    if Attrap.guess_date(card['name'], '[a-z]* ([0-9]{4})').year >= self.not_before.year:
-                        pages_to_parse.append(card['url'])
-
-        # Pour chaque année, on cherche les sous-pages de mois
-        month_pages_to_parse = []
-        for year_page in pages_to_parse:
-            for month_page in self.get_sub_pages_with_pager(
-                year_page,
-                'div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-                'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next',
-                None,
-                self.hostname
-            ):
-                if Attrap.guess_date(month_page['name'], '([a-zûé]* [0-9]{4})').replace(day=1) >= self.not_before.replace(day=1):
-                    month_pages_to_parse.append(month_page['url'])
-
-        # On parse les pages contenant des RAA
-        elements = []
-        for page in month_pages_to_parse:
-            page_content = self.get_page(page, 'get').content
-            for element in self.get_raa_elements(page_content):
-                elements.append(element)
-
-        self.parse_raa(elements[::-1], keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère chaque balise a
-        for a in soup.select('div.fr-downloads-group.fr-downloads-group--bordered ul li a'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Recueils des actes administratifs ([0-9]{4})'
+    Attrap_prefdpt.white_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'

Attrap_pref30.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref30(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.gard.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs'
+    full_name = 'Préfecture du Gard'
+    short_code = 'pref30'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'

Attrap_pref31.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref31(Attrap_prefdpt):
 
-
-class Attrap_pref31(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.haute-garonne.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs/Recueil-des-Actes-Administratifs-Haute-Garonne'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de la Haute-Garonne'
     short_code = 'pref31'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        # On cherche les pages de chaque mois
-        page_content = self.get_page(self.raa_page, 'get').content
-        month_pages = self.get_sub_pages(
-            page_content,
-            '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        )[::-1]
-
-        pages_to_parse = []
-
-        # On filtre les pages de mois pour limiter le nombre de requêtes
-        for month_page in month_pages:
-            guessed_date = Attrap.guess_date(month_page['name'], '([a-zéû]* [0-9]{4})')
-            if guessed_date >= self.not_before.replace(day=1):
-                pages_to_parse.append(month_page['url'])
-
-        elements = []
-        # On parse les pages des mois qu'on veut analyser
-        for element in self.get_raa_with_pager(
-            pages_to_parse,
-            ".fr-pagination__link.fr-pagination__link--next",
-            self.hostname
-        ):
-            elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère chaque balise a
-        for a in soup.select('div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link.menu-item-link'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.get_text().strip().capitalize()
-                date = datetime.datetime.strptime(a['title'].split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'

Attrap_pref33.py

-import os
-import re
-import datetime
-import logging
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref33(Attrap_prefdpt):
 
-logger = logging.getLogger(__name__)
-
-
-class Attrap_pref33(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.gironde.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de la Gironde'
     short_code = 'pref33'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        pages_to_parse = []
-
-        # Parfois un RAA est mal catégorisé et se retrouve sur la page racine, donc on la parse
-        pages_to_parse.append(self.raa_page)
-
-        # On détermine quelles pages d'année parser
-        year_pages_to_parse = []
-        page_content = self.get_page(self.raa_page, 'get').content
-        year_pages = self.get_sub_pages(
-            page_content,
-            '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        )
-        for year_page in year_pages:
-            year = 9999
-            try:
-                year = int(re.search('.*([0-9]{4})', year_page['name'].strip(), re.IGNORECASE).group(1))
-                if year is None:
-                    year = 9999
-            except Exception as exc:
-                logger.warning(f"Impossible de deviner l\'année de la page {year_page['name']}")
-                year = 9999
-
-            if year >= self.not_before.year:
-                year_pages_to_parse.append(year_page['url'])
-
-        # Pour chaque année, on cherche les sous-pages de mois
-        month_pages_to_parse = []
-        for year_page in year_pages_to_parse:
-            page_content = self.get_page(year_page, 'get').content
-            month_pages = self.get_sub_pages(
-                page_content,
-                '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-                self.hostname,
-                False
-            )[::-1]
-
-            for month_page in month_pages:
-                guessed_date = Attrap.guess_date(month_page['name'], '([a-zéû]* [0-9]{4})')
-                if guessed_date >= self.not_before.replace(day=1):
-                    pages_to_parse.append(month_page['url'])
-
-        # On parse les pages sélectionnées
-        elements = self.get_raa_with_pager(
-            pages_to_parse,
-            "ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label",
-            self.hostname
-        )[::-1]
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-
-        # On récupère chaque carte avec un RAA
-        for card in BeautifulSoup(page_content, 'html.parser').select('div.fr-card.fr-card--horizontal div.fr-card__body div.fr-card__content'):
-            # On récupère le lien
-            links = card.select('h2.fr-card__title a.fr-card__link.menu-item-link')
-            # On récupère la date
-            dates_raw = card.select('div.fr-card__end p.fr-card__detail')
-
-            # Si on a toutes les infos, on continue
-            if links and links[0] and dates_raw and dates_raw[0]:
-                a = links[0]
-                date_raw = dates_raw[0]
-
-                if a.get('href') and a['href'].endswith('.pdf'):
-                    if a['href'].startswith('/'):
-                        url = f"{self.hostname}{a['href']}"
-                    else:
-                        url = a['href']
-
-                    url = unquote(url)
-                    name = a.get_text().strip()
-                    date = datetime.datetime.strptime(date_raw.get_text().replace('Publié le', '').strip(), '%d/%m/%Y')
-
-                    raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                    elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Recueils{0,1} des [Aa]ctes [Aa]dministratifs de l\'année ([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'
+    Attrap_prefdpt.grey_card['follow_link_on_unrecognised_date'] = False

Attrap_pref34.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref34(Attrap_prefdpt):
 
-
-class Attrap_pref34(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.herault.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueils-des-actes-administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de l\'Hérault'
     short_code = 'pref34'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        pages_to_parse = []
-
-        year_pages = self.get_sub_pages_with_pager(
-            self.raa_page,
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-            'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content div.fr-card__end p.fr-card__detail',
-            self.hostname
-        )
-        for year_page in year_pages:
-            year = Attrap.guess_date(year_page['name'], '.*([0-9]{4})').year
-            if year >= self.not_before.year:
-                pages_to_parse.append(year_page['url'])
-
-        elements = []
-        for raa_page in pages_to_parse:
-            page_content = self.get_page(raa_page, 'get').content
-            for element in self.get_raa_elements(page_content):
-                elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère chaque balise a
-        for a in soup.select('a.fr-link.fr-link--download'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    year_regex = '(?:(?:Recueil des actes administratifs)|(?:Année))[ -]([0-9]{4})'
+    Attrap_prefdpt.white_card['regex']['year'] = year_regex
+    Attrap_prefdpt.grey_card['regex']['year'] = year_regex

Attrap_pref35.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref35(Attrap_prefdpt):
 
-
-class Attrap_pref35(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.ille-et-vilaine.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs'
-    raa_page_archives = f'{hostname}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture d\'Ille-et-Vilaine'
     short_code = 'pref35'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        year_pages_to_parse = []
-
-        # La page de l'année en cours est normalement listée sur raa_page
-        year_pages = self.get_sub_pages_with_pager(
-            self.raa_page,
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-            'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-            None,
-            self.hostname
+    # Configuration des widgets à analyser
+    year_regex = 'Recueil des actes administratifs ([0-9]{4})'
+    Attrap_prefdpt.white_card['regex']['year'] = year_regex
+
+    # On ajoute un widget de menu déroulant
+    Attrap_prefdpt.select_widgets.append(
+        Attrap_prefdpt.DptSelectWidget(
+            'menu_deroulant',
+            regex=year_regex,
+            css_path='div.fr-select-group select#Archives-des-RAA-liste-docs.fr-select',
+            type='year'
         )
-        for year_page in year_pages:
-            year = Attrap.guess_date(year_page['name'], '.*([0-9]{4})').year
-            if year >= self.not_before.year:
-                year_pages_to_parse.append(year_page['url'])
-
-        # Les URL des pages des années précédentes sont dans un menu déroulant de raa_page_archives
-        page_content = self.get_page(self.raa_page_archives, 'get').content
-        for option in BeautifulSoup(page_content, 'html.parser').select('div.fr-select-group select.fr-select option'):
-            if not option['value'] == '':
-                year = Attrap.guess_date(option.get_text().strip(), '.*([0-9]{4})').year
-                if year >= self.not_before.year:
-                    url = option['value']
-                    year_pages_to_parse.append(f'{self.hostname}/{url}')
-
-        elements = []
-        for raa_page in year_pages_to_parse:
-            page_content = self.get_page(raa_page, 'get').content
-            for element in self.get_raa_elements(page_content):
-                elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # Pour chaque balise a, on regarde si c'est un PDF, et si oui on le
-        # parse
-        for a in soup.find_all('a', href=True, class_='fr-link--download'):
-            if a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    )

Attrap_pref38.py

-import os
-import datetime
-import logging
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref38(Attrap_prefdpt):
 
-logger = logging.getLogger(__name__)
-
-
-class Attrap_pref38(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.isere.gouv.fr'
-    raa_page = [
-        f'{hostname}/Publications/RAA-Recueil-des-actes-administratifs',
-        f'{hostname}/Publications/RAA-Recueil-des-actes-administratifs/Archives'
-    ]
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
+    raa_page = f'{hostname}/Publications/RAA-Recueil-des-actes-administratifs'
     full_name = 'Préfecture de l\'Isère'
     short_code = 'pref38'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        pages_to_parse = []
-
-        # On cherche les pages d'années. Elles sont mélangées dans des blocs de cartes grises et des blocs blancs avec pager
-        for page in self.raa_page:
-            page_content = self.get_page(page, 'get').content
-            # On parse les cartes grises
-            cards = self.get_sub_pages(
-                page_content,
-                '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-                self.hostname,
-                False
-            )[::-1]
-            for card in cards:
-                year = Attrap.guess_date(card['name'].strip(), '.*([0-9]{4})').year
-                if year >= self.not_before.year and year < 9999:
-                    pages_to_parse.append(card['url'])
-            # On parse les blocs blancs
-            blocks = self.get_sub_pages_with_pager(
-                page_content,
-                'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-                'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-                None,
-                self.hostname
-            )
-            for block in blocks:
-                year = Attrap.guess_date(block['name'].strip(), '.*([0-9]{4})').year
-                if year >= self.not_before.year and year < 9999:
-                    pages_to_parse.append(block['url'])
-
-        elements = []
-        for raa_page in pages_to_parse:
-            page_content = self.get_page(raa_page, 'get').content
-            for element in self.get_raa_elements(page_content, raa_page):
-                elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content, raa_page):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère le select qui contient la liste des RAA
-        select_list = soup.select('select#-liste-docs')[0]
-        # On analyse chaque résultat
-        for option in select_list.find_all('option'):
-            if not option['value'] == "":
-                # On estime la date à partir du nom de fichier
-                guessed_date = Attrap.guess_date(option['title'], '.* n°[ 0-9]* du ([0-9]*(?:er)? [a-zéû]* [0-9]*)')
-
-                # Si la date estimée correspond à la plage d'analyse, on
-                # demande au serveur les détails du RAA
-                if guessed_date >= self.not_before:
-                    page_content = self.get_page(
-                        raa_page,
-                        'post',
-                        {
-                            '-liste-docs': option['value']
-                        }
-                    ).content
-
-                    # On parse la page de détails pour obtenir les propriétés
-                    # du RAA
-                    soup = BeautifulSoup(page_content, 'html.parser')
-                    a = soup.select('div.liste_deroulante a.fr-link.fr-link--download')[0]
-
-                    # Si la page contient une balise a qui renvoie vers un pdf,
-                    # c'est qu'on a obtenu les détails du RAA demandé, donc
-                    # on le parse
-                    if a.get('href') and a['href'].endswith('.pdf'):
-                        if a['href'].startswith('/'):
-                            url = f"{self.hostname}{a['href']}"
-                        else:
-                            url = a['href']
-
-                        url = unquote(url)
-                        name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                        date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                        raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                        elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    year_regex = '(?:(?:[Rr]ecueils{0,1} des [Aa]ctes [Aa]dministratifs de la [Pp]réfecture de l\'Isère[ -]*)|(?:Année ))([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['year'] = year_regex
+    Attrap_prefdpt.white_card['regex']['year'] = year_regex
+    Attrap_prefdpt.white_card['exclude'] = ['Vous recherchez "Le Journal officiel de la République française" ?']
+
+    # On ajoute un widget de menu déroulant
+    Attrap_prefdpt.select_widgets.append(
+        Attrap_prefdpt.DptSelectWidget(
+            'menu_deroulant',
+            regex='([0-9]{1,2}[er]{0,1} [a-zéû]* [0-9]{4})',
+            css_path='select#-liste-docs',
+            type='year-month-day'
+        )
+    )

Attrap_pref39.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref39(Attrap_prefdpt):
 
-
-class Attrap_pref39(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = "https://www.jura.gouv.fr"
     raa_page = f'{hostname}/Publications/Publications-legales/Recueil-des-Actes-Administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = "Préfecture du Jura"
     short_code = "pref39"
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        # On récupère les pages d'années
-        year_pages = []
-        page_content = self.get_page(self.raa_page, 'get').content
-        for card in self.get_sub_pages(
-            page_content,
-            'div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        ):
-            if Attrap.guess_date(card['name'], '.* ([0-9]{4})').year >= self.not_before.year:
-                year_pages.append(card['url'])
-
-        # On récupère tous les RAA en suivant la navigation
-        elements = self.get_raa_with_pager(
-            year_pages,
-            'a.fr-pagination__link.fr-pagination__link--next',
-            self.hostname
-        )
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # Pour chaque balise a, on regarde si c'est un PDF, et si oui on le parse
-        for a in soup.select('.fr-card__title a.fr-card__link.menu-item-link'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.get_text().strip()
-                date = datetime.datetime.strptime(a['title'].split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Année ([0-9]{4})'

Attrap_pref42.py

-import os
-import datetime
-import re
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref42(Attrap_prefdpt):
 
-
-class Attrap_pref42(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.loire.gouv.fr'
     raa_page = f'{hostname}/Publications/Publications-legales/Recueil-des-Actes-Administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de la Loire'
     short_code = 'pref42'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        year_pages_to_parse = []
-
-        # On détermine quelles pages d'année parser
-        year_pages = self.get_sub_pages_with_pager(
-            self.raa_page,
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-            'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content div.fr-card__end p.fr-card__detail',
-            self.hostname
-        )
-        for year_page in year_pages:
-            year = 9999
-            try:
-                year = int(re.search('([0-9]{4})', year_page['name'], re.IGNORECASE).group(1))
-                if year is None:
-                    year = 9999
-            except Exception as exc:
-                logger.warning(f"Impossible de deviner l\'année de la page {year_page['name']}")
-                year = 9999
-
-            if year >= self.not_before.year:
-                year_pages_to_parse.append(year_page['url'])
-
-        elements = []
-        # Pour chaque année, on parse les RAA
-        for year_page in year_pages_to_parse:
-            page_content = self.get_page(year_page, 'get').content
-            for element in self.get_raa_elements(page_content)[::-1]:
-                elements.append(element)
-
-        # On parse les RAA
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère chaque balise a
-        for a in soup.select('div.fr-downloads-group.fr-downloads-group--bordered ul li a'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.white_card['regex']['year'] = '([0-9]{4})'

Attrap_pref44.py

-import os
-import datetime
-import logging
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref44(Attrap_prefdpt):
 
-logger = logging.getLogger(__name__)
-
-
-class Attrap_pref44(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.loire-atlantique.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs-RAA-en-Loire-Atlantique'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture de la Loire-Atlantique'
     short_code = 'pref44'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        pages_to_parse = []
-
-        # Parfois un RAA est mal catégorisé et se retrouve sur la page racine, donc on la parse
-        pages_to_parse.append(self.raa_page)
-
-        # On détermine quelles pages d'année parser
-        year_pages_to_parse = []
-        page_content = self.get_page(self.raa_page, 'get').content
-        year_pages = self.get_sub_pages(
-            page_content,
-            '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        )
-        for year_page in year_pages:
-            year = 9999
-            try:
-                year = int(year_page['name'].strip())
-                if year is None:
-                    year = 9999
-            except Exception as exc:
-                logger.warning(f"Impossible de deviner l\'année de la page {year_page['name']}")
-                year = 9999
-
-            if year >= self.not_before.year:
-                year_pages_to_parse.append(year_page['url'])
-
-                # Parfois un RAA est mal catégorisé et se retrouve sur la page de l'année, donc on la parse
-                pages_to_parse.append(year_page['url'])
-
-        # Pour chaque année, on cherche les sous-pages de mois
-        month_pages_to_parse = []
-        for year_page in year_pages_to_parse:
-            page_content = self.get_page(year_page, 'get').content
-            month_pages = self.get_sub_pages(
-                page_content,
-                '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a',
-                self.hostname,
-                False
-            )[::-1]
-
-            for month_page in month_pages:
-                pages_to_parse.append(month_page['url'])
-
-        # On parse les pages sélectionnées
-        elements = self.get_raa_with_pager(
-            pages_to_parse,
-            "ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label",
-            self.hostname
-        )[::-1]
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-
-        # On récupère chaque carte avec un RAA
-        for card in BeautifulSoup(page_content, 'html.parser').select('div.fr-card.fr-card--horizontal div.fr-card__body div.fr-card__content'):
-            # On récupère le lien
-            links = card.select('h2.fr-card__title a.fr-card__link.menu-item-link')
-            # On récupère la date
-            dates_raw = card.select('div.fr-card__end p.fr-card__detail')
-
-            # Si on a toutes les infos, on continue
-            if links and links[0] and dates_raw and dates_raw[0]:
-                a = links[0]
-                date_raw = dates_raw[0]
-
-                if a.get('href') and a['href'].endswith('.pdf'):
-                    if a['href'].startswith('/'):
-                        url = f"{self.hostname}{a['href']}"
-                    else:
-                        url = a['href']
-
-                    url = unquote(url)
-                    name = a.get_text().strip()
-                    date = datetime.datetime.strptime(date_raw.get_text().replace('Publié le', '').strip(), '%d/%m/%Y')
-
-                    raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                    elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'
+    Attrap_prefdpt.grey_card['add_year_to_months'] = True

Attrap_pref52.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref52(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.haute-marne.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs-RAA'
+    full_name = 'Préfecture de la Haute-Marne'
+    short_code = 'pref52'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.white_card['regex']['year'] = 'Année ([0-9]{4})'

Attrap_pref54.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref54(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.meurthe-et-moselle.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs'
+    full_name = 'Préfecture de Meurthe-et-Moselle'
+    short_code = 'pref54'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.white_card['regex']['year'] = '([0-9]{4})'
+
+    # On ajoute un widget de menu déroulant
+    Attrap_prefdpt.select_widgets.append(
+        Attrap_prefdpt.DptSelectWidget(
+            'menu_deroulant',
+            regex='.* du ([0-9]*(?:er|ER)? [A-Za-zéÉûÛ]* [0-9]*)',
+            css_path='select#Liste-liste-docs',
+            type='year-month-day'
+        )
+    )

Attrap_pref55.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref55(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.meuse.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs-RAA'
+    full_name = 'Préfecture de la Meuse'
+    short_code = 'pref55'
+    timezone = 'Europe/Paris'
+
+    # On configure le widget de menu déroulant
+    Attrap_prefdpt.select_widgets.append(
+        Attrap_prefdpt.DptSelectWidget(
+            'menu_deroulant',
+            regex='RAA année ([0-9]{4})',
+            css_path='select#Liste-des-recueils-liste-docs',
+            type='year'
+        )
+    )

Attrap_pref59.py

-import os
-import datetime
-import dateparser
-import logging
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
+class Attrap_pref59(Attrap_prefdpt):
 
-logger = logging.getLogger(__name__)
-
-
-class Attrap_pref59(Attrap):
-
-    # Config
+    # Configuration de la préfecture
     hostname = 'https://www.nord.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueils-des-actes-administratifs/RAA-du-departement-du-Nord'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture du Nord'
     short_code = 'pref59'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        # On récupère les pages d'années
-        year_pages = []
-        page_content = self.get_page(self.raa_page, 'get').content
-        for card in self.get_sub_pages(
-            page_content,
-            'div.fr-card__body div.fr-card__content h2.fr-card__title a',
-            self.hostname,
-            False
-        ):
-            if Attrap.guess_date(card['name'], '([0-9]{4})').year >= self.not_before.year:
-                year_pages.append(card['url'])
-
-        elements = []
-        for raa_page in year_pages:
-            page_content = self.get_page(raa_page, 'get').content
-            sub_pages = self.get_sub_pages(
-                page_content,
-                'div.fr-card__body div.fr-card__content h2.fr-card__title a',
-                self.hostname,
-                True
-            )
-            for sub_page in sub_pages[::-1]:
-                sub_page_content = self.get_page(sub_page['url'], 'get').content
-                for element in self.get_raa_elements(sub_page_content):
-                    elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère chaque balise a
-        for a in soup.select('a.fr-link.fr-link--download'):
-            if a.get('href') and a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = '([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'
+    Attrap_prefdpt.grey_card['add_year_to_months'] = True

Attrap_pref61.py

+from Attrap_prefdpt import Attrap_prefdpt
+
+
+class Attrap_pref61(Attrap_prefdpt):
+
+    # Configuration de la préfecture
+    hostname = 'https://www.orne.gouv.fr'
+    raa_page = f'{hostname}/Publications/Recueil-des-Actes-Administratifs-RAA/Recueil-des-Actes-Administratifs-RAA'
+    full_name = 'Préfecture de l\'Orne'
+    short_code = 'pref61'
+    timezone = 'Europe/Paris'
+
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.grey_card['regex']['year'] = 'Le Recueil des actes administratifs ([0-9]{4})'
+    Attrap_prefdpt.grey_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'
+    Attrap_prefdpt.grey_card['add_year_to_months'] = True
+    Attrap_prefdpt.white_card['regex']['month'] = '([A-Za-zéû]* [0-9]{4})'
+    Attrap_prefdpt.white_card['add_year_to_months'] = True

Attrap_pref62.py

-import os
-import datetime
+from Attrap_prefdpt import Attrap_prefdpt
 
-from bs4 import BeautifulSoup
-from urllib.parse import unquote
 
-from Attrap import Attrap
-
-
-class Attrap_pref62(Attrap):
+class Attrap_pref62(Attrap_prefdpt):
 
     # Config
     hostname = 'https://www.pas-de-calais.gouv.fr'
     raa_page = f'{hostname}/Publications/Recueil-des-actes-administratifs'
-    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/115.0'
     full_name = 'Préfecture du Pas-de-Calais'
     short_code = 'pref62'
     timezone = 'Europe/Paris'
 
-    def __init__(self, data_dir):
-        super().__init__(data_dir, self.user_agent)
-        self.set_sleep_time(30)
-
-    def get_raa(self, keywords):
-        pages_to_parse = []
-
-        # On détermine quelles pages d'année parser
-        year_pages = self.get_sub_pages_with_pager(
-            self.raa_page,
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link',
-            'ul.fr-pagination__list li a.fr-pagination__link.fr-pagination__link--next.fr-pagination__link--lg-label',
-            'div.fr-card.fr-card--horizontal.fr-card--sm.fr-enlarge-link.fr-mb-3w div.fr-card__body div.fr-card__content div.fr-card__end p.fr-card__detail',
-            self.hostname
-        )
-        for year_page in year_pages:
-            if Attrap.guess_date(year_page['name'].strip(), '([0-9]{4}).*').year >= self.not_before.year:
-                pages_to_parse.append(year_page['url'])
-
-        elements = []
-        for raa_page in pages_to_parse:
-            page_content = self.get_page(raa_page, 'get').content
-            for element in self.get_raa_elements(page_content):
-                elements.append(element)
-
-        self.parse_raa(elements, keywords)
-        self.mailer()
-
-    def get_raa_elements(self, page_content):
-        elements = []
-        # On charge le parser
-        soup = BeautifulSoup(page_content, 'html.parser')
-
-        # On récupère le div qui contient la liste des RAA
-        cards = soup.select('div.fr-downloads-group.fr-downloads-group--bordered')[0]
-        # On analyse chaque balise a dans ce div
-        for a in cards.find_all('a', href=True):
-            if a['href'].endswith('.pdf'):
-                if a['href'].startswith('/'):
-                    url = f"{self.hostname}{a['href']}"
-                else:
-                    url = a['href']
-
-                url = unquote(url)
-                name = a.find('span').previous_sibling.replace('Télécharger ', '').strip()
-                date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y')
-
-                raa = Attrap.RAA(url, date, name, timezone=self.timezone)
-                elements.append(raa)
-        return elements[::-1]
+    # Configuration des widgets à analyser
+    Attrap_prefdpt.white_card['regex']['year'] = '([0-9]{4})'