diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c469def14332657ddd7754a90205088cf16c06c2..59941811eea69e38a085c13fb6f364e52d9ac905 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -121,6 +121,11 @@ test_pref62: PREF: "pref62" extends: .default_pref +test_pref65: + variables: + PREF: "pref65" + extends: .default_pref + test_pref69: variables: PREF: "pref69" diff --git a/RAAspotter_pref65.py b/RAAspotter_pref65.py new file mode 100644 index 0000000000000000000000000000000000000000..77de89a125c6e49e293176ee8593b0f320d414fe --- /dev/null +++ b/RAAspotter_pref65.py @@ -0,0 +1,73 @@ +import os, sys +import datetime + +from bs4 import BeautifulSoup +from urllib.parse import unquote + +from RAAspotter import RAAspotter + +class RAAspotter_pref65(RAAspotter): + + # Config + __HOST = 'https://www.hautes-pyrenees.gouv.fr' + __RAA_PAGE = {'2024': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2024', + '2023': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2023', + '2022': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2022', + '2021': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2021', + '2020': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2020', + '2019': f'{__HOST}/Publications/Recueil-d-actes-administratifs/RAA-2019'} + __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0' + full_name = 'Préfecture des Hautes-Pyrénées' + short_code = 'pref65' + + def __init__(self, data_dir): + super().__init__(data_dir, self.__USER_AGENT) + self.enable_tor(10) + + def get_raa(self, keywords): + self.print_output('RAAspotter_pref65') + self.print_output(f'Termes recherchés: {keywords}') + self.print_output('') + + pages_to_parse = [] + if self.not_before.year <= 2024: + pages_to_parse.append(self.__RAA_PAGE['2024']) + if self.not_before.year <= 2023: + pages_to_parse.append(self.__RAA_PAGE['2023']) + if self.not_before.year <= 2022: + pages_to_parse.append(self.__RAA_PAGE['2022']) + if self.not_before.year <= 2021: + pages_to_parse.append(self.__RAA_PAGE['2021']) + if self.not_before.year <= 2020: + pages_to_parse.append(self.__RAA_PAGE['2020']) + if self.not_before.year <= 2019: + pages_to_parse.append(self.__RAA_PAGE['2019']) + + for raa_page in pages_to_parse: + page_content = self.get_page(raa_page, 'get').content + raa_elements = self.get_raa_elements(page_content) + self.parse_raa(raa_elements, keywords.split(',')) + + self.mailer() + + def get_raa_elements(self, page_content): + elements = [] + # On charge le parser + soup = BeautifulSoup(page_content, 'html.parser') + + # Pour chaque balise a, on regarde si c'est un PDF, et si oui on le parse + for a in soup.select('a.fr-link.fr-link--download'): + if a.get('href') and a['href'].endswith('.pdf'): + if a['href'].startswith('/'): + url = f"{self.__HOST}{a['href']}" + else: + url = a['href'] + + url = unquote(url) + name = a.find('span').previous_sibling.replace('Télécharger ', '').strip() + date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y') + filename = url.split('/')[-1] + + raa = RAAspotter.RAA(url, date, name, filename) + elements.append(raa) + return elements diff --git a/README.md b/README.md index 65310ff5c223505c140d88a4db1cb2d5925e0a1e..ce6a8c1932334d1b614600d5a831db0b1e8f8f05 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Il est possible de ne lancer l'analyse que pour une seule administration, avec l - Préfecture de l'Isère (identifiant : `pref38`) - Préfecture du Nord (identifiant : `pref59`) - Préfecture du Pas-de-Calais (identifiant : `pref62`) +- Préfecture des Hautes-Pyrénées (identifiant : `pref65`) - Préfecture du Rhône (identifiant : `pref69`) - Préfecture du Var (identifiant : `pref83`) - Préfecture de Mayotte (identifiant : `pref976`) diff --git a/cli.py b/cli.py index 3cd4b649c7c9aa48ca96cbed0736ab4b57bbfa4f..d87017e2b7338d90089cfdce482790aad3b0e8fb 100755 --- a/cli.py +++ b/cli.py @@ -44,6 +44,7 @@ available_prefs = [ 'pref38', 'pref59', 'pref62', + 'pref65', 'pref69', 'pref83', 'pref976'