From 47073f5a2211bf37a76242d04fc6acadd86cb8a0 Mon Sep 17 00:00:00 2001 From: Bastien Le Querrec <blq@laquadrature.net> Date: Fri, 5 Apr 2024 18:23:14 +0200 Subject: [PATCH] =?UTF-8?q?pref87:=20ajout=20de=20la=20pr=C3=A9fecture=20d?= =?UTF-8?q?e=20la=20Haute-Vienne?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 4 +- RAAspotter_pref87.py | 111 +++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + cli.py | 1 + 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 RAAspotter_pref87.py diff --git a/Makefile b/Makefile index 2d1053d..34d0800 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -make: ppparis pref04 pref06 pref09 pref13 pref31 pref34 pref35 pref38 pref59 pref62 pref65 pref66 pref69 pref81 pref83 pref976 +make: ppparis pref04 pref06 pref09 pref13 pref31 pref34 pref35 pref38 pref59 pref62 pref65 pref66 pref69 pref81 pref83 pref87 pref976 ppparis: python cli.py --pref ppparis pref04: @@ -33,6 +33,8 @@ pref81: python cli.py --pref pref81 pref83: python cli.py --pref pref83 +pref87: + python cli.py --pref pref87 pref976: python cli.py --pref pref976 lint: diff --git a/RAAspotter_pref87.py b/RAAspotter_pref87.py new file mode 100644 index 0000000..e9814d0 --- /dev/null +++ b/RAAspotter_pref87.py @@ -0,0 +1,111 @@ +import os +import datetime + +from bs4 import BeautifulSoup +from urllib.parse import unquote + +from RAAspotter import RAAspotter + + +class RAAspotter_pref87(RAAspotter): + + # Config + __HOST = 'https://www.haute-vienne.gouv.fr' + __RAA_PAGE = { + '2024': [ + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JANVIER-JUIN-2024/JANVIER-JUIN-2024', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JUILLET-DECEMBRE-2024' + ], + '2023': [ + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JANVIER-JUIN-2023', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JUILLET-DECEMBRE-2023/JUILLET-DECEMBRE-2023' + ], + '2022': [ + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JANVIER-JUIN-2022', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/JUILLET-DECEMBRE-2022/Recueil-des-actes-administratifs-2022', + ], + '2021': [f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/2021'], + '2020': [f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/2020'], + '2019': [f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/2019'] + } + __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0' + full_name = 'Préfecture de la Haute-Vienne' + short_code = 'pref87' + + def __init__(self, data_dir): + super().__init__(data_dir, self.__USER_AGENT) + self.enable_tor(10) + + def get_raa(self, keywords): + self.print_output('RAAspotter_pref87') + self.print_output(f'Termes recherchés: {keywords}') + self.print_output('') + + year_pages_to_parse = [] + if self.not_before.year <= 2024: + for year_page in self.__RAA_PAGE['2024']: + year_pages_to_parse.append(year_page) + if self.not_before.year <= 2023: + for year_page in self.__RAA_PAGE['2023']: + year_pages_to_parse.append(year_page) + if self.not_before.year <= 2022: + for year_page in self.__RAA_PAGE['2022']: + year_pages_to_parse.append(year_page) + if self.not_before.year <= 2021: + for year_page in self.__RAA_PAGE['2021']: + year_pages_to_parse.append(year_page) + if self.not_before.year <= 2020: + for year_page in self.__RAA_PAGE['2020']: + year_pages_to_parse.append(year_page) + if self.not_before.year <= 2019: + for year_page in self.__RAA_PAGE['2019']: + year_pages_to_parse.append(year_page) + + pages_to_parse = year_pages_to_parse + # Pour chaque année, on cherche les éventuelles sous-pages de mois + for year_page in year_pages_to_parse: + page_content = self.get_page(year_page, 'get').content + month_pages = self.get_sub_pages( + page_content, + '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a', + self.__HOST, + False + )[::-1] + + # On filtre les pages de mois ne correspondant pas à la période analysée + for month_page in month_pages: + guessed_date = RAAspotter.guess_date(month_page['name'], '([a-zéû]* [0-9]{4})').replace(day=1) + if guessed_date >= self.not_before.replace(day=1): + pages_to_parse.append(month_page['url']) + + # On parse les pages contenant des RAA + elements = [] + for page in pages_to_parse: + page_content = self.get_page(page, 'get').content + for raa in self.get_raa_elements(page_content): + elements.append(raa) + + self.parse_raa(elements, keywords.split(',')) + self.mailer() + + def get_raa_elements(self, page_content): + elements = [] + # On charge le parser + soup = BeautifulSoup(page_content, 'html.parser') + + # On récupère chaque balise a + for a in soup.select('a.fr-link.fr-link--download'): + if a.get('href') and a['href'].endswith('.pdf'): + if a['href'].startswith('/'): + url = f"{self.__HOST}{a['href']}" + else: + url = a['href'] + + url = unquote(url) + name = a.find('span').previous_sibling.replace('Télécharger ', '').strip() + date = datetime.datetime.strptime(a.find('span').get_text().split(' - ')[-1].strip(), '%d/%m/%Y') + filename = url.split('/')[-1] + + raa = RAAspotter.RAA(url, date, name, filename) + elements.append(raa) + return elements diff --git a/README.md b/README.md index 63b18a3..779dec4 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Il est possible de ne lancer l'analyse que pour une seule administration, avec l - Préfecture du Rhône (identifiant : `pref69`) - Préfecture du Tarn (identifiant : `pref81`) - Préfecture du Var (identifiant : `pref83`) +- Préfecture de la Haute-Vienne (identifiant : `pref87`) - Préfecture de Mayotte (identifiant : `pref976`) ## Licence diff --git a/cli.py b/cli.py index 21dc6df..802c134 100755 --- a/cli.py +++ b/cli.py @@ -55,6 +55,7 @@ available_prefs = [ 'pref69', 'pref81', 'pref83', + 'pref87', 'pref976' ] -- GitLab