diff --git a/Makefile b/Makefile index 7ba36ae6b329cff15d84014a9fc76298c9134942..a7ce433abe8938314025c5c9d28368111c8a869a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -make: ppparis pref06 pref13 pref34 pref35 pref38 pref59 pref62 pref69 pref976 +make: ppparis pref06 pref13 pref34 pref35 pref38 pref59 pref62 pref69 pref83 pref976 ppparis: python cli.py --pref ppparis pref06: @@ -17,5 +17,7 @@ pref62: python cli.py --pref pref62 pref69: python cli.py --pref pref69 +pref83: + python cli.py --pref pref83 pref976: python cli.py --pref pref976 diff --git a/RAAspotter_pref83.py b/RAAspotter_pref83.py new file mode 100644 index 0000000000000000000000000000000000000000..15f7de39a28a9b48368e89f5aa872a7fd5ba1734 --- /dev/null +++ b/RAAspotter_pref83.py @@ -0,0 +1,92 @@ +import os, sys +import datetime + +from bs4 import BeautifulSoup +from urllib.parse import unquote + +from RAAspotter import RAAspotter + +class RAAspotter_pref83(RAAspotter): + + # Config + __HOST = 'https://www.var.gouv.fr' + __RAA_PAGE = {'2024': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2024', + '2023': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2023', + '2022': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2022', + '2021': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2021', + '2020': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2020', + '2019': f'{__HOST}/Publications/RAA-Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2019'} + __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0' + full_name = 'Préfecture du Var' + short_code = 'pref83' + + def __init__(self, data_dir): + super().__init__(data_dir, self.__USER_AGENT) + self.enable_tor(10) + + def get_raa(self, keywords): + self.print_output('RAAspotter_pref83') + self.print_output(f'Termes recherchés: {keywords}') + self.print_output('') + + pages_to_parse = [] + if self.not_before.year <= 2024: + pages_to_parse.append(self.__RAA_PAGE['2024']) + if self.not_before.year <= 2023: + pages_to_parse.append(self.__RAA_PAGE['2023']) + if self.not_before.year <= 2022: + pages_to_parse.append(self.__RAA_PAGE['2022']) + if self.not_before.year <= 2021: + pages_to_parse.append(self.__RAA_PAGE['2021']) + if self.not_before.year <= 2020: + pages_to_parse.append(self.__RAA_PAGE['2020']) + if self.not_before.year <= 2019: + pages_to_parse.append(self.__RAA_PAGE['2019']) + + sub_pages_to_parse = [] + + # Pour chaque année, on cherche les sous-pages de mois + for raa_page in pages_to_parse: + sub_pages_to_parse.append(raa_page) + page_content = self.get_page(raa_page, 'get').content + month_pages = self.get_sub_pages( + page_content, + '.fr-card.fr-card--sm.fr-card--grey.fr-enlarge-link div.fr-card__body div.fr-card__content h2.fr-card__title a', + self.__HOST, + False + )[::-1] + for month_page in month_pages: + sub_pages_to_parse.append(month_page['url']) + + # On parse les pages contenant des RAA + elements = self.get_raa_with_pager(sub_pages_to_parse[::-1], ".fr-pagination__link.fr-pagination__link--next", self.__HOST) + self.parse_raa(elements, keywords.split(',')) + + self.mailer() + + def get_raa_elements(self, page_content): + elements = [] + # On charge le parser + soup = BeautifulSoup(page_content, 'html.parser') + + # On récupère chaque section contenant un RAA + for a in soup.select('div.fr-card__body div.fr-card__content h2.fr-card__title a.fr-card__link.menu-item-link'): + if a.get('href') and a['href'].endswith('.pdf'): + if a['href'].startswith('/'): + url = f"{self.__HOST}{a['href']}" + else: + url = a['href'] + + url = unquote(url) + name = a.get_text().strip() + date = datetime.datetime.strptime(a['title'].split(' - ')[-1].strip(), '%d/%m/%Y') + filename = url.split('/')[-1] + + print(url) + print(name) + print(date) + print(filename) + + raa = RAAspotter.RAA(url, date, name, filename) + elements.append(raa) + return elements diff --git a/README.md b/README.md index 377f2cebe74626760914a0254f3c6efdb2bd1d84..b1fdb6141fe9b3012a0439793c219267f7217ffa 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Il est possible de ne lancer l'analyse que pour une seule administration, avec l - Préfecture du Nord (identifiant : `pref59`) - Préfecture du Pas-de-Calais (identifiant : `pref62`) - Préfecture du Rhône (identifiant : `pref69`) +- Préfecture du Var (identifiant : `pref83`) - Préfecture de Mayotte (identifiant : `pref976`) ## Licence diff --git a/cli.py b/cli.py index 317c2b0242e8d622088f0a581e001cef81a4d9d0..955ca3d6feaf48470f806b21041f2e027207bb7f 100755 --- a/cli.py +++ b/cli.py @@ -44,6 +44,7 @@ available_prefs = [ 'pref59', 'pref62', 'pref69', + 'pref83', 'pref976' ]