import os import datetime from bs4 import BeautifulSoup from urllib.parse import unquote from RAAspotter import RAAspotter class RAAspotter_pref06(RAAspotter): # Config __HOST = 'https://www.alpes-maritimes.gouv.fr' __RAA_PAGE = { '2024': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2024/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2024/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2024/Recueils-specifiques' ], '2023': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2023/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2023/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2023/Recueils-specifiques' ], '2022': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2022/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2022/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2022/Recueils-specifiques' ], '2021': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2021/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2021/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2021/Recueils-specifiques' ], '2020': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2020/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2020/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2020/Recueils-specifiques' ], '2019': [ f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2019/Recueils-mensuels', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2019/Recueils-speciaux', f'{__HOST}/Publications/Recueil-des-actes-administratifs-RAA/Annee-2019/Recueils-specifiques' ] } __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0' full_name = 'Préfecture des Alpes-Maritimes' short_code = 'pref06' def __init__(self, data_dir): super().__init__(data_dir, self.__USER_AGENT) self.enable_tor(20) def get_raa(self, keywords): self.print_output('RAAspotter_pref06') self.print_output(f'Termes recherchés: {keywords}') self.print_output('') pages_to_parse = [] if self.not_before.year <= 2024: for page in self.__RAA_PAGE['2024']: pages_to_parse.append(page) if self.not_before.year <= 2023: for page in self.__RAA_PAGE['2023']: pages_to_parse.append(page) if self.not_before.year <= 2022: for page in self.__RAA_PAGE['2022']: pages_to_parse.append(page) if self.not_before.year <= 2021: for page in self.__RAA_PAGE['2021']: pages_to_parse.append(page) if self.not_before.year <= 2020: for page in self.__RAA_PAGE['2020']: pages_to_parse.append(page) if self.not_before.year <= 2019: for page in self.__RAA_PAGE['2019']: pages_to_parse.append(page) elements = self.get_raa_with_pager( pages_to_parse, ".fr-pagination__link.fr-pagination__link--next", self.__HOST ) self.parse_raa(elements, keywords.split(',')) self.mailer() def get_raa_elements(self, page_content): elements = [] # On charge le parser soup = BeautifulSoup(page_content, 'html.parser') # Pour chaque élément fr-card__content, on cherche sa balise a, et si # c'est un PDF on le parse cards = soup.find_all('div', class_='fr-card__content') for card in cards: a = card.find('a') if a['href'].endswith('.pdf'): if a['href'].startswith('/'): url = f"{self.__HOST}{a['href']}" else: url = a['href'] url = unquote(url) name = a.get_text().strip() date = datetime.datetime.strptime(card.find('p', class_='fr-card__detail').get_text().replace('Publié le ', '').strip(), '%d/%m/%Y') filename = url.split('/')[-1] raa = RAAspotter.RAA(url, date, name, filename) elements.append(raa) return elements