diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b543acea34077718b685b81d90a7890bfc6cf3cb..8495522063beccb3f3eedef67e4154b8a398c00d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,16 +41,19 @@ install: - lib/ - pyvenv.cfg -test_ppparis: +test_pref13: stage: test image: ${IMAGE_NAME}:latest + tags: + - unprivileged needs: [install] script: - curl --silent --location --output artifacts.zip "${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/jobs/artifacts/${CI_COMMIT_BRANCH}/download?job=${CI_JOB_NAME}&job_token=${CI_JOB_TOKEN}" || true - - unzip -q artifacts.zip data/ppparis/* || true + - unzip -q artifacts.zip data/pref13/* || true - rm artifacts.zip || true - source bin/activate - - python ./ppparis.py + - /etc/init.d/tor start + - python ./pref13.py retry: 2 only: - main @@ -62,11 +65,11 @@ test_ppparis: - pyvenv.cfg artifacts: paths: - - data/ppparis/*.txt + - data/pref13/*.txt - output.log - expire_in: 12 hours + expire_in: 1 hour -test_pref13: +test_pref35: stage: test image: ${IMAGE_NAME}:latest tags: @@ -74,11 +77,11 @@ test_pref13: needs: [install] script: - curl --silent --location --output artifacts.zip "${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/jobs/artifacts/${CI_COMMIT_BRANCH}/download?job=${CI_JOB_NAME}&job_token=${CI_JOB_TOKEN}" || true - - unzip -q artifacts.zip data/pref13/* || true + - unzip -q artifacts.zip data/pref35/* || true - rm artifacts.zip || true - source bin/activate - /etc/init.d/tor start - - python ./pref13.py + - python ./pref35.py retry: 2 only: - main @@ -90,6 +93,33 @@ test_pref13: - pyvenv.cfg artifacts: paths: - - data/pref13/*.txt + - data/pref35/*.txt + - output.log + expire_in: 1 hour + +test_ppparis: + stage: test + image: ${IMAGE_NAME}:latest + tags: + - unprivileged + needs: [install] + script: + - curl --silent --location --output artifacts.zip "${CI_SERVER_PROTOCOL}://${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/projects/${CI_PROJECT_ID}/jobs/artifacts/${CI_COMMIT_BRANCH}/download?job=${CI_JOB_NAME}&job_token=${CI_JOB_TOKEN}" || true + - unzip -q artifacts.zip data/ppparis/* || true + - rm artifacts.zip || true + - source bin/activate + - python ./ppparis.py + retry: 2 + only: + - main + cache: + key: $CI_COMMIT_REF_SLUG + paths: + - bin/ + - lib/ + - pyvenv.cfg + artifacts: + paths: + - data/ppparis/*.txt - output.log - expire_in: 12 hours + expire_in: 1 hour diff --git a/Makefile b/Makefile index 5d22939ea0497646c7edf12a88f42334bfc6de9b..1f17ffcb77454f02635e05c777b081120a4556e4 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ -make: ppparis +make: pref13 pref35 ppparis +pref13: + python pref13.py +pref35: + python pref35.py ppparis: - python3 ppparis.py + python ppparis.py diff --git a/RAAspotter_pref35.py b/RAAspotter_pref35.py new file mode 100644 index 0000000000000000000000000000000000000000..b4b8639cdf6cf756260ec63aca1ec34f78460d97 --- /dev/null +++ b/RAAspotter_pref35.py @@ -0,0 +1,52 @@ +import os, sys + +from bs4 import BeautifulSoup +from urllib.parse import unquote + +from RAAspotter import RAAspotter + +class RAAspotter_pref35(RAAspotter): + + # Config + __HOST = 'https://www.ille-et-vilaine.gouv.fr' + __RAA_PAGE = [f'{__HOST}/Publications/Recueil-des-actes-administratifs/Recueil-des-actes-administratifs-2024', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/Recueil-des-actes-administratifs-2023', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/Recueil-des-actes-administratifs-2022', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/Recueil-des-actes-administratifs-2021', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/Recueil-des-actes-administratifs-2020', + f'{__HOST}/Publications/Recueil-des-actes-administratifs/Archives-des-recueils-des-actes-administratifs/Recueil-des-actes-administratifs-2019'] + __USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' + + def __init__(self, data_dir): + super().__init__(data_dir, self.__USER_AGENT) + self.enable_tor(10) + + def get_raa(self, keywords): + self.print_output('RAAspotter_pref35') + self.print_output(f'Termes recherchés: {keywords}') + + for raa_page in self.__RAA_PAGE: + page_content = self.get_page(raa_page).content + raa_elements = self.get_raa_elements(page_content) + self.parse_raa(raa_elements, keywords.split(',')) + + def get_raa_elements(self, page_content): + elements = [] + # On charge le parser + soup = BeautifulSoup(page_content, 'html.parser') + + # Pour chaque balise a, on regarde si c'est un PDF, et si oui on le parse + for a in soup.find_all('a', href=True, class_='fr-link--download'): + if a['href'].endswith('.pdf'): + if a['href'].startswith('/'): + url = f"{self.__HOST}{a['href']}" + else: + url = a['href'] + + name = a.find('span').previous_sibling.replace('Télécharger ', '').strip() + date = a.find('span').get_text().split(' - ')[-1].strip() + filename = unquote(url.split('/')[-1]) + + raa = RAAspotter.RAA(url, date, name, filename) + elements.append(raa) + return elements diff --git a/README.md b/README.md index 54d4a109bd0970e3a3da151a3edc733d01c14ea5..43af65ac6c4d4bc4e129b038cbd636db22169622 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,9 @@ Il est possible de ne lancer l'analyse que pour une seule administration, en lan ## Administrations supportées -- Préfecture de police de Paris (`ppparis.py`) - Préfecture des Bouches-du-Rhône (`pref13.py`) +- Préfecture d'Ille-et-Vilaine (`pref35.py`) +- Préfecture de police de Paris (`ppparis.py`) ## Licence diff --git a/pref35.py b/pref35.py new file mode 100755 index 0000000000000000000000000000000000000000..2bed80cbc218ad3219043e1bad1c2d795403d55c --- /dev/null +++ b/pref35.py @@ -0,0 +1,88 @@ +import os +import argparse +import logging + +from RAAspotter import RAAspotter +from RAAspotter_pref35 import RAAspotter_pref35 + +# Config +__KEYWORDS = os.getenv('KEYWORDS') or 'vidéoprotection,caméras,captation,aéronef' +__DATA_DIR = os.path.dirname(os.path.abspath(__file__))+'/data/pref35/' +__SMTP_HOSTNAME = os.getenv('SMTP_HOSTNAME') or 'localhost' +__SMTP_USERNAME = os.getenv('SMTP_USERNAME') or '' +__SMTP_PASSWORD = os.getenv('SMTP_PASSWORD') or '' +__EMAIL_FROM = os.getenv('EMAIL_FROM') +__EMAIL_TO = os.getenv('EMAIL_TO') +if os.getenv('SMTP_PORT'): + __SMTP_PORT = int(os.getenv('SMTP_PORT')) +else: + __SMTP_PORT = 587 +if os.getenv('SMTP_STARTTLS'): + __SMTP_STARTTLS = True +else: + __SMTP_STARTTLS = False +if os.getenv('SMTP_SSL'): + __SMTP_SSL = True +else: + __SMTP_SSL = False + +# Début du script +parser = argparse.ArgumentParser(prog='pref35.py', description='Télécharge les RAA de la préfecture d\'Ille-et-Vilaine et recherche des mots-clés') +parser.add_argument('-k', '--keywords', action='store', help='liste des termes recherchés, séparés par une virgule (par défaut : vidéoprotection,caméras,captation,aéronef)') +parser.add_argument('--smtp-hostname', action='store', help='nom d\'hôte SMTP (par défaut : localhost)') +parser.add_argument('--smtp-username', action='store', help='nom d\'utilisateur SMTP (par défaut : vide)') +parser.add_argument('--smtp-password', action='store', help='mot de passe SMTP (par défaut : vide)') +parser.add_argument('--smtp-port', action='store', help='port SMTP (par défaut : 587)') +parser.add_argument('--smtp-starttls', action='store_true', help='connexion SMTP avec STARTTLS') +parser.add_argument('--smtp-ssl', action='store_true', help='connexion SMTP avec SSL') +parser.add_argument('-f', '--email-from', action='store', help='adresse de courrier électronique expéditrice des notifications') +parser.add_argument('-t', '--email-to', action='store', help='adresses de courrier électronique destinataires des notifications (séparées par une virgule)') +parser.add_argument('-v', action='store_true', help='relève le niveau de verbosité à INFO') +parser.add_argument('-vv', action='store_true', help='relève le niveau de verbosité à DEBUG') +args = parser.parse_args() + +if args.v or os.getenv('VERBOSE'): + logging.basicConfig(level=logging.INFO) + +if args.vv or os.getenv('VVERBOSE'): + logging.basicConfig(level=logging.DEBUG) + +if args.keywords: + __KEYWORDS = args.keywords + +if args.smtp_hostname: + __SMTP_HOSTNAME = args.smtp_hostname + +if args.smtp_username: + __SMTP_USERNAME = args.smtp_username + +if args.smtp_password: + __SMTP_PASSWORD = args.smtp_password + +if args.smtp_port: + __SMTP_PORT = int(args.smtp_port) + +if args.smtp_starttls: + __SMTP_STARTTLS = True + +if args.smtp_ssl: + __SMTP_SSL = True + +if args.email_from: + __EMAIL_FROM = args.email_from + +if args.email_to: + __EMAIL_TO = args.email_to + +# On crée le dossier de téléchargement +os.makedirs(__DATA_DIR, exist_ok=True) + +raa_spotter = RAAspotter_pref35(__DATA_DIR) +raa_spotter.get_raa(__KEYWORDS) + +if raa_spotter.found == True and __SMTP_HOSTNAME and __SMTP_USERNAME and __SMTP_PASSWORD and __SMTP_PORT and __EMAIL_TO and __EMAIL_FROM: + output = open(raa_spotter.output_file_path) + RAAspotter.mailer(__SMTP_HOSTNAME, __SMTP_USERNAME, __SMTP_PASSWORD, __SMTP_PORT, + __SMTP_STARTTLS, __SMTP_SSL, __EMAIL_FROM, __EMAIL_TO, + '[RAAspotter] [pref35] Nouveaux éléments trouvés', + output.read())