cli.py

@@ -4,6 +4,7 @@ import os
 import argparse
 import logging
 import datetime
+import dateparser
 import importlib
 
 from Attrap import Attrap
@@ -29,9 +30,13 @@ if os.getenv('SMTP_SSL'):
 else:
     __SMTP_SSL = False
 if os.getenv('NOT_BEFORE'):
-    __NOT_BEFORE = datetime.datetime.strptime(
-        os.getenv('NOT_BEFORE'), '%Y-%m-%d'
-    )
+    try:
+        relative_date = dateparser.parse(os.getenv('NOT_BEFORE'))
+        __NOT_BEFORE = datetime.datetime(year=relative_date.year, month=relative_date.month, day=relative_date.day)
+    except Exception as exc:
+        __NOT_BEFORE = datetime.datetime.strptime(
+            os.getenv('NOT_BEFORE'), '%Y-%m-%d'
+        )
 else:
     __NOT_BEFORE = datetime.datetime(2024, 1, 1)
 __MASTODON_ACCESS_TOKEN = os.getenv('MASTODON_ACCESS_TOKEN')
@@ -40,20 +45,36 @@ __MASTODON_INSTANCE = os.getenv('MASTODON_INSTANCE')
 # Liste des administrations supportées
 available_administrations = [
     'ppparis',
+    'pref01',
+    'pref02',
+    'pref03',
     'pref04',
     'pref05',
     'pref06',
     'pref09',
+    'pref10',
+    'pref11',
     'pref13',
+    'pref2a',
+    'pref2b',
     'pref25',
+    'pref29',
+    'pref30',
     'pref31',
     'pref33',
     'pref34',
     'pref35',
     'pref38',
+    'pref39',
     'pref42',
     'pref44',
+    'pref49',
+    'pref50',
+    'pref52',
+    'pref54',
+    'pref55',
     'pref59',
+    'pref61',
     'pref62',
     'pref63',
     'pref64',
@@ -62,14 +83,20 @@ available_administrations = [
     'pref69',
     'pref73',
     'pref75',
+    'pref76',
+    'pref77',
     'pref80',
     'pref81',
     'pref83',
     'pref87',
+    'pref91',
+    'pref92',
     'pref93',
     'pref94',
     'pref976',
-    'prefIdf'
+    'prefbretagne',
+    'prefidf',
+    'prefpaca'
 ]
 
 # Début du script
@@ -166,7 +193,7 @@ parser.add_argument(
 )
 args = parser.parse_args()
 
-if args.v or os.getenv('VERBOSE'):
+if (args.v or os.getenv('VERBOSE')) and not args.vv and not os.getenv('VVERBOSE'):
     logging.basicConfig(level=logging.INFO)
     logging.getLogger("stem").setLevel(logging.WARNING)
 
@@ -178,7 +205,11 @@ if args.keywords:
     __KEYWORDS = args.keywords
 
 if args.not_before:
-    __NOT_BEFORE = datetime.datetime.strptime(args.not_before, '%Y-%m-%d')
+    try:
+        relative_date = dateparser.parse(args.not_before)
+        __NOT_BEFORE = datetime.datetime(year=relative_date.year, month=relative_date.month, day=relative_date.day)
+    except Exception as exc:
+        __NOT_BEFORE = datetime.datetime.strptime(args.not_before, '%Y-%m-%d')
 
 if args.smtp_hostname:
     __SMTP_HOSTNAME = args.smtp_hostname

s3/download-all-from-s3.sh → misc/download-all-from-s3.sh

@@ -30,8 +30,8 @@ pref16
 pref17
 pref18
 pref19
-pref2A
-pref2B
+pref2a
+pref2b
 pref21
 pref22
 pref23
@@ -111,7 +111,9 @@ pref971
 pref972
 pref973
 pref974
-pref976"
+pref976
+prefidf
+prefpaca"
 
 if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$dest"; then
 	echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket> <dest>"

misc/fix-pref75-prefidf-url.py

+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import re
+
+from urllib.parse import unquote
+
+import hashlib
+
+parser = argparse.ArgumentParser(
+    prog='./misc/fix-pref75-prefidf-url.py',
+    description='Met à jour les URL des RAA de Paris et d\'Idf'
+)
+
+parser.add_argument(
+    '--data-dir',
+    action='store',
+    help='dossier de données (par défaut: data/)'
+)
+
+parser.add_argument(
+    '--dry-run',
+    action='store_true',
+    help='ne modifie aucun fichier, affiche seulement les modifications nécessaires (par défaut: false)'
+)
+
+args = parser.parse_args()
+
+if args.data_dir:
+    data_dir = args.data_dir
+else:
+    data_dir = 'data/'
+dry_run = args.dry_run
+
+if data_dir.startswith('/'):
+    data_dir = os.path.abspath(data_dir)
+else:
+    data_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../' + data_dir)
+
+for administration in os.listdir(data_dir):
+    # On ne cherche que les dossiers prefidf et pref75
+    if administration == 'prefidf' or administration == 'pref75':
+        administration_path = os.path.abspath(data_dir + '/' + administration + '/raa/')
+        for raa in os.listdir(administration_path):
+            if raa.endswith('.json'):
+                raa_id = re.sub('\\.json$', '', raa)
+                fixed = False
+
+                raa_path = os.path.abspath(administration_path + '/' + raa)
+                txt_path = re.sub('\\.json$', '.txt', raa_path)
+                raa_file_read = open(raa_path, 'r')
+                raa_json = json.load(raa_file_read)
+                raa_file_read.close()
+
+                url = raa_json.get('url')
+                if url.startswith('https://www.prefectures-regions.gouv.fr/ile-de-france/ile-de-france/ile-de-france/irecontenu/telechargement/'):
+                    raa_json['url'] = url.replace('https://www.prefectures-regions.gouv.fr/ile-de-france/ile-de-france/ile-de-france/irecontenu/telechargement/', 'https://www.prefectures-regions.gouv.fr/ile-de-france/irecontenu/telechargement/')
+                    fixed_raa_json = {}
+                    for key in raa_json:
+                        fixed_raa_json[key] = raa_json[key]
+                    fixed_raa_id = hashlib.sha256(unquote(raa_json['url']).encode('utf-8')).hexdigest()
+                    fixed_raa_path = raa_path.replace(raa_id, fixed_raa_id)
+                    fixed_txt_path = txt_path.replace(raa_id, fixed_raa_id)
+
+                    print(f'{raa_id} -> {fixed_raa_id}:')
+                    print(f"    {raa_json['url']}")
+                    print('')
+
+                    if not dry_run:
+                        raa_file_write = open(fixed_raa_path, 'w')
+                        raa_file_write.write(json.dumps(fixed_raa_json))
+                        raa_file_write.close()
+                        os.remove(raa_path)
+                        os.rename(txt_path, fixed_txt_path)

s3/upload-all-to-s3.sh → misc/upload-all-to-s3.sh

@@ -30,8 +30,8 @@ pref16
 pref17
 pref18
 pref19
-pref2A
-pref2B
+pref2a
+pref2b
 pref21
 pref22
 pref23
@@ -111,7 +111,9 @@ pref971
 pref972
 pref973
 pref974
-pref976"
+pref976
+prefidf
+prefpaca"
 
 if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$data"; then
 	echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket> <data>"

requirements.txt

@@ -2,10 +2,10 @@ beautifulsoup4>=4.12.3
 dateparser>=1.2.0
 ftfy>=6.2.0
 Mastodon.py>=1.8.1
-ocrmypdf>=16.3.1
+ocrmypdf<16.4.0
 pycodestyle>=2.11.1
 pypdf>=4.2.0
+pytz>=2024.2
 PyVirtualDisplay>=3.0
 requests>=2.31.0
 selenium>=4.19.0
-stem>=1.8.2