Skip to content
Extraits de code Groupes Projets

Comparer les révisions

Les modifications sont affichées comme si la révision source était fusionnée avec la révision cible. En savoir plus sur la comparaison des révisions.

Source

Sélectionner le projet cible
No results found

Cible

Sélectionner le projet cible
  • la-quadrature-du-net/Attrap
  • foggyfrog/Attrap
  • skhwiz/Attrap
  • precambrien/Attrap
  • ketsapiwiq/Attrap
  • Joseki/Attrap
  • kr1p/attrap-pref-12
  • kr1p/attrap-pref-46
  • kr1p/attrap-pi
  • Guinness/Attrap
  • astroidgritty/attrap-pref-84
  • davinov/Attrap
  • maettellite/attrap-pref-01
  • m242/Attrap
  • multi/Attrap
  • mverdeil/Attrap
  • olpo/Attrap
17 résultats
Afficher les modifications
Tout étendre Masquer les modifications d'espaces
En ligne Côte à côte
cli.py
Voir le fichier @ f1d8eaac
#!/usr/bin/env python3
import os
import argparse
import logging
import datetime
import dateparser
import importlib
from Attrap import Attrap
......@@ -27,9 +30,13 @@ if os.getenv('SMTP_SSL'):
else:
__SMTP_SSL = False
if os.getenv('NOT_BEFORE'):
__NOT_BEFORE = datetime.datetime.strptime(
os.getenv('NOT_BEFORE'), '%Y-%m-%d'
)
try:
relative_date = dateparser.parse(os.getenv('NOT_BEFORE'))
__NOT_BEFORE = datetime.datetime(year=relative_date.year, month=relative_date.month, day=relative_date.day)
except Exception as exc:
__NOT_BEFORE = datetime.datetime.strptime(
os.getenv('NOT_BEFORE'), '%Y-%m-%d'
)
else:
__NOT_BEFORE = datetime.datetime(2024, 1, 1)
__MASTODON_ACCESS_TOKEN = os.getenv('MASTODON_ACCESS_TOKEN')
......@@ -38,30 +45,58 @@ __MASTODON_INSTANCE = os.getenv('MASTODON_INSTANCE')
# Liste des administrations supportées
available_administrations = [
'ppparis',
'pref01',
'pref02',
'pref03',
'pref04',
'pref05',
'pref06',
'pref09',
'pref10',
'pref11',
'pref13',
'pref2a',
'pref2b',
'pref25',
'pref29',
'pref30',
'pref31',
'pref33',
'pref34',
'pref35',
'pref38',
'pref39',
'pref42',
'pref44',
'pref49',
'pref50',
'pref52',
'pref54',
'pref55',
'pref59',
'pref61',
'pref62',
'pref63',
'pref64',
'pref65',
'pref66',
'pref69',
'pref73',
'pref75',
'pref76',
'pref77',
'pref80',
'pref81',
'pref83',
'pref87',
'pref976'
'pref91',
'pref92',
'pref93',
'pref94',
'pref976',
'prefbretagne',
'prefidf',
'prefpaca'
]
# Début du script
......@@ -158,7 +193,7 @@ parser.add_argument(
)
args = parser.parse_args()
if args.v or os.getenv('VERBOSE'):
if (args.v or os.getenv('VERBOSE')) and not args.vv and not os.getenv('VVERBOSE'):
logging.basicConfig(level=logging.INFO)
logging.getLogger("stem").setLevel(logging.WARNING)
......@@ -170,7 +205,11 @@ if args.keywords:
__KEYWORDS = args.keywords
if args.not_before:
__NOT_BEFORE = datetime.datetime.strptime(args.not_before, '%Y-%m-%d')
try:
relative_date = dateparser.parse(args.not_before)
__NOT_BEFORE = datetime.datetime(year=relative_date.year, month=relative_date.month, day=relative_date.day)
except Exception as exc:
__NOT_BEFORE = datetime.datetime.strptime(args.not_before, '%Y-%m-%d')
if args.smtp_hostname:
__SMTP_HOSTNAME = args.smtp_hostname
......
s3/download-all-from-s3.sh → misc/download-all-from-s3.sh
Voir le fichier @ f1d8eaac
......@@ -6,6 +6,7 @@ s3_key="${1}"
s3_secret="${2}"
s3_host="${3}"
s3_bucket="${4}"
dest="${5}"
root_path=$(dirname $(realpath "${BASH_SOURCE[0]}"))
......@@ -29,8 +30,8 @@ pref16
pref17
pref18
pref19
pref2A
pref2B
pref2a
pref2b
pref21
pref22
pref23
......@@ -85,6 +86,7 @@ pref71
pref72
pref73
pref74
pref75
pref76
pref77
pref78
......@@ -109,14 +111,16 @@ pref971
pref972
pref973
pref974
pref976"
pref976
prefidf
prefpaca"
if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket"; then
echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket>"
if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$dest"; then
echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket> <dest>"
exit 1
fi
for i in $administrations; do
${root_path}/download-from-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" || true
rm "${root_path}/../data/${i}.zip" || true
${root_path}/download-from-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" "$dest" || true
rm "${dest}/${i}.zip" || true
done
s3/download-from-s3.sh → misc/download-from-s3.sh
Voir le fichier @ f1d8eaac
......@@ -7,14 +7,17 @@ s3_key="${2}"
s3_secret="${3}"
s3_host="${4}"
s3_bucket="${5}"
dest="${6}"
if test -z "$pref" || test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket"; then
echo "Usage: ${0} <pref> <s3_key> <s3_secret> <s3_host> <s3_bucket>"
if test -z "$pref" || test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$dest"; then
echo "Usage: ${0} <pref> <s3_key> <s3_secret> <s3_host> <s3_bucket> <dest>"
exit 1
fi
mkdir -p "$(dirname $(realpath "${BASH_SOURCE[0]}"))/../data/"
cd "$(dirname $(realpath "${BASH_SOURCE[0]}"))/../data/"
dest=$(realpath "${dest}")
mkdir -p "${dest}/"
cd "${dest}/"
file="${pref}.zip"
......
misc/fix-pref75-prefidf-url.py 0 → 100755
Voir le fichier @ f1d8eaac
#!/usr/bin/env python3
import argparse
import json
import os
import re
from urllib.parse import unquote
import hashlib
parser = argparse.ArgumentParser(
prog='./misc/fix-pref75-prefidf-url.py',
description='Met à jour les URL des RAA de Paris et d\'Idf'
)
parser.add_argument(
'--data-dir',
action='store',
help='dossier de données (par défaut: data/)'
)
parser.add_argument(
'--dry-run',
action='store_true',
help='ne modifie aucun fichier, affiche seulement les modifications nécessaires (par défaut: false)'
)
args = parser.parse_args()
if args.data_dir:
data_dir = args.data_dir
else:
data_dir = 'data/'
dry_run = args.dry_run
if data_dir.startswith('/'):
data_dir = os.path.abspath(data_dir)
else:
data_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + '/../' + data_dir)
for administration in os.listdir(data_dir):
# On ne cherche que les dossiers prefidf et pref75
if administration == 'prefidf' or administration == 'pref75':
administration_path = os.path.abspath(data_dir + '/' + administration + '/raa/')
for raa in os.listdir(administration_path):
if raa.endswith('.json'):
raa_id = re.sub('\\.json$', '', raa)
fixed = False
raa_path = os.path.abspath(administration_path + '/' + raa)
txt_path = re.sub('\\.json$', '.txt', raa_path)
raa_file_read = open(raa_path, 'r')
raa_json = json.load(raa_file_read)
raa_file_read.close()
url = raa_json.get('url')
if url.startswith('https://www.prefectures-regions.gouv.fr/ile-de-france/ile-de-france/ile-de-france/irecontenu/telechargement/'):
raa_json['url'] = url.replace('https://www.prefectures-regions.gouv.fr/ile-de-france/ile-de-france/ile-de-france/irecontenu/telechargement/', 'https://www.prefectures-regions.gouv.fr/ile-de-france/irecontenu/telechargement/')
fixed_raa_json = {}
for key in raa_json:
fixed_raa_json[key] = raa_json[key]
fixed_raa_id = hashlib.sha256(unquote(raa_json['url']).encode('utf-8')).hexdigest()
fixed_raa_path = raa_path.replace(raa_id, fixed_raa_id)
fixed_txt_path = txt_path.replace(raa_id, fixed_raa_id)
print(f'{raa_id} -> {fixed_raa_id}:')
print(f" {raa_json['url']}")
print('')
if not dry_run:
raa_file_write = open(fixed_raa_path, 'w')
raa_file_write.write(json.dumps(fixed_raa_json))
raa_file_write.close()
os.remove(raa_path)
os.rename(txt_path, fixed_txt_path)
misc/update_metadata_format.py 0 → 100755
Voir le fichier @ f1d8eaac
Ce diff est replié.
s3/upload-all-to-s3.sh → misc/upload-all-to-s3.sh
Voir le fichier @ f1d8eaac
......@@ -6,6 +6,7 @@ s3_key="${1}"
s3_secret="${2}"
s3_host="${3}"
s3_bucket="${4}"
data="${5}"
root_path=$(dirname $(realpath "${BASH_SOURCE[0]}"))
......@@ -29,8 +30,8 @@ pref16
pref17
pref18
pref19
pref2A
pref2B
pref2a
pref2b
pref21
pref22
pref23
......@@ -85,6 +86,7 @@ pref71
pref72
pref73
pref74
pref75
pref76
pref77
pref78
......@@ -109,13 +111,15 @@ pref971
pref972
pref973
pref974
pref976"
pref976
prefidf
prefpaca"
if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket"; then
echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket>"
if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$data"; then
echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket> <data>"
exit 1
fi
for i in $administrations; do
${root_path}/upload-to-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" || true
${root_path}/upload-to-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" "$data" || true
done
requirements.txt
Voir le fichier @ f1d8eaac
......@@ -2,9 +2,10 @@ beautifulsoup4>=4.12.3
dateparser>=1.2.0
ftfy>=6.2.0
Mastodon.py>=1.8.1
ocrmypdf<16.4.0
pycodestyle>=2.11.1
pypdf>=4.2.0
pytz>=2024.2
PyVirtualDisplay>=3.0
requests>=2.31.0
selenium>=4.19.0
stem>=1.8.2