misc/upload-to-s3.sh

+#!/usr/bin/env bash
+
+set -e
+
+pref="${1}"
+s3_key="${2}"
+s3_secret="${3}"
+s3_host="${4}"
+s3_bucket="${5}"
+data="${6}"
+
+if test -z "$pref" || test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket" || test -z "$data"; then
+	echo "Usage: ${0} <pref> <s3_key> <s3_secret> <s3_host> <s3_bucket> <data>"
+	exit 1
+fi
+
+data=$(realpath "${data}")
+
+cd "${data}/${pref}/raa"
+cd "../../"
+
+file="${pref}.zip"
+find . -name "${file}" -type f -delete
+zip "${file}" "${pref}/raa/"*.txt "${pref}/raa/"*.json > /dev/null
+
+echo "Uploading ${file}..."
+
+ressource="/${s3_bucket}/${file}"
+content_type=$(file --mime-type "${file}")
+date=$(date --utc -R)
+
+signature=$(echo -en "PUT\n\n${content_type}\n${date}\n${ressource}" | openssl sha1 -hmac "${s3_secret}" -binary | base64)
+
+curl -X PUT \
+	 -T "${file}" \
+	 -H "Date: ${date}" \
+	 -H "Content-Type: ${content_type}" \
+	 -H "Authorization: AWS ${s3_key}:${signature}" \
+	 "${s3_host}${ressource}"
+
+rm "${file}"

requirements.txt

 beautifulsoup4>=4.12.3
 dateparser>=1.2.0
+ftfy>=6.2.0
 Mastodon.py>=1.8.1
-pdfminer.six>=20231228
-pycodestyle>=2.2.0
+ocrmypdf<16.4.0
+pycodestyle>=2.11.1
+pypdf>=4.2.0
+pytz>=2024.2
 PyVirtualDisplay>=3.0
 requests>=2.31.0
-selenium>=4.18.1
-stem>=1.8.2
+selenium>=4.19.0