From da57a5ab9f00e68bdefe58de0fa34baac89b0438 Mon Sep 17 00:00:00 2001
From: Bastien Le Querrec <blq@laquadrature.net>
Date: Sun, 12 May 2024 00:15:14 +0200
Subject: [PATCH] =?UTF-8?q?s3:=20ajout=20d'un=20script=20permettant=20d'en?=
 =?UTF-8?q?voyer/de=20r=C3=A9cup=C3=A9rer=20toute=20la=20base=20de=20donn?=
 =?UTF-8?q?=C3=A9es?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 s3/download-all-from-s3.sh | 122 +++++++++++++++++++++++++++++++++++++
 s3/download-from-s3.sh     |   2 +-
 s3/upload-all-to-s3.sh     | 121 ++++++++++++++++++++++++++++++++++++
 s3/upload-to-s3.sh         |   2 +-
 4 files changed, 245 insertions(+), 2 deletions(-)
 create mode 100755 s3/download-all-from-s3.sh
 create mode 100755 s3/upload-all-to-s3.sh

diff --git a/s3/download-all-from-s3.sh b/s3/download-all-from-s3.sh
new file mode 100755
index 0000000..be63608
--- /dev/null
+++ b/s3/download-all-from-s3.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+set -e
+
+s3_key="${1}"
+s3_secret="${2}"
+s3_host="${3}"
+s3_bucket="${4}"
+
+root_path=$(dirname $(realpath "${BASH_SOURCE[0]}"))
+
+administrations="ppparis
+pref01
+pref02
+pref03
+pref04
+pref05
+pref06
+pref07
+pref08
+pref09
+pref10
+pref11
+pref12
+pref13
+pref14
+pref15
+pref16
+pref17
+pref18
+pref19
+pref2A
+pref2B
+pref21
+pref22
+pref23
+pref24
+pref25
+pref26
+pref27
+pref28
+pref29
+pref30
+pref31
+pref32
+pref33
+pref34
+pref35
+pref36
+pref37
+pref38
+pref39
+pref40
+pref41
+pref42
+pref43
+pref44
+pref45
+pref46
+pref47
+pref49
+pref49
+pref50
+pref51
+pref52
+pref53
+pref54
+pref55
+pref56
+pref57
+pref58
+pref59
+pref60
+pref61
+pref62
+pref63
+pref64
+pref65
+pref66
+pref67
+pref68
+pref69
+pref70
+pref71
+pref72
+pref73
+pref74
+pref76
+pref77
+pref78
+pref79
+pref80
+pref81
+pref82
+pref83
+pref84
+pref85
+pref86
+pref87
+pref88
+pref89
+pref90
+pref91
+pref92
+pref93
+pref94
+pref95
+pref971
+pref972
+pref973
+pref974
+pref976"
+
+if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket"; then
+	echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket>"
+	exit 1
+fi
+
+for i in $administrations; do
+	${root_path}/download-from-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" || true
+	rm "${root_path}/../data/${i}.zip" || true
+done
diff --git a/s3/download-from-s3.sh b/s3/download-from-s3.sh
index 1ebc53d..1a36d55 100755
--- a/s3/download-from-s3.sh
+++ b/s3/download-from-s3.sh
@@ -18,7 +18,7 @@ cd "$(dirname $(realpath "${BASH_SOURCE[0]}"))/../data/"
 
 file="${pref}.zip"
 
-echo "Downloading..."
+echo "Downloading ${pref}..."
 
 ressource="/${s3_bucket}/${file}"
 content_type="application/octet-stream"
diff --git a/s3/upload-all-to-s3.sh b/s3/upload-all-to-s3.sh
new file mode 100755
index 0000000..8062253
--- /dev/null
+++ b/s3/upload-all-to-s3.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+
+set -e
+
+s3_key="${1}"
+s3_secret="${2}"
+s3_host="${3}"
+s3_bucket="${4}"
+
+root_path=$(dirname $(realpath "${BASH_SOURCE[0]}"))
+
+administrations="ppparis
+pref01
+pref02
+pref03
+pref04
+pref05
+pref06
+pref07
+pref08
+pref09
+pref10
+pref11
+pref12
+pref13
+pref14
+pref15
+pref16
+pref17
+pref18
+pref19
+pref2A
+pref2B
+pref21
+pref22
+pref23
+pref24
+pref25
+pref26
+pref27
+pref28
+pref29
+pref30
+pref31
+pref32
+pref33
+pref34
+pref35
+pref36
+pref37
+pref38
+pref39
+pref40
+pref41
+pref42
+pref43
+pref44
+pref45
+pref46
+pref47
+pref49
+pref49
+pref50
+pref51
+pref52
+pref53
+pref54
+pref55
+pref56
+pref57
+pref58
+pref59
+pref60
+pref61
+pref62
+pref63
+pref64
+pref65
+pref66
+pref67
+pref68
+pref69
+pref70
+pref71
+pref72
+pref73
+pref74
+pref76
+pref77
+pref78
+pref79
+pref80
+pref81
+pref82
+pref83
+pref84
+pref85
+pref86
+pref87
+pref88
+pref89
+pref90
+pref91
+pref92
+pref93
+pref94
+pref95
+pref971
+pref972
+pref973
+pref974
+pref976"
+
+if test -z "$s3_key" || test -z "$s3_secret" || test -z "$s3_host" || test -z "$s3_bucket"; then
+	echo "Usage: ${0} <s3_key> <s3_secret> <s3_host> <s3_bucket>"
+	exit 1
+fi
+
+for i in $administrations; do
+	${root_path}/upload-to-s3.sh "${i}" "$s3_key" "$s3_secret" "$s3_host" "$s3_bucket" || true
+done
diff --git a/s3/upload-to-s3.sh b/s3/upload-to-s3.sh
index cfa2f67..90a2ac4 100755
--- a/s3/upload-to-s3.sh
+++ b/s3/upload-to-s3.sh
@@ -20,7 +20,7 @@ file="${pref}.zip"
 find . -name "${file}" -type f -delete
 zip "${file}" "${pref}/raa/"*.txt "${pref}/raa/"*.json > /dev/null
 
-echo "Uploading..."
+echo "Uploading ${file}..."
 
 ressource="/${s3_bucket}/${file}"
 content_type=$(file --mime-type "${file}")
-- 
GitLab