From 4ee4c50f5f50429160bea4a38b62c59796a26501 Mon Sep 17 00:00:00 2001
From: Bastien Le Querrec <blq@laquadrature.net>
Date: Tue, 19 Mar 2024 00:45:18 +0100
Subject: [PATCH] =?UTF-8?q?RAAspotter:=20temporise=20en=20cas=20d'erreur?=
=?UTF-8?q?=20r=C3=A9seau?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
RAAspotter.py | 30 +++++++++++++++++++++---------
1 file changed, 21 insertions(+), 9 deletions(-)
diff --git a/RAAspotter.py b/RAAspotter.py
index a51222c..8b8e7a2 100644
--- a/RAAspotter.py
+++ b/RAAspotter.py
@@ -211,17 +211,29 @@ class RAAspotter:
f.close()
def get_page(self, url):
- logger.debug(f'Chargement de la page {url}')
- if self.sleep_time > 0:
- time.sleep(self.sleep_time)
- page = self.session.get(url)
-
- if self.tor_enabled:
- self.tor_requests+=1
- if self.tor_max_requests>0 and self.tor_requests>self.tor_max_requests:
+ try:
+ logger.debug(f'Chargement de la page {url}')
+ if self.sleep_time > 0:
+ time.sleep(self.sleep_time)
+ page = self.session.get(url)
+
+ if page.status_code == 429:
+ logger.debug(f'Erreur 429 Too Many Requests reçue, temporisation...')
self.tor_get_new_id()
+ time.sleep(55)
+ return self.get_page(url)
+
+ if self.tor_enabled:
+ self.tor_requests+=1
+ if self.tor_max_requests>0 and self.tor_requests>self.tor_max_requests:
+ self.tor_get_new_id()
- return page
+ return page
+ except requests.exceptions.ConnectionError as exc:
+ logger.debug(f'Erreur de connexion, temporisation...')
+ self.tor_get_new_id()
+ time.sleep(55)
+ return self.get_page(url)
def update_user_agent(self, user_agent):
self.user_agent = user_agent
--
GitLab