diff --git a/RAAspotter.py b/RAAspotter.py index a51222c7f9e508a9166882063c66e82d44fd88da..8b8e7a27cdfef88748f683956ac4da2673eea863 100644 --- a/RAAspotter.py +++ b/RAAspotter.py @@ -211,17 +211,29 @@ class RAAspotter: f.close() def get_page(self, url): - logger.debug(f'Chargement de la page {url}') - if self.sleep_time > 0: - time.sleep(self.sleep_time) - page = self.session.get(url) - - if self.tor_enabled: - self.tor_requests+=1 - if self.tor_max_requests>0 and self.tor_requests>self.tor_max_requests: + try: + logger.debug(f'Chargement de la page {url}') + if self.sleep_time > 0: + time.sleep(self.sleep_time) + page = self.session.get(url) + + if page.status_code == 429: + logger.debug(f'Erreur 429 Too Many Requests reçue, temporisation...') self.tor_get_new_id() + time.sleep(55) + return self.get_page(url) + + if self.tor_enabled: + self.tor_requests+=1 + if self.tor_max_requests>0 and self.tor_requests>self.tor_max_requests: + self.tor_get_new_id() - return page + return page + except requests.exceptions.ConnectionError as exc: + logger.debug(f'Erreur de connexion, temporisation...') + self.tor_get_new_id() + time.sleep(55) + return self.get_page(url) def update_user_agent(self, user_agent): self.user_agent = user_agent