From b56b812d65929a7734f016f5bd6147a0367dc004 Mon Sep 17 00:00:00 2001 From: Okhin Date: Wed, 3 Apr 2019 15:25:19 +0200 Subject: [PATCH 1/2] Let's manage errors if they happens on OpenGraph stuff --- apps/rp/models.py | 7 ++++++- requirements.txt | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/rp/models.py b/apps/rp/models.py index dd11adb..8018fe6 100644 --- a/apps/rp/models.py +++ b/apps/rp/models.py @@ -185,7 +185,12 @@ class Article(VoteMixin): article.upvote(by) # Let's get metadata import - article.metadata = og.OpenGraph(url=url) + try: + metadata = og.OpenGraph(url=url.decode()) + article.metadata = metadata.to_json() + except Exception: + # There's nothing behind the URL, so no metadata for us + pass article.save() return article diff --git a/requirements.txt b/requirements.txt index 15e0b96..6423d50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ url coreapi pygments markdown -opengraph-py3 +opengraph_py3 -e git+https://github.com/mariocesar/sorl-thumbnail.git#egg=sorl-thumbnail -- GitLab From 4ca3bc5630baa00abe21d4969b6596afbb6cfb9a Mon Sep 17 00:00:00 2001 From: Okhin Date: Wed, 3 Apr 2019 17:52:28 +0200 Subject: [PATCH 2/2] Adding a parse_metadata action on the Article model --- .../core/management/commands/import_old_rp.py | 12 ++++++---- apps/rp/models.py | 23 ++++++++++++------- apps/rp/templates/rp/article_form.html | 1 + apps/rp/views/articles.py | 2 ++ 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/apps/core/management/commands/import_old_rp.py b/apps/core/management/commands/import_old_rp.py index e3ae4f3..803ba04 100644 --- a/apps/core/management/commands/import_old_rp.py +++ b/apps/core/management/commands/import_old_rp.py @@ -41,13 +41,15 @@ class Command(BaseCommand): # First, let's get the data from presse table presse = c.execute("SELECT * FROM presse") - print("Importing 0/{} from previous database".format(presse)) + print("Importing 0/{} from previous database\r".format(presse)) # And here we go done = 0 + errors = 0 for item in c.fetchall(): done += 1 - print("Importing {}/{} from previous database".format(done, - presse)) + print("Importing {}/{} from previous database - {} errors\r".format(done, + presse, + errors)) c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],)) node = c.fetchone() if node is None: @@ -57,6 +59,7 @@ class Command(BaseCommand): (node['vid'],)) revision = c.fetchone() if revision is None: + errors += 1 continue # Récupérons l'article si il existe en base @@ -85,8 +88,9 @@ class Command(BaseCommand): try: article.fetch_content() article.fetch_image() + article.fetch_metadata() except Exception: - pass + errors += 1 if article.status not in ("DRAFT", "PUBLISHED", ): article.recover() diff --git a/apps/rp/models.py b/apps/rp/models.py index 8018fe6..83a825c 100644 --- a/apps/rp/models.py +++ b/apps/rp/models.py @@ -6,7 +6,6 @@ from taggit.managers import TaggableManager from newspaper import Article as ArticleParser from django_und.models import VoteMixin from django_fsm import FSMField, transition, RETURN_VALUE -import opengraph_py3 as og from io import BytesIO from datetime import datetime @@ -184,13 +183,6 @@ class Article(VoteMixin): if by is not None: article.upvote(by) - # Let's get metadata import - try: - metadata = og.OpenGraph(url=url.decode()) - article.metadata = metadata.to_json() - except Exception: - # There's nothing behind the URL, so no metadata for us - pass article.save() return article @@ -208,6 +200,21 @@ class Article(VoteMixin): self.extracts = article.text self.save() + def fetch_metadata(self): + import opengraph_py3 as og + + if self.lang != "NA": + article = ArticleParser(url=self.url, language=self.lang.lower()) + else: + article = ArticleParser(url=self.url) + + try: + metadata = og.OpenGraph(url=self.url) + article.metadata = metadata.to_json() + article.save() + except Exception: + pass + def fetch_image(self): import requests import imghdr diff --git a/apps/rp/templates/rp/article_form.html b/apps/rp/templates/rp/article_form.html index 8379b84..3e4d0db 100644 --- a/apps/rp/templates/rp/article_form.html +++ b/apps/rp/templates/rp/article_form.html @@ -25,6 +25,7 @@ Fetch content Fetch image + Fetch metadata

Save and  diff --git a/apps/rp/views/articles.py b/apps/rp/views/articles.py index 6b3ca06..6f81308 100644 --- a/apps/rp/views/articles.py +++ b/apps/rp/views/articles.py @@ -102,6 +102,8 @@ class ArticleEdit(PermissionRequiredMixin, UpdateView): self.object.fetch_content() elif 'fetch_image' in self.request.GET: self.object.fetch_image() + elif 'fetch_metadata' in self.request.GET: + self.object.fetch_metadata() context = self.get_context_data(object=self.object) return self.render_to_response(context) -- GitLab