diff --git a/apps/core/management/commands/import_old_rp.py b/apps/core/management/commands/import_old_rp.py index e3ae4f320e328a2b12bc76096f8c4716f9656bb7..803ba042fd87c44c98cb71c7dfb5a6677f64220c 100644 --- a/apps/core/management/commands/import_old_rp.py +++ b/apps/core/management/commands/import_old_rp.py @@ -41,13 +41,15 @@ class Command(BaseCommand): # First, let's get the data from presse table presse = c.execute("SELECT * FROM presse") - print("Importing 0/{} from previous database".format(presse)) + print("Importing 0/{} from previous database\r".format(presse)) # And here we go done = 0 + errors = 0 for item in c.fetchall(): done += 1 - print("Importing {}/{} from previous database".format(done, - presse)) + print("Importing {}/{} from previous database - {} errors\r".format(done, + presse, + errors)) c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],)) node = c.fetchone() if node is None: @@ -57,6 +59,7 @@ class Command(BaseCommand): (node['vid'],)) revision = c.fetchone() if revision is None: + errors += 1 continue # Récupérons l'article si il existe en base @@ -85,8 +88,9 @@ class Command(BaseCommand): try: article.fetch_content() article.fetch_image() + article.fetch_metadata() except Exception: - pass + errors += 1 if article.status not in ("DRAFT", "PUBLISHED", ): article.recover() diff --git a/apps/rp/models.py b/apps/rp/models.py index dd11adb4c79213b3c00f5ceeed8fbe9e37220661..83a825cbde39ce906bf3e3cb409b62f8b1914647 100644 --- a/apps/rp/models.py +++ b/apps/rp/models.py @@ -6,7 +6,6 @@ from taggit.managers import TaggableManager from newspaper import Article as ArticleParser from django_und.models import VoteMixin from django_fsm import FSMField, transition, RETURN_VALUE -import opengraph_py3 as og from io import BytesIO from datetime import datetime @@ -184,8 +183,6 @@ class Article(VoteMixin): if by is not None: article.upvote(by) - # Let's get metadata import - article.metadata = og.OpenGraph(url=url) article.save() return article @@ -203,6 +200,21 @@ class Article(VoteMixin): self.extracts = article.text self.save() + def fetch_metadata(self): + import opengraph_py3 as og + + if self.lang != "NA": + article = ArticleParser(url=self.url, language=self.lang.lower()) + else: + article = ArticleParser(url=self.url) + + try: + metadata = og.OpenGraph(url=self.url) + article.metadata = metadata.to_json() + article.save() + except Exception: + pass + def fetch_image(self): import requests import imghdr diff --git a/apps/rp/templates/rp/article_form.html b/apps/rp/templates/rp/article_form.html index 8379b84d135a78e789537b56974c6bd88bd72420..3e4d0dbebb96fe6a2e7ab11a68145bf5bd494b3a 100644 --- a/apps/rp/templates/rp/article_form.html +++ b/apps/rp/templates/rp/article_form.html @@ -25,6 +25,7 @@ Fetch content Fetch image + Fetch metadata

Save and  diff --git a/apps/rp/views/articles.py b/apps/rp/views/articles.py index 6b3ca0621ea14b8227052726e8cae48354e2b8bc..6f8130871d52391e590408479b555316ad4df50b 100644 --- a/apps/rp/views/articles.py +++ b/apps/rp/views/articles.py @@ -102,6 +102,8 @@ class ArticleEdit(PermissionRequiredMixin, UpdateView): self.object.fetch_content() elif 'fetch_image' in self.request.GET: self.object.fetch_image() + elif 'fetch_metadata' in self.request.GET: + self.object.fetch_metadata() context = self.get_context_data(object=self.object) return self.render_to_response(context) diff --git a/requirements.txt b/requirements.txt index 15e0b96a19beb024f61f62c3356c5a6eed8d2da5..6423d500b32e106b0a673d8e37ece7823855609c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ url coreapi pygments markdown -opengraph-py3 +opengraph_py3 -e git+https://github.com/mariocesar/sorl-thumbnail.git#egg=sorl-thumbnail