diff --git a/apps/core/management/commands/import_old_rp.py b/apps/core/management/commands/import_old_rp.py index e3ae4f320e328a2b12bc76096f8c4716f9656bb7..803ba042fd87c44c98cb71c7dfb5a6677f64220c 100644 --- a/apps/core/management/commands/import_old_rp.py +++ b/apps/core/management/commands/import_old_rp.py @@ -41,13 +41,15 @@ class Command(BaseCommand): # First, let's get the data from presse table presse = c.execute("SELECT * FROM presse") - print("Importing 0/{} from previous database".format(presse)) + print("Importing 0/{} from previous database\r".format(presse)) # And here we go done = 0 + errors = 0 for item in c.fetchall(): done += 1 - print("Importing {}/{} from previous database".format(done, - presse)) + print("Importing {}/{} from previous database - {} errors\r".format(done, + presse, + errors)) c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],)) node = c.fetchone() if node is None: @@ -57,6 +59,7 @@ class Command(BaseCommand): (node['vid'],)) revision = c.fetchone() if revision is None: + errors += 1 continue # Récupérons l'article si il existe en base @@ -85,8 +88,9 @@ class Command(BaseCommand): try: article.fetch_content() article.fetch_image() + article.fetch_metadata() except Exception: - pass + errors += 1 if article.status not in ("DRAFT", "PUBLISHED", ): article.recover() diff --git a/apps/rp/models.py b/apps/rp/models.py index dd11adb4c79213b3c00f5ceeed8fbe9e37220661..83a825cbde39ce906bf3e3cb409b62f8b1914647 100644 --- a/apps/rp/models.py +++ b/apps/rp/models.py @@ -6,7 +6,6 @@ from taggit.managers import TaggableManager from newspaper import Article as ArticleParser from django_und.models import VoteMixin from django_fsm import FSMField, transition, RETURN_VALUE -import opengraph_py3 as og from io import BytesIO from datetime import datetime @@ -184,8 +183,6 @@ class Article(VoteMixin): if by is not None: article.upvote(by) - # Let's get metadata import - article.metadata = og.OpenGraph(url=url) article.save() return article @@ -203,6 +200,21 @@ class Article(VoteMixin): self.extracts = article.text self.save() + def fetch_metadata(self): + import opengraph_py3 as og + + if self.lang != "NA": + article = ArticleParser(url=self.url, language=self.lang.lower()) + else: + article = ArticleParser(url=self.url) + + try: + metadata = og.OpenGraph(url=self.url) + article.metadata = metadata.to_json() + article.save() + except Exception: + pass + def fetch_image(self): import requests import imghdr diff --git a/apps/rp/templates/rp/article_form.html b/apps/rp/templates/rp/article_form.html index 8379b84d135a78e789537b56974c6bd88bd72420..3e4d0dbebb96fe6a2e7ab11a68145bf5bd494b3a 100644 --- a/apps/rp/templates/rp/article_form.html +++ b/apps/rp/templates/rp/article_form.html @@ -25,6 +25,7 @@ Fetch content Fetch image + Fetch metadata