Merge branch '8-backend-parsing-contenu-metadonnees-images' into 'rp2'

Resolve "Backend : parsing contenu/métadonnées/images"

See merge request !26

apps/core/management/commands/import_old_rp.py

@@ -41,13 +41,15 @@ class Command(BaseCommand):
         # First, let's get the data from presse table
         presse = c.execute("SELECT * FROM presse")
 
-        print("Importing 0/{} from previous database".format(presse))
+        print("Importing 0/{} from previous database\r".format(presse))
         # And here we go
         done = 0
+        errors = 0
         for item in c.fetchall():
             done += 1
-            print("Importing {}/{} from previous database".format(done,
-                                                                  presse))
+            print("Importing {}/{} from previous database - {} errors\r".format(done,
+                                                                  presse,
+                                                                  errors))
             c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],))
             node = c.fetchone()
             if node is None:
@@ -57,6 +59,7 @@ class Command(BaseCommand):
                       (node['vid'],))
             revision = c.fetchone()
             if revision is None:
+                errors += 1
                 continue
 
             # Récupérons l'article si il existe en base
@@ -85,8 +88,9 @@ class Command(BaseCommand):
                 try:
                     article.fetch_content()
                     article.fetch_image()
+                    article.fetch_metadata()
                 except Exception:
-                    pass
+                    errors += 1
                 if article.status not in ("DRAFT", "PUBLISHED", ):
                     article.recover()
 

apps/rp/models.py

@@ -6,7 +6,6 @@ from taggit.managers import TaggableManager
 from newspaper import Article as ArticleParser
 from django_und.models import VoteMixin
 from django_fsm import FSMField, transition, RETURN_VALUE
-import opengraph_py3 as og
 
 from io import BytesIO
 from datetime import datetime
@@ -184,8 +183,6 @@ class Article(VoteMixin):
         if by is not None:
             article.upvote(by)
 
-        # Let's get metadata import
-        article.metadata = og.OpenGraph(url=url)
         article.save()
         return article
 
@@ -203,6 +200,21 @@ class Article(VoteMixin):
         self.extracts = article.text
         self.save()
 
+    def fetch_metadata(self):
+        import opengraph_py3 as og
+
+        if self.lang != "NA":
+            article = ArticleParser(url=self.url, language=self.lang.lower())
+        else:
+            article = ArticleParser(url=self.url)
+
+        try:
+            metadata = og.OpenGraph(url=self.url)
+            article.metadata = metadata.to_json()
+            article.save()
+        except Exception:
+            pass
+
     def fetch_image(self):
         import requests
         import imghdr

apps/rp/templates/rp/article_form.html

@@ -25,6 +25,7 @@
 
         <a class="btn btn-outline-primary" href="?fetch_content">Fetch content</a>
         <a class="btn btn-outline-primary" href="?fetch_image">Fetch image</a>
+        <a class="btn btn-outline-primary" href="?fetch_metadata">Fetch metadata</a>
       </p>
       <div class="ml-auto">
         <span>Save and&nbsp;</span>

apps/rp/views/articles.py

@@ -102,6 +102,8 @@ class ArticleEdit(PermissionRequiredMixin, UpdateView):
             self.object.fetch_content()
         elif 'fetch_image' in self.request.GET:
             self.object.fetch_image()
+        elif 'fetch_metadata' in self.request.GET:
+            self.object.fetch_metadata()
 
         context = self.get_context_data(object=self.object)
         return self.render_to_response(context)

requirements.txt

@@ -15,5 +15,5 @@ url
 coreapi
 pygments
 markdown
-opengraph-py3
+opengraph_py3
 -e git+https://github.com/mariocesar/sorl-thumbnail.git#egg=sorl-thumbnail