Commit 3e82812a authored by okhin's avatar okhin 🚴

Merge branch '21-preprocess-and-import-v1-data' into 'rp2'

Resolve "Preprocess and import v1 data"

Closes #21

See merge request !25
parents 73a3d35d 3b154331
Pipeline #2554 passed with stages
in 3 minutes and 20 seconds
...@@ -38,31 +38,29 @@ class Command(BaseCommand): ...@@ -38,31 +38,29 @@ class Command(BaseCommand):
password=options['password'][0], password=options['password'][0],
db='site') db='site')
c = db.cursor(DictCursor) c = db.cursor(DictCursor)
# First, let's get the data from presse table # First, let's get the data from presse table.
presse = c.execute("SELECT * FROM presse") # We're also grouping a lot of queries
presse = c.execute("""SELECT *, nr.body as body, GROUP_CONCAT(DISTINCT t.name) as tags
FROM presse p
JOIN node n
ON n.nid = p.nid
JOIN node_revisions nr
ON nr.vid = n.vid
JOIN term_node tn
ON tn.vid = n.vid
JOIN term_data t
ON t.tid = tn.tid
GROUP BY n.nid""")
print("Importing 0/{} from previous database\r".format(presse)) print("Importing 0/{} from previous database".format(presse), end='\r')
# And here we go # And here we go
done = 0 done = 0
errors = 0 errors = 0
for item in c.fetchall(): for item in c.fetchall():
done += 1 done += 1
print("Importing {}/{} from previous database - {} errors\r".format(done, print("Importing {}/{} from previous database ({} errors for now)".format(done,
presse, presse, errors), end='\r')
errors)) # Fetch the article, or creates it
c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],))
node = c.fetchone()
if node is None:
continue
c.execute("SELECT body FROM node_revisions WHERE vid=%s",
(node['vid'],))
revision = c.fetchone()
if revision is None:
errors += 1
continue
# Récupérons l'article si il existe en base
article = Article.add_new_url(url=item['url']) article = Article.add_new_url(url=item['url'])
if item['lang'] != "": if item['lang'] != "":
article.lang = item['lang'] article.lang = item['lang']
...@@ -74,7 +72,7 @@ class Command(BaseCommand): ...@@ -74,7 +72,7 @@ class Command(BaseCommand):
if website: if website:
article.website = website.group(1) article.website = website.group(1)
# Augmentons le score si nécessaire # Raise the score if needed
if item['note'] > 0: if item['note'] > 0:
article.und_score_up = item['note'] article.und_score_up = item['note']
if item['note'] < 0: if item['note'] < 0:
...@@ -82,9 +80,12 @@ class Command(BaseCommand): ...@@ -82,9 +80,12 @@ class Command(BaseCommand):
article.save() article.save()
article.refresh_from_db() article.refresh_from_db()
# Insert tags
article.tags.set(*item['tags'].lower().split(','))
# Publish or draft as needed
if item['published'] >= 1: if item['published'] >= 1:
# Let's get the extracts # Let's get the extracts
article.extracts = revision['body'] article.extracts = item['body']
try: try:
article.fetch_content() article.fetch_content()
article.fetch_image() article.fetch_image()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment