Commit 3e82812a authored by okhin's avatar okhin 🚴

Merge branch '21-preprocess-and-import-v1-data' into 'rp2'

Resolve "Preprocess and import v1 data"

Closes #21

See merge request !25
parents 73a3d35d 3b154331
Pipeline #2554 passed with stages
in 3 minutes and 20 seconds
......@@ -38,31 +38,29 @@ class Command(BaseCommand):
password=options['password'][0],
db='site')
c = db.cursor(DictCursor)
# First, let's get the data from presse table
presse = c.execute("SELECT * FROM presse")
# First, let's get the data from presse table.
# We're also grouping a lot of queries
presse = c.execute("""SELECT *, nr.body as body, GROUP_CONCAT(DISTINCT t.name) as tags
FROM presse p
JOIN node n
ON n.nid = p.nid
JOIN node_revisions nr
ON nr.vid = n.vid
JOIN term_node tn
ON tn.vid = n.vid
JOIN term_data t
ON t.tid = tn.tid
GROUP BY n.nid""")
print("Importing 0/{} from previous database\r".format(presse))
print("Importing 0/{} from previous database".format(presse), end='\r')
# And here we go
done = 0
errors = 0
for item in c.fetchall():
done += 1
print("Importing {}/{} from previous database - {} errors\r".format(done,
presse,
errors))
c.execute("SELECT nid, vid FROM node WHERE nid=%s", (item['nid'],))
node = c.fetchone()
if node is None:
continue
c.execute("SELECT body FROM node_revisions WHERE vid=%s",
(node['vid'],))
revision = c.fetchone()
if revision is None:
errors += 1
continue
# Récupérons l'article si il existe en base
print("Importing {}/{} from previous database ({} errors for now)".format(done,
presse, errors), end='\r')
# Fetch the article, or creates it
article = Article.add_new_url(url=item['url'])
if item['lang'] != "":
article.lang = item['lang']
......@@ -74,7 +72,7 @@ class Command(BaseCommand):
if website:
article.website = website.group(1)
# Augmentons le score si nécessaire
# Raise the score if needed
if item['note'] > 0:
article.und_score_up = item['note']
if item['note'] < 0:
......@@ -82,9 +80,12 @@ class Command(BaseCommand):
article.save()
article.refresh_from_db()
# Insert tags
article.tags.set(*item['tags'].lower().split(','))
# Publish or draft as needed
if item['published'] >= 1:
# Let's get the extracts
article.extracts = revision['body']
article.extracts = item['body']
try:
article.fetch_content()
article.fetch_image()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment