Commit a88381e8 authored by Nicolas Joyard's avatar Nicolas Joyard

First pass at recommendations import

parent eeb3b220
......@@ -258,6 +258,10 @@ LOGGING = {
'handlers': ['console'],
'level': LOG_LEVEL,
},
'representatives_recommendations': {
'handlers': ['console'],
'level': LOG_LEVEL
},
'representatives_votes': {
'handlers': ['console'],
'level': LOG_LEVEL,
......
# coding: utf-8
import csv
import django
from django.apps import apps
from django.db import transaction
import logging
import sys
from representatives_recommendations.models import Recommendation
from representatives_votes.models import Dossier, Proposal
logger = logging.getLogger(__name__)
dossier_mappings = {
"Resolution on Anti-Counterfeiting Trade Agreement (ACTA)":
"Resolution on the Anti-Counterfeiting Trade Agreement (ACTA)",
"Rapport Bono on cultural industries in Europe":
"Cultural industries in Europe",
"Rapport Gallo on enforcement of intellectual property rights in the internal market":
"Enforcement of intellectual property rights in the internal market",
"Rapport Lambrinidis on strengthening security and fundamental freedoms on the Internet":
"Strengthening security and fundamental freedoms on the Internet",
"Criminal measures aimed at ensuring the enforcement of intellectual property rights (IPRED 2), 1st reading":
"Criminal measures aimed at ensuring the enforcement of intellectual property rights"
}
class RecommendationImporter:
def __init__(self):
self.dossier_cache = {}
def get_dossier(self, title):
alt_title = dossier_mappings.get(title, None)
if alt_title is not None:
title = alt_title
dossier = self.dossier_cache.get(title, None)
if dossier is None:
try:
dossier = Dossier.objects.get(title__iexact=title)
except Dossier.DoesNotExist:
dossier = None
return dossier
def get_proposal(self, dossier, kind):
try:
return Proposal.objects.get(dossier=dossier, kind__iexact=kind)
except Proposal.DoesNotExist:
return None
def import_row(self, row):
dossier = self.get_dossier(row['title'])
if dossier is None:
logger.warn('Could not find dossier "%s"' % row['title'])
return False
proposal = self.get_proposal(dossier, row['part'])
if proposal is None:
logger.warn('Could not find proposal "%s" for dossier "%s"' % (
row['part'], row['title']))
return False
try:
recom = Recommendation.objects.get(proposal=proposal)
except Recommendation.DoesNotExist:
recom = Recommendation(
proposal=proposal,
recommendation=row['recommendation'],
title=row['description'],
weight=row['weight']
)
recom.save()
logger.info('Created recommendation with weight %s for %s: %s' % (
row['weight'],
row['title'],
row['part']
))
return True
def main(stream=None):
"""
Imports recommendations from an old memopol instance.
Usage:
cat recommendations.csv | memopol_import_recommendations > rejected.csv
The input CSV file should be generated by the following query:
SELECT CONCAT(r.description, ';', r.weight, ';', r.recommendation, ';',
r.part, ';', p.title)
FROM votes_recommendation r
LEFT JOIN votes_proposal p ON r.proposal_id = p.id
WHERE p.institution = 'EU'
"""
if not apps.ready:
django.setup()
importer = RecommendationImporter()
rejected = []
imported = 0
reader = csv.DictReader(stream or sys.stdin, delimiter=';', fieldnames=[
'description',
'weight',
'recommendation',
'part',
'title'
], quoting=csv.QUOTE_NONE)
for row in reader:
if not importer.import_row(row):
rejected.append(row)
else:
imported = imported + 1
logger.info('%d rows imported, %d rows rejected', imported, len(rejected))
......@@ -41,5 +41,10 @@ setup(name='political-memory',
'pytest-cov==2.2.0',
'codecov',
]
},
entry_points={
'console_scripts': [
'memopol_import_recommendations = representatives_recommendations.contrib.import_recommendations:main'
]
}
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment