From a88381e8ec0cc6447b9d5912caad1897229bcbb6 Mon Sep 17 00:00:00 2001
From: Nicolas Joyard <joyard.nicolas@gmail.com>
Date: Sun, 29 May 2016 18:33:19 +0200
Subject: [PATCH] First pass at recommendations import
---
memopol/settings.py | 4 +
.../contrib/__init__.py | 0
.../contrib/import_recommendations.py | 123 ++++++++++++++++++
setup.py | 5 +
4 files changed, 132 insertions(+)
create mode 100644 representatives_recommendations/contrib/__init__.py
create mode 100644 representatives_recommendations/contrib/import_recommendations.py
diff --git a/memopol/settings.py b/memopol/settings.py
index 1cf752ae..48def9fe 100644
--- a/memopol/settings.py
+++ b/memopol/settings.py
@@ -258,6 +258,10 @@ LOGGING = {
'handlers': ['console'],
'level': LOG_LEVEL,
},
+ 'representatives_recommendations': {
+ 'handlers': ['console'],
+ 'level': LOG_LEVEL
+ },
'representatives_votes': {
'handlers': ['console'],
'level': LOG_LEVEL,
diff --git a/representatives_recommendations/contrib/__init__.py b/representatives_recommendations/contrib/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/representatives_recommendations/contrib/import_recommendations.py b/representatives_recommendations/contrib/import_recommendations.py
new file mode 100644
index 00000000..6f30307c
--- /dev/null
+++ b/representatives_recommendations/contrib/import_recommendations.py
@@ -0,0 +1,123 @@
+# coding: utf-8
+
+import csv
+import django
+from django.apps import apps
+from django.db import transaction
+import logging
+import sys
+
+from representatives_recommendations.models import Recommendation
+from representatives_votes.models import Dossier, Proposal
+
+logger = logging.getLogger(__name__)
+
+dossier_mappings = {
+ "Resolution on Anti-Counterfeiting Trade Agreement (ACTA)":
+ "Resolution on the Anti-Counterfeiting Trade Agreement (ACTA)",
+ "Rapport Bono on cultural industries in Europe":
+ "Cultural industries in Europe",
+ "Rapport Gallo on enforcement of intellectual property rights in the internal market":
+ "Enforcement of intellectual property rights in the internal market",
+ "Rapport Lambrinidis on strengthening security and fundamental freedoms on the Internet":
+ "Strengthening security and fundamental freedoms on the Internet",
+ "Criminal measures aimed at ensuring the enforcement of intellectual property rights (IPRED 2), 1st reading":
+ "Criminal measures aimed at ensuring the enforcement of intellectual property rights"
+}
+
+
+class RecommendationImporter:
+ def __init__(self):
+ self.dossier_cache = {}
+
+ def get_dossier(self, title):
+ alt_title = dossier_mappings.get(title, None)
+ if alt_title is not None:
+ title = alt_title
+
+ dossier = self.dossier_cache.get(title, None)
+
+ if dossier is None:
+ try:
+ dossier = Dossier.objects.get(title__iexact=title)
+ except Dossier.DoesNotExist:
+ dossier = None
+
+ return dossier
+
+ def get_proposal(self, dossier, kind):
+ try:
+ return Proposal.objects.get(dossier=dossier, kind__iexact=kind)
+ except Proposal.DoesNotExist:
+ return None
+
+ def import_row(self, row):
+ dossier = self.get_dossier(row['title'])
+ if dossier is None:
+ logger.warn('Could not find dossier "%s"' % row['title'])
+ return False
+
+ proposal = self.get_proposal(dossier, row['part'])
+ if proposal is None:
+ logger.warn('Could not find proposal "%s" for dossier "%s"' % (
+ row['part'], row['title']))
+ return False
+
+ try:
+ recom = Recommendation.objects.get(proposal=proposal)
+ except Recommendation.DoesNotExist:
+ recom = Recommendation(
+ proposal=proposal,
+ recommendation=row['recommendation'],
+ title=row['description'],
+ weight=row['weight']
+ )
+ recom.save()
+ logger.info('Created recommendation with weight %s for %s: %s' % (
+ row['weight'],
+ row['title'],
+ row['part']
+ ))
+
+ return True
+
+
+def main(stream=None):
+ """
+ Imports recommendations from an old memopol instance.
+
+ Usage:
+ cat recommendations.csv | memopol_import_recommendations > rejected.csv
+
+ The input CSV file should be generated by the following query:
+ SELECT CONCAT(r.description, ';', r.weight, ';', r.recommendation, ';',
+ r.part, ';', p.title)
+ FROM votes_recommendation r
+ LEFT JOIN votes_proposal p ON r.proposal_id = p.id
+ WHERE p.institution = 'EU'
+
+ """
+
+ if not apps.ready:
+ django.setup()
+
+ importer = RecommendationImporter()
+ rejected = []
+ imported = 0
+
+ reader = csv.DictReader(stream or sys.stdin, delimiter=';', fieldnames=[
+ 'description',
+ 'weight',
+ 'recommendation',
+ 'part',
+ 'title'
+ ], quoting=csv.QUOTE_NONE)
+
+ for row in reader:
+ if not importer.import_row(row):
+ rejected.append(row)
+ else:
+ imported = imported + 1
+
+ logger.info('%d rows imported, %d rows rejected', imported, len(rejected))
+
diff --git a/setup.py b/setup.py
index 37e46b9d..9c917c96 100644
--- a/setup.py
+++ b/setup.py
@@ -41,5 +41,10 @@ setup(name='political-memory',
'pytest-cov==2.2.0',
'codecov',
]
+ },
+ entry_points={
+ 'console_scripts': [
+ 'memopol_import_recommendations = representatives_recommendations.contrib.import_recommendations:main'
+ ]
}
)
--
GitLab