diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000000000000000000000000000000000000..85b23c96b57c772170faa712c3d9b51c0df63f80
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,4 @@
+[run]
+omit =
+ representatives_positions/contrib/*
+ representatives_recommendations/contrib/*
diff --git a/memopol/settings.py b/memopol/settings.py
index 1cf752ae456ded9de3fc5e7753f0f4c17ea8031b..ffdd506af1839ddc01a8eb5169ecb969ac33a6a9 100644
--- a/memopol/settings.py
+++ b/memopol/settings.py
@@ -258,6 +258,14 @@ LOGGING = {
'handlers': ['console'],
'level': LOG_LEVEL,
},
+ 'representatives_positions': {
+ 'handlers': ['console'],
+ 'level': LOG_LEVEL
+ },
+ 'representatives_recommendations': {
+ 'handlers': ['console'],
+ 'level': LOG_LEVEL
+ },
'representatives_votes': {
'handlers': ['console'],
'level': LOG_LEVEL,
diff --git a/pytest.ini b/pytest.ini
index 0ea23da77840032793af4df2f96df5462590ac5e..246beef238daadfd1c2652e6a64dbd86fa3a45dc 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,3 +1,3 @@
[pytest]
DJANGO_SETTINGS_MODULE=memopol.settings
-addopts = --cov=. --create-db
+addopts = --cov-config .coveragerc --cov=. --create-db
diff --git a/representatives_positions/contrib/__init__.py b/representatives_positions/contrib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/representatives_positions/contrib/import_positions.py b/representatives_positions/contrib/import_positions.py
new file mode 100644
index 0000000000000000000000000000000000000000..95c1936bd1f7da8dc4d8b5957de8a1a3b976d317
--- /dev/null
+++ b/representatives_positions/contrib/import_positions.py
@@ -0,0 +1,107 @@
+# coding: utf-8
+
+import csv
+import django
+from django.apps import apps
+import logging
+import sys
+import re
+
+from representatives_positions.models import Position
+from representatives.models import Representative
+
+logger = logging.getLogger(__name__)
+
+
+class PositionImporter:
+ def __init__(self):
+ self.rep_cache = {}
+
+ def get_rep(self, first_name, last_name):
+ key = '%s %s' % (first_name, last_name)
+ rep = self.rep_cache.get(key, None)
+
+ if rep is None:
+ try:
+ rep = Representative.objects.get(first_name=first_name,
+ last_name=last_name)
+ self.rep_cache[key] = rep
+ except Representative.DoesNotExist:
+ rep = None
+
+ return rep
+
+ def import_row(self, row):
+ if len(row['date']) == 0:
+ logger.warn('Cannot import dateless position for %s %s on URL %s' %
+ (row['first_name'], row['last_name'], row['url']))
+ return False
+
+ rep = self.get_rep(row['first_name'], row['last_name'])
+ if rep is None:
+ logger.warn('Could not find rep %s %s' % (row['first_name'],
+ row['last_name']))
+ return False
+
+ text = re.sub('(^<p>|</p>$)', '', row['content'])
+ if row['title'] is not None and len(row['title']) > 0:
+ text = '%s\n%s' % (row['title'], text)
+
+ try:
+ position = Position.objects.get(representative=rep,
+ link=row['url'])
+ except Position.DoesNotExist:
+ position = Position(
+ representative=rep,
+ link=row['url'],
+ datetime=row['date'],
+ text=text,
+ published=True
+ )
+ position.save()
+ logger.info('Created position for %s %s on URL %s' % (
+ row['first_name'], row['last_name'], row['url']))
+
+ return True
+
+
+def main(stream=None):
+ """
+ Imports positions from an old memopol instance.
+
+ Usage:
+ cat positions.csv | memopol_import_positions
+
+ The input CSV file should be generated by the following query:
+ SELECT CONCAT(o.content, '|', o.url, '|', o.title, '|', ro.date, '|',
+ r.first_name, '|', r.last_name)
+ FROM reps_opinion o
+ INNER JOIN reps_opinionrep ro ON ro.opinion_id = o.id
+ INNER JOIN reps_representative r ON r.id = ro.representative_id
+ WHERE o.institution='EU'
+
+ """
+
+ if not apps.ready:
+ django.setup()
+
+ importer = PositionImporter()
+ rejected = []
+ imported = 0
+
+ reader = csv.DictReader(stream or sys.stdin, delimiter='|', fieldnames=[
+ 'content',
+ 'url',
+ 'title',
+ 'date',
+ 'first_name',
+ 'last_name'
+ ], quoting=csv.QUOTE_NONE)
+
+ for row in reader:
+ if not importer.import_row(row):
+ rejected.append(row)
+ else:
+ imported = imported + 1
+
+ logger.info('%d rows imported, %d rows rejected', imported, len(rejected))
diff --git a/representatives_positions/models.py b/representatives_positions/models.py
index 491622e5b8fd92febd0b6670ee85469c9c86621e..1e68c8c977b674ab2a79d314dfb86967a925a7a5 100644
--- a/representatives_positions/models.py
+++ b/representatives_positions/models.py
@@ -2,10 +2,11 @@ from django.db import models
from django.core.urlresolvers import reverse
from django.template.defaultfilters import truncatewords
from taggit.managers import TaggableManager
+from representatives.models import Representative
class Position(models.Model):
- representative = models.ForeignKey('representatives.representative',
+ representative = models.ForeignKey(Representative,
related_name='positions')
datetime = models.DateField()
text = models.TextField()
diff --git a/representatives_recommendations/contrib/__init__.py b/representatives_recommendations/contrib/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/representatives_recommendations/contrib/import_data.py b/representatives_recommendations/contrib/import_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d3c3df00f4713d6a03106146d9ec4aed05cfad6
--- /dev/null
+++ b/representatives_recommendations/contrib/import_data.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+# flake8: noqa
+
+dossier_mappings = {
+ # Patent law: patentability of computer-implemented inventions
+ "Directive on patentability of \"computer-implemented inventions\" (software patents), 1st reading":
+ "2002/0047(COD)",
+
+ # Criminal measures aimed at ensuring the enforcement of intellectual property rights
+ "Criminal measures aimed at ensuring the enforcement of intellectual property rights (IPRED 2), 1st reading":
+ "2005/0127(COD)",
+
+ # Electronic communications: common regulatory framework for networks and services, access, interconnection and authorisation ['Telecoms Package' (amend. Directives 2002/19/EC, 2002/20/EC and 2002/21/EC)]
+ "Directives reforming the EU's regulatory framework for electronic communications networks and services (telecoms package), 1st reading":
+ "2007/0247(COD)",
+
+ "Directives reforming the EU's regulatory framework for electronic communications networks and services (telecoms package), 2nd reading":
+ "2007/0247(COD)",
+
+ # Cultural industries in Europe
+ "Rapport Bono on cultural industries in Europe":
+ "2007/2153(INI)",
+
+ # Strengthening security and fundamental freedoms on the Internet
+ "Rapport Lambrinidis on strengthening security and fundamental freedoms on the Internet":
+ "2008/2160(INI)",
+
+ # Enforcement of intellectual property rights in the internal market
+ "Rapport Gallo on enforcement of intellectual property rights in the internal market":
+ "2009/2178(INI)",
+
+ # Resolution on the Anti-Counterfeiting Trade Agreement (ACTA)
+ "Resolution on Anti-Counterfeiting Trade Agreement (ACTA)":
+ "2010/2935(RSP)",
+
+ # Enhanced cooperation in the area of the creation of unitary patent protection: implementation
+ "A7-0001/2012":
+ "2011/0093(COD)",
+
+ # EU/Australia, Canada, Japan, Korea, Mexico, Morocco, New Zealand, Singapore, Switzerland and United States Anti-Counterfeiting Trade Agreement (ACTA)
+ "A7-0204/2012":
+ "2011/0167(NLE)",
+}
+
+resolutions = [
+ u'résolution législative',
+ u'résolution',
+ 'legislative resolution',
+ 'resolution'
+]
diff --git a/representatives_recommendations/contrib/import_recommendations.py b/representatives_recommendations/contrib/import_recommendations.py
new file mode 100644
index 0000000000000000000000000000000000000000..c056afdd8d3f9191594e93b72d40a945cfe59d9a
--- /dev/null
+++ b/representatives_recommendations/contrib/import_recommendations.py
@@ -0,0 +1,131 @@
+# coding: utf-8
+
+import csv
+import django
+from django.apps import apps
+import logging
+import sys
+
+from representatives_recommendations.models import Recommendation
+from representatives_votes.models import Dossier, Proposal
+
+from .import_data import dossier_mappings, resolutions
+
+logger = logging.getLogger(__name__)
+
+
+class RecommendationImporter:
+ def __init__(self):
+ self.dossier_cache = {}
+
+ def get_dossier(self, title):
+ dossier = self.dossier_cache.get(title, None)
+
+ if dossier is None:
+ ref = dossier_mappings.get(title, None)
+ if ref is not None:
+ query = {'reference': ref}
+ else:
+ query = {'title__iexact': title}
+
+ try:
+ dossier = Dossier.objects.get(**query)
+ self.dossier_cache[title] = dossier
+ except Dossier.DoesNotExist:
+ dossier = None
+
+ return dossier
+
+ def get_proposal(self, dossier, kind):
+ kinds = [kind]
+
+ try:
+ resolutions.index(kind.lower())
+ kinds.extend(resolutions)
+ except ValueError:
+ pass
+
+ for k in kinds:
+ try:
+ return Proposal.objects.get(dossier=dossier, kind__iexact=k)
+ except Proposal.DoesNotExist:
+ continue
+
+ return None
+
+ def import_row(self, row):
+ dossier = self.get_dossier(row['title'])
+ if dossier is None:
+ logger.warn('No dossier "%s"' % row['title'])
+ return False
+
+ proposal = self.get_proposal(dossier, row['part'])
+ if proposal is None:
+ logger.warn('No proposal "%s" for dossier %s (%d): "%s"' % (
+ row['part'].decode('utf-8'), dossier.reference, dossier.pk,
+ row['title']))
+ return False
+
+ weight = int(row['weight']) * int(row['ponderation'])
+ descr = row['description'].strip()
+ if len(descr) == 0:
+ descr = '%s on %s' % (row['part'], dossier.reference)
+
+ try:
+ recom = Recommendation.objects.get(proposal=proposal)
+ except Recommendation.DoesNotExist:
+ recom = Recommendation(
+ proposal=proposal,
+ recommendation=row['recommendation'],
+ title=descr,
+ weight=weight
+ )
+ recom.save()
+ logger.info('Created recommendation with weight %s for %s: %s' % (
+ weight,
+ row['title'],
+ row['part']
+ ))
+
+ return True
+
+
+def main(stream=None):
+ """
+ Imports recommendations from an old memopol instance.
+
+ Usage:
+ cat recommendations.csv | memopol_import_recommendations
+
+ The input CSV file should be generated by the following query:
+ SELECT CONCAT(r.description, '|', r.weight, '|', r.recommendation, '|',
+ r.part, '|', p.title, '|', p.ponderation)
+ FROM votes_recommendation r
+ LEFT JOIN votes_proposal p ON r.proposal_id = p.id
+ WHERE p.institution = 'EU'
+
+ """
+
+ if not apps.ready:
+ django.setup()
+
+ importer = RecommendationImporter()
+ rejected = []
+ imported = 0
+
+ reader = csv.DictReader(stream or sys.stdin, delimiter='|', fieldnames=[
+ 'description',
+ 'weight',
+ 'recommendation',
+ 'part',
+ 'title',
+ 'ponderation'
+ ], quoting=csv.QUOTE_NONE)
+
+ for row in reader:
+ if not importer.import_row(row):
+ rejected.append(row)
+ else:
+ imported = imported + 1
+
+ logger.info('%d rows imported, %d rows rejected', imported, len(rejected))
diff --git a/setup.py b/setup.py
index 37e46b9d4c52ef983298a61e2a754278bad3ecb4..307e63f398ecf44200400bb92c863733eb7e5434 100644
--- a/setup.py
+++ b/setup.py
@@ -41,5 +41,11 @@ setup(name='political-memory',
'pytest-cov==2.2.0',
'codecov',
]
+ },
+ entry_points={
+ 'console_scripts': [
+ 'memopol_import_positions = representatives_positions.contrib.import_positions:main', # noqa
+ 'memopol_import_recommendations = representatives_recommendations.contrib.import_recommendations:main', # noqa
+ ]
}
)