From 04c8dffaf0bc9b468d1f7b0c226615e27449f564 Mon Sep 17 00:00:00 2001
From: jpic <jamespic@gmail.com>
Date: Wed, 18 Nov 2015 21:59:32 +0100
Subject: [PATCH] Added parltrack import commands in contrib
Also:
- code was PEP8'ed
- tests were added,
- untested code was removed,
- setup.py was fixed and requirements.txt is gone,
---
.coveragerc | 3 +
.travis.yml | 20 +-
COPYING => LICENSE | 0
MANIFEST.in | 2 +
README.md | 1 +
pytest.ini | 3 +
representatives_votes/admin.py | 27 +-
.../{management => contrib}/__init__.py | 0
.../parltrack}/__init__.py | 0
.../contrib/parltrack/import_dossiers.py | 56 ++++
.../contrib/parltrack/import_votes.py | 186 ++++++++++++
.../parltrack/tests/dossiers_expected.json | 67 ++++
.../parltrack/tests/dossiers_fixture.json | 118 ++++++++
.../contrib/parltrack/tests/test_import.py | 52 ++++
.../parltrack/tests/votes_expected.json | 112 +++++++
.../parltrack/tests/votes_fixture.json | 285 ++++++++++++++++++
.../parltrack/tests/votes_initial.json | 68 +++++
.../commands/import_dossier_from_toutatis.py | 39 ---
.../commands/import_proposal_from_toutatis.py | 39 ---
.../migrations/0001_initial.py | 2 +-
.../migrations/0002_auto_20150707_1611.py | 16 +-
.../migrations/0003_auto_20150708_1358.py | 6 +-
.../migrations/0004_auto_20150709_0819.py | 13 +-
.../0005_make_dossier_reference_unique.py | 19 ++
.../migrations/0006_duplicates.py | 30 ++
...unique_together_proposal_representative.py | 18 ++
.../migrations/0008_unique_proposal_title.py | 19 ++
representatives_votes/models.py | 22 +-
representatives_votes/serializers.py | 166 ----------
representatives_votes/tasks.py | 100 ------
.../tests}/__init__.py | 0
representatives_votes/tests/settings.py | 57 ++++
representatives_votes/tests/urls.py | 1 +
requirements.txt | 2 -
setup.py | 18 +-
test_project/manage.py | 10 -
test_project/test_project/settings.py | 104 -------
test_project/test_project/urls.py | 21 --
test_project/test_project/wsgi.py | 16 -
39 files changed, 1185 insertions(+), 533 deletions(-)
create mode 100644 .coveragerc
rename COPYING => LICENSE (100%)
create mode 100644 MANIFEST.in
create mode 100644 pytest.ini
rename representatives_votes/{management => contrib}/__init__.py (100%)
rename representatives_votes/{management/commands => contrib/parltrack}/__init__.py (100%)
create mode 100644 representatives_votes/contrib/parltrack/import_dossiers.py
create mode 100644 representatives_votes/contrib/parltrack/import_votes.py
create mode 100644 representatives_votes/contrib/parltrack/tests/dossiers_expected.json
create mode 100644 representatives_votes/contrib/parltrack/tests/dossiers_fixture.json
create mode 100644 representatives_votes/contrib/parltrack/tests/test_import.py
create mode 100644 representatives_votes/contrib/parltrack/tests/votes_expected.json
create mode 100644 representatives_votes/contrib/parltrack/tests/votes_fixture.json
create mode 100644 representatives_votes/contrib/parltrack/tests/votes_initial.json
delete mode 100644 representatives_votes/management/commands/import_dossier_from_toutatis.py
delete mode 100644 representatives_votes/management/commands/import_proposal_from_toutatis.py
create mode 100644 representatives_votes/migrations/0005_make_dossier_reference_unique.py
create mode 100644 representatives_votes/migrations/0006_duplicates.py
create mode 100644 representatives_votes/migrations/0007_vote_unique_together_proposal_representative.py
create mode 100644 representatives_votes/migrations/0008_unique_proposal_title.py
delete mode 100644 representatives_votes/serializers.py
delete mode 100644 representatives_votes/tasks.py
rename {test_project/test_project => representatives_votes/tests}/__init__.py (100%)
create mode 100644 representatives_votes/tests/settings.py
create mode 100644 representatives_votes/tests/urls.py
delete mode 100644 requirements.txt
delete mode 100755 test_project/manage.py
delete mode 100644 test_project/test_project/settings.py
delete mode 100644 test_project/test_project/urls.py
delete mode 100644 test_project/test_project/wsgi.py
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..3e2e9da
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,3 @@
+[run]
+omit = representatives_votes/tests/*
+omit = representatives_votes/migrations/*
diff --git a/.travis.yml b/.travis.yml
index 6f9bd2e..f4e45ee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,10 +1,20 @@
sudo: false
language: python
+env:
+- DJANGO="django>1.8,<1.9" DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
python:
- - "2.7"
+- "2.7"
+before_install:
+- pip install codecov
install:
- - pip install django
- - pip install -e git+https://github.com/political-memory/django-representatives.git#egg=representatives
- - pip install -e .
+- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov
+- pip install https://github.com/political-memory/django-representatives/archive/parltrack.tar.gz#egg=django-representatives
+- pip install -e .
script:
- - test_project/manage.py migrate
+- django-admin migrate
+- flake8 representatives_votes/ --exclude migrations --ignore E128
+- py.test
+- cat representatives_votes/contrib/parltrack/tests/dossiers_fixture.json | parltrack_import_dossiers
+- cat representatives_votes/contrib/parltrack/tests/votes_fixture.json | parltrack_import_votes
+after_success:
+- codecov
diff --git a/COPYING b/LICENSE
similarity index 100%
rename from COPYING
rename to LICENSE
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..702ef56
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.rst *.txt README LICENSE AUTHORS CHANGELOG
+recursive-include representatives_votes *.html *.css *.js *.py *.po *.mo *.json *.png *.gif
diff --git a/README.md b/README.md
index c366f37..e4acb9b 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,2 @@
[](https://travis-ci.org/political-memory/django-representatives-votes)
+[](https://codecov.io/github/political-memory/django-representatives-votes?branch=master)
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..77a3ed6
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
+addopts = --cov=representatives_votes --create-db
diff --git a/representatives_votes/admin.py b/representatives_votes/admin.py
index 6d185c4..8f17f7a 100644
--- a/representatives_votes/admin.py
+++ b/representatives_votes/admin.py
@@ -1,6 +1,7 @@
# coding: utf-8
from django.contrib import admin
+
from .models import Dossier, Proposal, Vote
@@ -10,8 +11,19 @@ class DossierAdmin(admin.ModelAdmin):
class ProposalAdmin(admin.ModelAdmin):
- list_display = ('id', 'fingerprint', 'reference', 'dossier_reference', 'title', 'datetime', 'kind', 'total_abstain', 'total_against', 'total_for')
+ list_display = (
+ 'id',
+ 'fingerprint',
+ 'reference',
+ 'dossier_reference',
+ 'title',
+ 'datetime',
+ 'kind',
+ 'total_abstain',
+ 'total_against',
+ 'total_for')
search_fields = ('reference', 'dossier__reference', 'title', 'fingerprint')
+
def dossier_reference(self, obj):
return obj.dossier.reference
@@ -22,18 +34,23 @@ class NoneMatchingFilter(admin.SimpleListFilter):
def lookups(self, request, model_admin):
return [('None', 'Unknown')]
-
+
def queryset(self, request, queryset):
if self.value() == 'None':
return queryset.filter(representative=None)
else:
- return queryset
+ return queryset
class VoteAdmin(admin.ModelAdmin):
- list_display = ('id', 'proposal_reference', 'position', 'representative', 'representative_name')
+ list_display = (
+ 'id',
+ 'proposal_reference',
+ 'position',
+ 'representative',
+ 'representative_name')
list_filter = (NoneMatchingFilter,)
-
+
def proposal_reference(self, obj):
return obj.proposal.reference
diff --git a/representatives_votes/management/__init__.py b/representatives_votes/contrib/__init__.py
similarity index 100%
rename from representatives_votes/management/__init__.py
rename to representatives_votes/contrib/__init__.py
diff --git a/representatives_votes/management/commands/__init__.py b/representatives_votes/contrib/parltrack/__init__.py
similarity index 100%
rename from representatives_votes/management/commands/__init__.py
rename to representatives_votes/contrib/parltrack/__init__.py
diff --git a/representatives_votes/contrib/parltrack/import_dossiers.py b/representatives_votes/contrib/parltrack/import_dossiers.py
new file mode 100644
index 0000000..e688633
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/import_dossiers.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+import logging
+import sys
+
+import ijson
+import django
+from django.apps import apps
+
+from representatives_votes.models import Dossier
+
+logger = logging.getLogger(__name__)
+
+URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz'
+LOCAL_PATH = 'ep_dossiers.json.xz'
+
+
+def parse_dossier_data(data):
+ """Parse data from parltarck dossier export (1 dossier) Update dossier
+ if it existed before, this function goal is to import and update a
+ dossier, not to import all parltrack data
+ """
+ changed = False
+ ref = data['procedure']['reference']
+
+ logger.debug('Processing dossier %s', ref)
+
+ try:
+ dossier = Dossier.objects.get(reference=ref)
+ except Dossier.DoesNotExist:
+ dossier = Dossier(reference=ref)
+ logger.debug('Dossier did not exist')
+ changed = True
+
+ if dossier.title != data['procedure']['title']:
+ logger.debug('Title changed from "%s" to "%s"', dossier.title,
+ data['procedure']['title'])
+ dossier.title = data['procedure']['title']
+ changed = True
+
+ source = data['meta']['source'].replace('&l=en', '')
+ if dossier.link != source:
+ logger.debug('Source changed from "%s" to "%s"', dossier.link, source)
+ dossier.link = source
+ changed = True
+
+ if changed:
+ logger.info('Updated dossier %s', ref)
+ dossier.save()
+
+
+def main(stream=None):
+ if not apps.ready:
+ django.setup()
+
+ for data in ijson.items(stream or sys.stdin, 'item'):
+ parse_dossier_data(data)
diff --git a/representatives_votes/contrib/parltrack/import_votes.py b/representatives_votes/contrib/parltrack/import_votes.py
new file mode 100644
index 0000000..068830d
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/import_votes.py
@@ -0,0 +1,186 @@
+# coding: utf-8
+import logging
+import sys
+from os.path import join
+
+import django.dispatch
+import ijson
+import django
+from django.apps import apps
+from dateutil.parser import parse as date_parse
+from django.db import transaction
+from django.utils.timezone import make_aware as date_make_aware
+from pytz import timezone as date_timezone
+
+from representatives.models import Representative
+from representatives_votes.models import Dossier, Proposal, Vote
+
+
+logger = logging.getLogger(__name__)
+vote_pre_import = django.dispatch.Signal(providing_args=['vote_data'])
+
+
+def _parse_date(date_str):
+ return date_make_aware(
+ date_parse(date_str),
+ date_timezone('Europe/Brussels'))
+
+JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
+DESTINATION = join('/tmp', 'ep_votes.json')
+
+
+class Command(object):
+ def init_cache(self):
+ self.cache = dict()
+ self.index_representatives()
+ self.index_dossiers()
+
+ def parse_vote_data(self, vote_data):
+ """
+ Parse data from parltrack votes db dumps (1 proposal)
+ """
+ if 'epref' not in vote_data.keys():
+ logger.debug('Could not import data without epref %s',
+ vote_data['title'])
+ return
+
+ dossier_pk = self.get_dossier(vote_data['epref'])
+
+ if not dossier_pk:
+ logger.debug('Cannot find dossier with remote id %s',
+ vote_data['epref'])
+ return
+
+ return self.parse_proposal_data(
+ proposal_data=vote_data,
+ dossier_pk=dossier_pk
+ )
+
+ @transaction.atomic
+ def parse_proposal_data(self, proposal_data, dossier_pk):
+ """Get or Create a proposal model from raw data"""
+ proposal_display = '{} ({})'.format(proposal_data['title'].encode(
+ 'utf-8'), proposal_data.get('report', '').encode('utf-8'))
+
+ if 'issue_type' not in proposal_data.keys():
+ logger.debug('This proposal data without issue_type: %s',
+ proposal_data['epref'])
+ return
+
+ changed = False
+ try:
+ proposal = Proposal.objects.get(title=proposal_data['title'])
+ except Proposal.DoesNotExist:
+ proposal = Proposal(title=proposal_data['title'])
+ changed = True
+
+ data_map = dict(
+ title=proposal_data['title'],
+ datetime=_parse_date(proposal_data['ts']),
+ dossier_id=dossier_pk,
+ reference=proposal_data.get('report'),
+ kind=proposal_data.get('issue_type')
+ )
+
+ for position in ('For', 'Abstain', 'Against'):
+ position_data = proposal_data.get(position, {})
+ position_total = position_data.get('total', 0)
+
+ if isinstance(position_total, str) and position_total.isdigit():
+ position_total = int(position_total)
+
+ data_map['total_%s' % position.lower()] = position_total
+
+ for key, value in data_map.items():
+ if value != getattr(proposal, key, None):
+ setattr(proposal, key, value)
+ changed = True
+
+ if changed:
+ proposal.save()
+
+ responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
+
+ for receiver, response in responses:
+ if response is False:
+ logger.debug(
+ 'Skipping dossier %s', proposal_data.get(
+ 'epref', proposal_data['title']))
+ return
+
+ positions = ['For', 'Abstain', 'Against']
+ logger.info(
+ 'Looking for votes in proposal {}'.format(proposal_display))
+ for position in positions:
+ for group_vote_data in proposal_data.get(
+ position,
+ {}).get(
+ 'groups',
+ {}):
+ for vote_data in group_vote_data['votes']:
+ if not isinstance(vote_data, dict):
+ logger.error('Skipping vote data %s for proposal %s',
+ vote_data, proposal_data['_id'])
+ continue
+
+ representative_pk = self.get_representative(vote_data)
+
+ if representative_pk is None:
+ logger.error('Could not find mep for %s', vote_data)
+ continue
+
+ representative_name = vote_data.get('orig', '')
+
+ changed = False
+ try:
+ vote = Vote.objects.get(
+ representative_id=representative_pk,
+ proposal_id=proposal.pk)
+ except Vote.DoesNotExist:
+ vote = Vote(proposal_id=proposal.pk,
+ representative_id=representative_pk)
+ changed = True
+
+ if vote.position != position.lower():
+ changed = True
+ vote.position = position.lower()
+
+ if vote.representative_name != representative_name:
+ changed = True
+ vote.representative_name = representative_name
+
+ if changed:
+ vote.save()
+ logger.debug('Save vote %s for MEP %s on %s #%s to %s',
+ vote.pk, representative_pk, proposal_data['title'],
+ proposal.pk, position)
+
+ return proposal
+
+ def index_dossiers(self):
+ self.cache['dossiers'] = {
+ d[0]: d[1] for d in Dossier.objects.values_list('reference', 'pk')
+ }
+
+ def get_dossier(self, reference):
+ return self.cache['dossiers'].get(reference, None)
+
+ def index_representatives(self):
+ self.cache['meps'] = {int(l[0]): l[1] for l in
+ Representative.objects.values_list('remote_id', 'pk')}
+
+ def get_representative(self, vote_data):
+ if vote_data.get('ep_id', None) is None:
+ return
+ return self.cache['meps'].get(int(vote_data['ep_id']), None)
+
+
+def main(stream=None):
+ if not apps.ready:
+ django.setup()
+
+ command = Command()
+ command.init_cache()
+
+ for vote_data in ijson.items(stream or sys.stdin, 'item'):
+ command.parse_vote_data(vote_data)
diff --git a/representatives_votes/contrib/parltrack/tests/dossiers_expected.json b/representatives_votes/contrib/parltrack/tests/dossiers_expected.json
new file mode 100644
index 0000000..1cb5827
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/dossiers_expected.json
@@ -0,0 +1,67 @@
+[
+{
+ "fields": {
+ "updated": "2015-12-13T10:11:31.369Z",
+ "reference": "2012/2002(INI)",
+ "title": "Agenda for change: the future of EU development policy",
+ "text": "",
+ "created": "2015-12-13T10:11:31.369Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+ "fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 1
+},
+{
+ "fields": {
+ "updated": "2015-12-13T10:11:31.378Z",
+ "reference": "2015/2132(BUD)",
+ "title": "2016 general budget: all sections",
+ "text": "",
+ "created": "2015-12-13T10:11:31.378Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
+ "fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 2
+},
+{
+ "fields": {
+ "updated": "2015-12-13T10:11:31.388Z",
+ "reference": "2013/2857(DEA)",
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "text": "",
+ "created": "2015-12-13T10:11:31.388Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)",
+ "fingerprint": "50745a1a6e47b8db097c55ef21a4f11fc1ef0d97"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 3
+},
+{
+ "fields": {
+ "updated": "2015-12-13T10:11:31.398Z",
+ "reference": "2015/2623(DEA)",
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "text": "",
+ "created": "2015-12-13T10:11:31.398Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)",
+ "fingerprint": "65fce7af2e020ff58849d1663f2c30ab0b1a35db"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 4
+},
+{
+ "fields": {
+ "updated": "2015-12-13T10:11:31.408Z",
+ "reference": "2009/0051(COD)",
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "text": "",
+ "created": "2015-12-13T10:11:31.408Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)",
+ "fingerprint": "f2abba201c10df8a7cec1d734f91b7988eec8260"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 5
+}
+]
diff --git a/representatives_votes/contrib/parltrack/tests/dossiers_fixture.json b/representatives_votes/contrib/parltrack/tests/dossiers_fixture.json
new file mode 100644
index 0000000..2ac515c
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/dossiers_fixture.json
@@ -0,0 +1,118 @@
+[
+ {
+ "meta": {
+ "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+ "updated": "2012-10-25T02:02:36.882000"
+ },
+ "procedure": {
+ "dossier_of_the_committee": "DEVE/7/08564",
+ "legal_basis": [
+ "Rules of Procedure of the European Parliament EP 048"
+ ],
+ "reference": "2012/2002(INI)",
+ "stage_reached": "Procedure completed",
+ "subject": [
+ "6.30 Development cooperation"
+ ],
+ "subtype": "Initiative",
+ "title": "Agenda for change: the future of EU development policy",
+ "type": "INI - Own-initiative procedure"
+ }
+ },
+ {
+ "meta": {
+ "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)&l=en",
+ "updated": "2015-11-18T02:23:18.933000"
+ },
+ "procedure": {
+ "dossier_of_the_committee": "BUDE/8/04900;BUDG/8/03789",
+ "reference": "2015/2132(BUD)",
+ "stage_reached": "Budgetary conciliation committee convened",
+ "subject": [
+ "8.70.56 2016 budget"
+ ],
+ "subtype": "Budget",
+ "title": "2016 general budget: all sections",
+ "type": "BUD - Budgetary procedure"
+ }
+ },
+ {
+ "meta": {
+ "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)&l=en",
+ "updated": "2014-12-06T03:56:21.811000"
+ },
+ "procedure": {
+ "dossier_of_the_committee": "PECH/7/14126",
+ "geographical_area": [
+ "Atlantic Ocean area"
+ ],
+ "reference": "2013/2857(DEA)",
+ "stage_reached": "Awaiting Council decision on delegated act",
+ "subject": [
+ "3.15.01 Fish stocks, conservation of fishery resources",
+ "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+ "3.15.15 Fisheries agreements and cooperation"
+ ],
+ "subtype": "Examination of delegated act",
+ "summary": [
+ "Supplementing"
+ ],
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "type": "DEA - Delegated acts procedure"
+ }
+ },
+ {
+ "meta": {
+ "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)&l=en",
+ "updated": "2015-07-22T00:46:11.603000"
+ },
+ "procedure": {
+ "dossier_of_the_committee": "PECH/8/03020",
+ "reference": "2015/2623(DEA)",
+ "stage_reached": "Awaiting Council decision on delegated act",
+ "subject": [
+ "3.15.01 Fish stocks, conservation of fishery resources",
+ "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+ "3.15.15 Fisheries agreements and cooperation"
+ ],
+ "subtype": "Examination of delegated act",
+ "summary": [
+ "Supplementing"
+ ],
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "type": "DEA - Delegated acts procedure"
+ }
+ },
+ {
+ "meta": {
+ "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)&l=en",
+ "updated": "2015-11-04T17:01:33.812000"
+ },
+ "procedure": {
+ "Modified legal basis": "Rules of Procedure of the European Parliament EP 150",
+ "dossier_of_the_committee": "PECH/7/00288",
+ "final": {
+ "title": "Regulation 2010/1236",
+ "url": "http://eur-lex.europa.eu/smartapi/cgi/sga_doc?smartapi!celexplus!prod!CELEXnumdoc&lg=EN&numdoc=32010R1236"
+ },
+ "instrument": "Regulation",
+ "legal_basis": [
+ "Treaty on the Functioning of the EU TFEU 043-p2"
+ ],
+ "reference": "2009/0051(COD)",
+ "stage_reached": "Procedure completed",
+ "subject": [
+ "3.15.01 Fish stocks, conservation of fishery resources",
+ "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+ "3.15.15 Fisheries agreements and cooperation"
+ ],
+ "subtype": "Legislation",
+ "summary": [
+ "Amended by",
+ "Repealing Regulation (EC) No 2791/1999"
+ ],
+ "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+ "type": "COD - Ordinary legislative procedure (ex-codecision procedure)"
+ }
+ }
+]
diff --git a/representatives_votes/contrib/parltrack/tests/test_import.py b/representatives_votes/contrib/parltrack/tests/test_import.py
new file mode 100644
index 0000000..1ea8182
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/test_import.py
@@ -0,0 +1,52 @@
+import pytest
+import os
+import copy
+
+from django.core.serializers.json import Deserializer
+from django.core.management import call_command
+from representatives_votes.contrib.parltrack import import_dossiers
+from representatives_votes.contrib.parltrack import import_votes
+from representatives_votes.models import Dossier, Proposal, Vote
+from representatives.models import Representative
+from representatives.contrib import parltrack
+
+
+def _test_import(scenario, callback):
+ fixture = os.path.join(os.path.dirname(__file__),
+ '%s_fixture.json' % scenario)
+ expected = os.path.join(os.path.dirname(__file__),
+ '%s_expected.json' % scenario)
+
+ # Disable django auto fields
+ exclude = ('id', '_state', 'created', 'updated')
+
+ with open(fixture, 'r') as f:
+ callback(f)
+
+ with open(expected, 'r') as f:
+ for obj in Deserializer(f.read()):
+ compare = copy.copy(obj.object.__dict__)
+
+ for f in exclude:
+ if f in compare:
+ compare.pop(f)
+
+ type(obj.object).objects.get(**compare)
+
+
+@pytest.mark.django_db
+def test_parltrack_import_dossiers():
+ _test_import('dossiers', import_dossiers.main)
+
+
+@pytest.mark.django_db
+def test_parltrack_import_votes():
+ for model in (Representative, Dossier, Proposal, Vote):
+ model.objects.all().delete()
+
+ call_command('loaddata', os.path.join(os.path.abspath(
+ parltrack.__path__[0]), 'tests', 'representatives_expected.json'))
+ call_command('loaddata', os.path.join(os.path.dirname(__file__),
+ 'dossiers_expected.json'))
+
+ _test_import('votes', import_votes.main)
diff --git a/representatives_votes/contrib/parltrack/tests/votes_expected.json b/representatives_votes/contrib/parltrack/tests/votes_expected.json
new file mode 100644
index 0000000..5ab0956
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/votes_expected.json
@@ -0,0 +1,112 @@
+[
+{
+ "fields": {
+ "representative_name": "",
+ "position": "abstain",
+ "proposal": 1,
+ "representative": 2
+ },
+ "model": "representatives_votes.vote",
+ "pk": 1
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "abstain",
+ "proposal": 1,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 2
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "for",
+ "proposal": 2,
+ "representative": 2
+ },
+ "model": "representatives_votes.vote",
+ "pk": 3
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "for",
+ "proposal": 2,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 4
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "against",
+ "proposal": 3,
+ "representative": 2
+ },
+ "model": "representatives_votes.vote",
+ "pk": 5
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "against",
+ "proposal": 3,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 6
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "for",
+ "proposal": 4,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 7
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "against",
+ "proposal": 4,
+ "representative": 2
+ },
+ "model": "representatives_votes.vote",
+ "pk": 8
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "abstain",
+ "proposal": 5,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 9
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "against",
+ "proposal": 5,
+ "representative": 2
+ },
+ "model": "representatives_votes.vote",
+ "pk": 10
+},
+{
+ "fields": {
+ "representative_name": "",
+ "position": "for",
+ "proposal": 6,
+ "representative": 1
+ },
+ "model": "representatives_votes.vote",
+ "pk": 11
+}
+]
diff --git a/representatives_votes/contrib/parltrack/tests/votes_fixture.json b/representatives_votes/contrib/parltrack/tests/votes_fixture.json
new file mode 100644
index 0000000..cabfcbe
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/votes_fixture.json
@@ -0,0 +1,285 @@
+[
+ {
+ "Abstain": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 96673,
+ "name": "Ludvigsson",
+ "userid": 5860
+ },
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "2"
+ },
+ "_id": "56617c15ecc52ed712fc7901",
+ "dossierid": "4f33d5ddb819f2756a000009",
+ "epref": "2012/2002(INI)",
+ "eptitle": "Agenda for change: the future of EU development policy",
+ "issue_type": "\u00a7 31",
+ "rapporteur": [
+ {
+ "name": "Charles Goerens",
+ "ref": 840
+ }
+ ],
+ "report": "A7-0234/2012",
+ "title": "A7-0234/2012 - Charles Goerens - \u00a7 31",
+ "ts": "2012-10-23T18:31:10",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2012/10-23/votes_nominaux/xml/P7_PV(2012)10-23(RCV)_XC.xml",
+ "voteid": "17393"
+ },
+ {
+ "For": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 96673,
+ "name": "Ludvigsson",
+ "userid": 5860
+ }
+ ]
+ },
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "2"
+ },
+ "_id": "56617c15ecc52ed712fc7902",
+ "dossierid": "4f33d5ddb819f2756a000009",
+ "epref": "2012/2002(INI)",
+ "eptitle": "Agenda for change: the future of EU development policy",
+ "issue_type": "R\u00e9solution",
+ "rapporteur": [
+ {
+ "name": "Charles Goerens",
+ "ref": 840
+ }
+ ],
+ "report": "A7-0234/2012",
+ "title": "A7-0234/2012 - Charles Goerens - R\u00e9solution",
+ "ts": "2012-10-23T18:34:32",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2012/10-23/votes_nominaux/xml/P7_PV(2012)10-23(RCV)_XC.xml",
+ "voteid": "17394"
+ },
+ {
+ "Against": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 96673,
+ "name": "Ludvigsson",
+ "userid": 5860
+ }
+ ]
+ },
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "2"
+ },
+ "_id": "56617a11ecc52ed712fc6b14",
+ "dossierid": "55a4580ed1d1c57c1e000002",
+ "epref": "2015/2132(BUD)",
+ "eptitle": "2016 general budget: all sections",
+ "issue_type": "Am 4",
+ "rapporteur": [
+ {
+ "name": "Jos\u00e9 Manuel Fernandes",
+ "ref": 96899
+ },
+ {
+ "name": " G\u00e9rard Deprez",
+ "ref": 1473
+ }
+ ],
+ "report": "A8-0298/2015",
+ "title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 4",
+ "ts": "2015-10-28T12:59:35",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
+ "voteid": "59145"
+ },
+ {
+ "For": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "1"
+ },
+ "Against": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 96673,
+ "name": "Ludvigsson",
+ "userid": 5860
+ }
+ ]
+ }
+ ],
+ "total": "1"
+ },
+ "_id": "56617a11ecc52ed712fc6b15",
+ "dossierid": "55a4580ed1d1c57c1e000002",
+ "epref": "2015/2132(BUD)",
+ "eptitle": "2016 general budget: all sections",
+ "issue_type": "Am 29",
+ "rapporteur": [
+ {
+ "name": "Jos\u00e9 Manuel Fernandes",
+ "ref": 96899
+ },
+ {
+ "name": " G\u00e9rard Deprez",
+ "ref": 1473
+ }
+ ],
+ "report": "A8-0298/2015",
+ "title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 29",
+ "ts": "2015-10-28T13:00:12",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
+ "voteid": "59146"
+ },
+ {
+ "Abstain": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "1"
+ },
+ "Against": {
+ "groups": [
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 96673,
+ "name": "Ludvigsson",
+ "userid": 5860
+ }
+ ]
+ }
+ ],
+ "total": "1"
+ },
+ "_id": "56617a11ecc52ed712fc6b16",
+ "dossierid": "55a4580ed1d1c57c1e000002",
+ "epref": "2015/2132(BUD)",
+ "eptitle": "2016 general budget: all sections",
+ "issue_type": "Am 31",
+ "rapporteur": [
+ {
+ "name": "Jos\u00e9 Manuel Fernandes",
+ "ref": 96899
+ },
+ {
+ "name": " G\u00e9rard Deprez",
+ "ref": 1473
+ }
+ ],
+ "report": "A8-0298/2015",
+ "title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 31",
+ "ts": "2015-10-28T13:00:42",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
+ "voteid": "59147"
+ },
+ {
+ "For": {
+ "groups": [
+ {
+ "group": "PPED",
+ "votes": [
+ {
+ "ep_id": 12323196673,
+ "name": "FAILLLLL",
+ "userid": 5860
+ },
+ {
+ "name": "FAILLLLL",
+ "userid": 5860
+ }
+ ]
+ },
+ {
+ "group": "PPE",
+ "votes": [
+ {
+ "ep_id": 2307,
+ "name": "Pirker",
+ "userid": 4611
+ }
+ ]
+ }
+ ],
+ "total": "2"
+ },
+ "_id": "56617a11ecc52ed712fc6b17",
+ "dossierid": "55a4580ed1d1c57c1e000002",
+ "epref": "2015/2132(BUD)",
+ "eptitle": "2016 general budget: all sections",
+ "issue_type": "Am 30",
+ "rapporteur": [
+ {
+ "name": "Jos\u00e9 Manuel Fernandes",
+ "ref": 96899
+ },
+ {
+ "name": " G\u00e9rard Deprez",
+ "ref": 1473
+ }
+ ],
+ "report": "A8-0298/2015",
+ "title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 30",
+ "ts": "2015-10-28T13:01:09",
+ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
+ "voteid": "59151"
+ }
+]
diff --git a/representatives_votes/contrib/parltrack/tests/votes_initial.json b/representatives_votes/contrib/parltrack/tests/votes_initial.json
new file mode 100644
index 0000000..090fc8e
--- /dev/null
+++ b/representatives_votes/contrib/parltrack/tests/votes_initial.json
@@ -0,0 +1,68 @@
+[
+{
+ "fields": {
+ "updated": "2015-12-13T01:10:09.683Z",
+ "reference": "2012/2002(INI)",
+ "title": "Agenda for change: the future of EU development policy",
+ "text": "",
+ "created": "2015-12-13T01:10:09.683Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+ "fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 64
+},
+{
+ "fields": {
+ "updated": "2015-12-13T01:10:09.698Z",
+ "reference": "2015/2132(BUD)",
+ "title": "2016 general budget: all sections",
+ "text": "",
+ "created": "2015-12-13T01:10:09.698Z",
+ "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
+ "fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
+ },
+ "model": "representatives_votes.dossier",
+ "pk": 65
+},
+{
+ "fields": {
+ "updated": "2015-12-12T23:06:32.305Z",
+ "last_name": "PIRKER",
+ "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
+ "created": "2015-12-12T22:56:14.401Z",
+ "gender": 2,
+ "remote_id": "2307",
+ "first_name": "Hubert",
+ "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
+ "active": false,
+ "birth_place": "Gries",
+ "full_name": "Hubert PIRKER",
+ "fingerprint": "2a3c90346d40e9c540050534d832ceb3e0d25a49",
+ "birth_date": "1948-10-03",
+ "slug": "hubert-pirker"
+ },
+ "model": "representatives.representative",
+ "pk": 1
+},
+{
+ "fields": {
+ "updated": "2015-12-12T23:06:32.656Z",
+ "last_name": "LUDVIGSSON",
+ "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
+ "created": "2015-12-12T22:56:14.757Z",
+ "gender": 2,
+ "remote_id": "96673",
+ "first_name": "Olle",
+ "cv": "",
+ "active": true,
+ "birth_place": "H\u00e4ls\u00f6",
+ "full_name": "Olle LUDVIGSSON",
+ "fingerprint": "314d0f4c25af31bfa2a6b286838367994b902615",
+ "birth_date": "1948-10-28",
+ "slug": "olle-ludvigsson"
+ },
+ "model": "representatives.representative",
+ "pk": 2
+}
+]
diff --git a/representatives_votes/management/commands/import_dossier_from_toutatis.py b/representatives_votes/management/commands/import_dossier_from_toutatis.py
deleted file mode 100644
index 14eb2cd..0000000
--- a/representatives_votes/management/commands/import_dossier_from_toutatis.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-
-# This file is part of toutatis.
-#
-# toutatis is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of
-# the License, or any later version.
-#
-# toutatis is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU General Affero Public
-# License along with Foobar.
-# If not, see <http://www.gnu.org/licenses/>.
-#
-# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
-# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-
-from django.core.management.base import BaseCommand
-from representatives_votes.tasks import import_a_dossier_from_toutatis
-
-class Command(BaseCommand):
- """
- Command to import a dossier from a toutatis server
- """
-
- def add_arguments(self, parser):
- parser.add_argument('--celery', action='store_true', default=False)
- parser.add_argument('fingerprint')
-
- def handle(self, *args, **options):
- fingerprint = options['fingerprint']
- if options['celery']:
- import_a_dossier_from_toutatis.delay(fingerprint)
- else:
- import_a_dossier_from_toutatis(fingerprint)
diff --git a/representatives_votes/management/commands/import_proposal_from_toutatis.py b/representatives_votes/management/commands/import_proposal_from_toutatis.py
deleted file mode 100644
index 7e2966b..0000000
--- a/representatives_votes/management/commands/import_proposal_from_toutatis.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-
-# This file is part of toutatis.
-#
-# toutatis is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of
-# the License, or any later version.
-#
-# toutatis is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU General Affero Public
-# License along with Foobar.
-# If not, see <http://www.gnu.org/licenses/>.
-#
-# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
-# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-
-from django.core.management.base import BaseCommand
-from representatives_votes.tasks import import_a_proposal_from_toutatis
-
-class Command(BaseCommand):
- """
- Command to import a dossier from a toutatis server
- """
-
- def add_arguments(self, parser):
- parser.add_argument('--celery', action='store_true', default=False)
- parser.add_argument('fingerprint')
-
- def handle(self, *args, **options):
- fingerprint = options['fingerprint']
- if options['celery']:
- import_a_proposal_from_toutatis.delay(fingerprint, delay=True)
- else:
- import_a_proposal_from_toutatis(fingerprint, delay=False)
diff --git a/representatives_votes/migrations/0001_initial.py b/representatives_votes/migrations/0001_initial.py
index 22a3618..ebf94c0 100644
--- a/representatives_votes/migrations/0001_initial.py
+++ b/representatives_votes/migrations/0001_initial.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-from django.db import models, migrations
+from django.db import migrations, models
class Migration(migrations.Migration):
diff --git a/representatives_votes/migrations/0002_auto_20150707_1611.py b/representatives_votes/migrations/0002_auto_20150707_1611.py
index 5b553b5..5565949 100644
--- a/representatives_votes/migrations/0002_auto_20150707_1611.py
+++ b/representatives_votes/migrations/0002_auto_20150707_1611.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-from django.db import models, migrations
+from django.db import migrations, models
class Migration(migrations.Migration):
@@ -19,17 +19,25 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='vote',
name='representative',
- field=models.ForeignKey(related_name='votes', to='representatives.Representative', null=True),
+ field=models.ForeignKey(
+ related_name='votes',
+ to='representatives.Representative',
+ null=True),
),
migrations.AddField(
model_name='vote',
name='representative_fingerprint',
- field=models.CharField(max_length=200, blank=True),
+ field=models.CharField(
+ max_length=200,
+ blank=True),
),
migrations.AlterField(
model_name='vote',
name='representative_name',
- field=models.CharField(default='', max_length=200, blank=True),
+ field=models.CharField(
+ default='',
+ max_length=200,
+ blank=True),
preserve_default=False,
),
]
diff --git a/representatives_votes/migrations/0003_auto_20150708_1358.py b/representatives_votes/migrations/0003_auto_20150708_1358.py
index 2939234..390f1ec 100644
--- a/representatives_votes/migrations/0003_auto_20150708_1358.py
+++ b/representatives_votes/migrations/0003_auto_20150708_1358.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-from django.db import models, migrations
+from django.db import migrations, models
class Migration(migrations.Migration):
@@ -19,6 +19,8 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='proposal',
name='representatives',
- field=models.ManyToManyField(to='representatives.Representative', through='representatives_votes.Vote'),
+ field=models.ManyToManyField(
+ to='representatives.Representative',
+ through='representatives_votes.Vote'),
),
]
diff --git a/representatives_votes/migrations/0004_auto_20150709_0819.py b/representatives_votes/migrations/0004_auto_20150709_0819.py
index 1f897df..9437c23 100644
--- a/representatives_votes/migrations/0004_auto_20150709_0819.py
+++ b/representatives_votes/migrations/0004_auto_20150709_0819.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
-from django.db import models, migrations
+from django.db import migrations, models
class Migration(migrations.Migration):
@@ -13,15 +13,20 @@ class Migration(migrations.Migration):
operations = [
migrations.AlterModelOptions(
name='proposal',
- options={'ordering': ['datetime']},
+ options={
+ 'ordering': ['datetime']},
),
migrations.AlterModelOptions(
name='vote',
- options={'ordering': ['proposal__datetime']},
+ options={
+ 'ordering': ['proposal__datetime']},
),
migrations.AlterField(
model_name='proposal',
name='representatives',
- field=models.ManyToManyField(related_name='proposals', through='representatives_votes.Vote', to='representatives.Representative'),
+ field=models.ManyToManyField(
+ related_name='proposals',
+ through='representatives_votes.Vote',
+ to='representatives.Representative'),
),
]
diff --git a/representatives_votes/migrations/0005_make_dossier_reference_unique.py b/representatives_votes/migrations/0005_make_dossier_reference_unique.py
new file mode 100644
index 0000000..57f4f83
--- /dev/null
+++ b/representatives_votes/migrations/0005_make_dossier_reference_unique.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('representatives_votes', '0004_auto_20150709_0819'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='dossier',
+ name='reference',
+ field=models.CharField(unique=True, max_length=200),
+ ),
+ ]
diff --git a/representatives_votes/migrations/0006_duplicates.py b/representatives_votes/migrations/0006_duplicates.py
new file mode 100644
index 0000000..9a13adf
--- /dev/null
+++ b/representatives_votes/migrations/0006_duplicates.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.db.models import Count
+
+
+def remove_duplicate(apps, schema_editor):
+ Vote = apps.get_model('representatives_votes', 'Vote')
+ duplicates = Vote.objects.values('proposal_id',
+ 'representative_id').annotate(Count('id')).filter(id__count__gt=1)
+
+ for duplicate in duplicates:
+ remove = Vote.objects.filter(
+ proposal_id=duplicate['proposal_id'],
+ representative_id=duplicate['representative_id'])
+
+ for i in remove.values_list('pk')[1:]:
+ Vote.objects.get(pk=i[0]).delete()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('representatives_votes', '0005_make_dossier_reference_unique'),
+ ]
+
+ operations = [
+ migrations.RunPython(remove_duplicate)
+ ]
diff --git a/representatives_votes/migrations/0007_vote_unique_together_proposal_representative.py b/representatives_votes/migrations/0007_vote_unique_together_proposal_representative.py
new file mode 100644
index 0000000..1ffb38d
--- /dev/null
+++ b/representatives_votes/migrations/0007_vote_unique_together_proposal_representative.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('representatives_votes', '0006_duplicates'),
+ ]
+
+ operations = [
+ migrations.AlterUniqueTogether(
+ name='vote',
+ unique_together=set([('proposal', 'representative')]),
+ ),
+ ]
diff --git a/representatives_votes/migrations/0008_unique_proposal_title.py b/representatives_votes/migrations/0008_unique_proposal_title.py
new file mode 100644
index 0000000..faae473
--- /dev/null
+++ b/representatives_votes/migrations/0008_unique_proposal_title.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('representatives_votes', '0007_vote_unique_together_proposal_representative'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='proposal',
+ name='title',
+ field=models.CharField(unique=True, max_length=1000),
+ ),
+ ]
diff --git a/representatives_votes/models.py b/representatives_votes/models.py
index e2292fd..41514fd 100644
--- a/representatives_votes/models.py
+++ b/representatives_votes/models.py
@@ -18,24 +18,25 @@
from django.db import models
-from representatives.models import TimeStampedModel, HashableModel, Representative
+from representatives.models import (HashableModel, Representative,
+ TimeStampedModel)
class Dossier(HashableModel, TimeStampedModel):
title = models.CharField(max_length=1000)
- reference = models.CharField(max_length=200)
+ reference = models.CharField(max_length=200, unique=True)
text = models.TextField(blank=True, default='')
link = models.URLField()
hashable_fields = ['title', 'reference']
-
+
def __unicode__(self):
return unicode(self.title)
class Proposal(HashableModel, TimeStampedModel):
dossier = models.ForeignKey(Dossier, related_name='proposals')
- title = models.CharField(max_length=1000)
+ title = models.CharField(max_length=1000, unique=True)
description = models.TextField(blank=True, default='')
reference = models.CharField(max_length=200, blank=True, null=True)
datetime = models.DateTimeField()
@@ -47,22 +48,21 @@ class Proposal(HashableModel, TimeStampedModel):
representatives = models.ManyToManyField(
Representative, through='Vote', related_name='proposals'
)
-
+
hashable_fields = ['dossier', 'title', 'reference',
'kind', 'total_abstain', 'total_against',
'total_for']
class Meta:
ordering = ['datetime']
-
-
+
@property
def status(self):
if self.total_for > self.total_against:
return 'adopted'
else:
return 'rejected'
-
+
def __unicode__(self):
return unicode(self.title)
@@ -75,8 +75,9 @@ class Vote(models.Model):
)
proposal = models.ForeignKey(Proposal, related_name='votes')
-
- representative = models.ForeignKey(Representative, related_name='votes', null=True)
+
+ representative = models.ForeignKey(
+ Representative, related_name='votes', null=True)
# Save representative name in case of we don't find the representative
representative_name = models.CharField(max_length=200, blank=True)
@@ -84,3 +85,4 @@ class Vote(models.Model):
class Meta:
ordering = ['proposal__datetime']
+ unique_together = (('proposal', 'representative'))
diff --git a/representatives_votes/serializers.py b/representatives_votes/serializers.py
deleted file mode 100644
index d2b1a3c..0000000
--- a/representatives_votes/serializers.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# coding: utf-8
-
-# This file is part of toutatis.
-#
-# toutatis is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of
-# the License, or any later version.
-#
-# toutatis is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU General Affero Public
-# License along with django-representatives.
-# If not, see <http://www.gnu.org/licenses/>.
-#
-# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-
-import representatives_votes.models as models
-from representatives.models import Representative
-from rest_framework import serializers
-
-
-class VoteSerializer(serializers.ModelSerializer):
- """
- Vote serializer
- """
- proposal = serializers.CharField(
- source='proposal.fingerprint'
- )
- representative = serializers.CharField(
- source='representative.fingerprint',
- allow_null=True
- )
-
- class Meta:
- model = models.Vote
- fields = (
- 'id',
- 'proposal',
- 'representative',
- 'representative_name',
- 'position'
- )
-
- def to_internal_value(self, data):
- data = super(VoteSerializer, self).to_internal_value(data)
- data['proposal'] = models.Proposal.objects.get(
- fingerprint=data['proposal']['fingerprint']
- )
- if data['representative']['fingerprint']:
- data['representative'] = Representative.objects.get(
- fingerprint=data['representative']['fingerprint']
- )
- else:
- data['representative'] = None
-
- return data
-
-
-class ProposalSerializer(serializers.ModelSerializer):
- dossier = serializers.CharField(
- source='dossier.fingerprint'
- )
-
- dossier_title = serializers.CharField(
- source='dossier.title',
- read_only=True
- )
-
- dossier_reference = serializers.CharField(
- source='dossier.reference',
- read_only=True
- )
-
- class Meta:
- model = models.Proposal
- fields = (
- 'id',
- 'fingerprint',
- 'dossier',
- 'dossier_title',
- 'dossier_reference',
- 'title',
- 'description',
- 'reference',
- 'datetime',
- 'kind',
- 'total_abstain',
- 'total_against',
- 'total_for',
- 'url',
- )
-
- def to_internal_value(self, data):
- validated_data = super(ProposalSerializer, self).to_internal_value(data)
- validated_data['dossier'] = models.Dossier.objects.get(
- fingerprint=validated_data['dossier']['fingerprint']
- )
- validated_data['votes'] = data['votes']
- return validated_data
-
- def _create_votes(self, votes_data, proposal):
- for vote in votes_data:
- serializer = VoteSerializer(data=vote)
- if serializer.is_valid():
- serializer.save()
- else:
- raise Exception(serializer.errors)
-
- def create(self, validated_data):
- votes_data = validated_data.pop('votes')
- proposal = models.Proposal.objects.create(
- **validated_data
- )
- self._create_votes(votes_data, proposal)
- return proposal
-
- def update(self, instance, validated_data):
- validated_data.pop('votes')
- for attr, value in validated_data.iteritems():
- setattr(instance, attr, value)
- instance.save()
- return instance
-
-
-class ProposalDetailSerializer(ProposalSerializer):
- """ Proposal serializer that includes votes """
- votes = VoteSerializer(many=True)
-
- class Meta(ProposalSerializer.Meta):
- fields = ProposalSerializer.Meta.fields + (
- 'votes',
- )
-
-
-class DossierSerializer(serializers.ModelSerializer):
- """ Base dossier serializer """
- class Meta:
- model = models.Dossier
- fields = (
- 'id',
- 'fingerprint',
- 'title',
- 'reference',
- 'text',
- 'link',
- 'url',
- )
-
-
-class DossierDetailSerializer(DossierSerializer):
- """
- Dossier serializer that includes proposals
- and votes
- """
- proposals = ProposalDetailSerializer(
- many = True,
- )
-
- class Meta(DossierSerializer.Meta):
- fields = DossierSerializer.Meta.fields + (
- 'proposals',
- )
diff --git a/representatives_votes/tasks.py b/representatives_votes/tasks.py
deleted file mode 100644
index b9004e6..0000000
--- a/representatives_votes/tasks.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# coding: utf-8
-
-# This file is part of toutatis.
-#
-# toutatis is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of
-# the License, or any later version.
-#
-# toutatis is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU General Affero Public
-# License along with django-representatives.
-# If not, see <http://www.gnu.org/licenses/>.
-#
-# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-
-from __future__ import absolute_import
-
-import logging
-import json
-
-from django.conf import settings
-
-import redis
-from celery import shared_task
-from urllib2 import urlopen
-
-from representatives_votes.models import Dossier, Proposal, Vote
-from representatives.tasks import import_a_model
-from representatives_votes.serializers import DossierSerializer, ProposalSerializer, VoteSerializer
-
-logger = logging.getLogger(__name__)
-
-
-def import_a_dossier_from_toutatis(fingerprint):
- '''
- Import a complete dossier from a toutatis server
- '''
- toutatis_server = settings.TOUTATIS_SERVER
- search_url = '{}/api/dossiers/?fingerprint={}'.format(
- toutatis_server,
- fingerprint
- )
- logger.info('Import dossier with fingerprint {} from {}'.format(
- fingerprint,
- search_url
- ))
- data = json.load(urlopen(search_url))
- if data['count'] != 1:
- raise Exception('Search should return one and only one result')
- detail_url = data['results'][0]['url']
- data = json.load(urlopen(detail_url))
- dossier = import_a_model(data, Dossier, DossierSerializer)
- for proposal in data['proposals']:
- logger.info('Import proposal {}'.format(proposal['title']))
- import_a_model(proposal, Proposal, ProposalSerializer)
- return dossier
-
-def import_a_proposal_from_toutatis(fingerprint):
- '''
- Import a partial dossier from a toutatis server
- '''
- toutatis_server = settings.TOUTATIS_SERVER
- search_url = '{}/api/proposals/?fingerprint={}'.format(
- toutatis_server,
- fingerprint
- )
- logger.info('Import proposal with fingerprint {} from {}'.format(
- fingerprint,
- search_url
- ))
- proposal_data = json.load(urlopen(search_url))
- if proposal_data['count'] != 1:
- raise Exception('Search should return one and only one result')
- detail_url = proposal_data['results'][0]['url']
- proposal_data = json.load(urlopen(detail_url))
- search_url = '{}/api/dossiers/?fingerprint={}'.format(
- toutatis_server,
- proposal_data['dossier']
- )
- dossier_data = json.load(urlopen(search_url))
- if dossier_data['count'] != 1:
- raise Exception('Search should return one and only one result')
- import_a_model(dossier_data['results'][0], Dossier, DossierSerializer)
- return import_a_model(proposal_data, Proposal, ProposalSerializer)
-
-
-def export_a_dossier(dossier):
- serialized = DossierDetailSerializer(dossier)
- return serialized.data
-
-def export_dossiers(filters={}):
- return [export_a_dossier(dossier) for dossier in Dossier.objects.filter(**filters)]
-
-def export_all_dossier():
- return export_dossiers()
diff --git a/test_project/test_project/__init__.py b/representatives_votes/tests/__init__.py
similarity index 100%
rename from test_project/test_project/__init__.py
rename to representatives_votes/tests/__init__.py
diff --git a/representatives_votes/tests/settings.py b/representatives_votes/tests/settings.py
new file mode 100644
index 0000000..8623f2e
--- /dev/null
+++ b/representatives_votes/tests/settings.py
@@ -0,0 +1,57 @@
+import os
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+SECRET_KEY = 'notsecret'
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': os.path.join(BASE_DIR, 'test.db'),
+ }
+}
+
+INSTALLED_APPS = (
+ 'representatives',
+ 'representatives_votes',
+)
+
+DEBUG = True
+LANGUAGE_CODE = 'en-us'
+TIME_ZONE = 'UTC'
+ROOT_URLCONF = 'representatives_votes.tests.urls'
+
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+
+LOGGING = {
+ 'version': 1,
+ 'disable_existing_loggers': False,
+ 'formatters': {
+ 'simple': {
+ 'format': '%(levelname)s[%(module)s]: %(message)s'
+ },
+ },
+ 'handlers': {
+ 'console': {
+ 'level': 'DEBUG',
+ 'class': 'logging.StreamHandler',
+ 'formatter': 'simple'
+ },
+ },
+ 'loggers': {
+ 'django': {
+ 'handlers': ['console'],
+ 'level': 'INFO'
+ },
+ 'representatives': {
+ 'handlers': ['console'],
+ 'level': 'DEBUG'
+ },
+ 'representatives_votes': {
+ 'handlers': ['console'],
+ 'level': 'DEBUG'
+ },
+ },
+}
diff --git a/representatives_votes/tests/urls.py b/representatives_votes/tests/urls.py
new file mode 100644
index 0000000..637600f
--- /dev/null
+++ b/representatives_votes/tests/urls.py
@@ -0,0 +1 @@
+urlpatterns = []
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index ca5b397..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-django>=1.8,<1.9
-djangorestframework
diff --git a/setup.py b/setup.py
index 473a810..1d839e0 100644
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,10 @@
-from setuptools import setup, find_packages
-
+from setuptools import find_packages, setup
setup(
name='django-representatives-votes',
- version='0.0.1',
+ version='0.0.7',
description='Base app for government representative votes',
- author='Olivier Le Thanh Duong',
+ author='Olivier Le Thanh Duong, Laurent Peuch, Arnaud Fabre, James Pic',
author_email='olivier@lethanh.be',
url='http://github.com/political-memory/django-representatives-votes',
packages=find_packages(),
@@ -14,9 +13,18 @@ setup(
keywords='django government parliament votes',
install_requires=[
'django-representatives',
+ 'py-dateutil',
+ 'pytz',
+ 'ijson',
],
+ entry_points={
+ 'console_scripts': [
+ 'parltrack_import_dossiers = representatives_votes.contrib.parltrack.import_dossiers:main',
+ 'parltrack_import_votes = representatives_votes.contrib.parltrack.import_votes:main',
+ ]
+ },
classifiers=[
- 'Development Status :: 1 - Alpha/Planning',
+ 'Development Status :: 4 - Beta',
'Environment :: Web Environment',
'Framework :: Django',
'Intended Audience :: Developers',
diff --git a/test_project/manage.py b/test_project/manage.py
deleted file mode 100755
index 0fc36a3..0000000
--- a/test_project/manage.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if __name__ == "__main__":
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project.settings")
-
- from django.core.management import execute_from_command_line
-
- execute_from_command_line(sys.argv)
diff --git a/test_project/test_project/settings.py b/test_project/test_project/settings.py
deleted file mode 100644
index d5f1852..0000000
--- a/test_project/test_project/settings.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""
-Django settings for test_project project.
-
-Generated by 'django-admin startproject' using Django 1.8.5.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.8/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/1.8/ref/settings/
-"""
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-import os
-
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'hul=ds-wua5*3r+o2k-%&jlleub99q7dot+lb(((v*^-@yk%g7'
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = []
-
-
-# Application definition
-
-INSTALLED_APPS = (
- 'django.contrib.admin',
- 'django.contrib.auth',
- 'django.contrib.contenttypes',
- 'django.contrib.sessions',
- 'django.contrib.messages',
- 'django.contrib.staticfiles',
- 'representatives',
- 'representatives_votes',
-)
-
-MIDDLEWARE_CLASSES = (
- 'django.contrib.sessions.middleware.SessionMiddleware',
- 'django.middleware.common.CommonMiddleware',
- 'django.middleware.csrf.CsrfViewMiddleware',
- 'django.contrib.auth.middleware.AuthenticationMiddleware',
- 'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
- 'django.contrib.messages.middleware.MessageMiddleware',
- 'django.middleware.clickjacking.XFrameOptionsMiddleware',
- 'django.middleware.security.SecurityMiddleware',
-)
-
-ROOT_URLCONF = 'test_project.urls'
-
-TEMPLATES = [
- {
- 'BACKEND': 'django.template.backends.django.DjangoTemplates',
- 'DIRS': [],
- 'APP_DIRS': True,
- 'OPTIONS': {
- 'context_processors': [
- 'django.template.context_processors.debug',
- 'django.template.context_processors.request',
- 'django.contrib.auth.context_processors.auth',
- 'django.contrib.messages.context_processors.messages',
- ],
- },
- },
-]
-
-WSGI_APPLICATION = 'test_project.wsgi.application'
-
-
-# Database
-# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
-
-DATABASES = {
- 'default': {
- 'ENGINE': 'django.db.backends.sqlite3',
- 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
- }
-}
-
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.8/topics/i18n/
-
-LANGUAGE_CODE = 'en-us'
-
-TIME_ZONE = 'UTC'
-
-USE_I18N = True
-
-USE_L10N = True
-
-USE_TZ = True
-
-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.8/howto/static-files/
-
-STATIC_URL = '/static/'
diff --git a/test_project/test_project/urls.py b/test_project/test_project/urls.py
deleted file mode 100644
index 7dc47ea..0000000
--- a/test_project/test_project/urls.py
+++ /dev/null
@@ -1,21 +0,0 @@
-"""test_project URL Configuration
-
-The `urlpatterns` list routes URLs to views. For more information please see:
- https://docs.djangoproject.com/en/1.8/topics/http/urls/
-Examples:
-Function views
- 1. Add an import: from my_app import views
- 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home')
-Class-based views
- 1. Add an import: from other_app.views import Home
- 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home')
-Including another URLconf
- 1. Add an import: from blog import urls as blog_urls
- 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls))
-"""
-from django.conf.urls import include, url
-from django.contrib import admin
-
-urlpatterns = [
- url(r'^admin/', include(admin.site.urls)),
-]
diff --git a/test_project/test_project/wsgi.py b/test_project/test_project/wsgi.py
deleted file mode 100644
index cb26c81..0000000
--- a/test_project/test_project/wsgi.py
+++ /dev/null
@@ -1,16 +0,0 @@
-"""
-WSGI config for test_project project.
-
-It exposes the WSGI callable as a module-level variable named ``application``.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/
-"""
-
-import os
-
-from django.core.wsgi import get_wsgi_application
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project.settings")
-
-application = get_wsgi_application()
--
GitLab