Added parltrack import commands in contrib

Also:

- code was PEP8'ed
- tests were added,
- untested code was removed,
- setup.py was fixed and requirements.txt is gone,

.coveragerc

+[run]
+omit = representatives_votes/tests/*
+omit = representatives_votes/migrations/*

.travis.yml

 sudo: false
 language: python
+env:
+- DJANGO="django>1.8,<1.9" DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
 python:
-  - "2.7"
+- "2.7"
+before_install:
+- pip install codecov
 install:
-  - pip install django
-  - pip install -e git+https://github.com/political-memory/django-representatives.git#egg=representatives
-  - pip install -e .
+- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov
+- pip install https://github.com/political-memory/django-representatives/archive/parltrack.tar.gz#egg=django-representatives
+- pip install -e .
 script:
-  - test_project/manage.py migrate
+- django-admin migrate
+- flake8 representatives_votes/ --exclude migrations --ignore E128
+- py.test
+- cat representatives_votes/contrib/parltrack/tests/dossiers_fixture.json | parltrack_import_dossiers
+- cat representatives_votes/contrib/parltrack/tests/votes_fixture.json | parltrack_import_votes
+after_success:
+- codecov

MANIFEST.in

+include *.rst *.txt README LICENSE AUTHORS CHANGELOG
+recursive-include representatives_votes *.html *.css *.js *.py *.po *.mo *.json *.png *.gif

README.md

 [![Build Status](https://travis-ci.org/political-memory/django-representatives-votes.svg?branch=travis)](https://travis-ci.org/political-memory/django-representatives-votes)
+[![codecov.io](https://codecov.io/github/political-memory/django-representatives-votes/coverage.svg?branch=master)](https://codecov.io/github/political-memory/django-representatives-votes?branch=master)

pytest.ini

+[pytest]
+DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
+addopts = --cov=representatives_votes --create-db

representatives_votes/admin.py

 # coding: utf-8
 
 from django.contrib import admin
+
 from .models import Dossier, Proposal, Vote
 
 
@@ -10,8 +11,19 @@ class DossierAdmin(admin.ModelAdmin):
 
 
 class ProposalAdmin(admin.ModelAdmin):
-    list_display = ('id', 'fingerprint', 'reference', 'dossier_reference', 'title', 'datetime', 'kind', 'total_abstain', 'total_against', 'total_for')
+    list_display = (
+        'id',
+        'fingerprint',
+        'reference',
+        'dossier_reference',
+        'title',
+        'datetime',
+        'kind',
+        'total_abstain',
+        'total_against',
+        'total_for')
     search_fields = ('reference', 'dossier__reference', 'title', 'fingerprint')
+
     def dossier_reference(self, obj):
         return obj.dossier.reference
 
@@ -22,18 +34,23 @@ class NoneMatchingFilter(admin.SimpleListFilter):
 
     def lookups(self, request, model_admin):
         return [('None', 'Unknown')]
-    
+
     def queryset(self, request, queryset):
         if self.value() == 'None':
             return queryset.filter(representative=None)
         else:
-            return queryset            
+            return queryset
 
 
 class VoteAdmin(admin.ModelAdmin):
-    list_display = ('id', 'proposal_reference', 'position', 'representative', 'representative_name')
+    list_display = (
+        'id',
+        'proposal_reference',
+        'position',
+        'representative',
+        'representative_name')
     list_filter = (NoneMatchingFilter,)
-    
+
     def proposal_reference(self, obj):
         return obj.proposal.reference
 

representatives_votes/contrib/parltrack/import_dossiers.py

+# coding: utf-8
+import logging
+import sys
+
+import ijson
+import django
+from django.apps import apps
+
+from representatives_votes.models import Dossier
+
+logger = logging.getLogger(__name__)
+
+URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz'
+LOCAL_PATH = 'ep_dossiers.json.xz'
+
+
+def parse_dossier_data(data):
+    """Parse data from parltarck dossier export (1 dossier) Update dossier
+    if it existed before, this function goal is to import and update a
+    dossier, not to import all parltrack data
+    """
+    changed = False
+    ref = data['procedure']['reference']
+
+    logger.debug('Processing dossier %s', ref)
+
+    try:
+        dossier = Dossier.objects.get(reference=ref)
+    except Dossier.DoesNotExist:
+        dossier = Dossier(reference=ref)
+        logger.debug('Dossier did not exist')
+        changed = True
+
+    if dossier.title != data['procedure']['title']:
+        logger.debug('Title changed from "%s" to "%s"', dossier.title,
+                     data['procedure']['title'])
+        dossier.title = data['procedure']['title']
+        changed = True
+
+    source = data['meta']['source'].replace('&l=en', '')
+    if dossier.link != source:
+        logger.debug('Source changed from "%s" to "%s"', dossier.link, source)
+        dossier.link = source
+        changed = True
+
+    if changed:
+        logger.info('Updated dossier %s', ref)
+        dossier.save()
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        parse_dossier_data(data)

representatives_votes/contrib/parltrack/import_votes.py

+# coding: utf-8
+import logging
+import sys
+from os.path import join
+
+import django.dispatch
+import ijson
+import django
+from django.apps import apps
+from dateutil.parser import parse as date_parse
+from django.db import transaction
+from django.utils.timezone import make_aware as date_make_aware
+from pytz import timezone as date_timezone
+
+from representatives.models import Representative
+from representatives_votes.models import Dossier, Proposal, Vote
+
+
+logger = logging.getLogger(__name__)
+vote_pre_import = django.dispatch.Signal(providing_args=['vote_data'])
+
+
+def _parse_date(date_str):
+    return date_make_aware(
+        date_parse(date_str),
+        date_timezone('Europe/Brussels'))
+
+JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
+DESTINATION = join('/tmp', 'ep_votes.json')
+
+
+class Command(object):
+    def init_cache(self):
+        self.cache = dict()
+        self.index_representatives()
+        self.index_dossiers()
+
+    def parse_vote_data(self, vote_data):
+        """
+        Parse data from parltrack votes db dumps (1 proposal)
+        """
+        if 'epref' not in vote_data.keys():
+            logger.debug('Could not import data without epref %s',
+                vote_data['title'])
+            return
+
+        dossier_pk = self.get_dossier(vote_data['epref'])
+
+        if not dossier_pk:
+            logger.debug('Cannot find dossier with remote id %s',
+                         vote_data['epref'])
+            return
+
+        return self.parse_proposal_data(
+            proposal_data=vote_data,
+            dossier_pk=dossier_pk
+        )
+
+    @transaction.atomic
+    def parse_proposal_data(self, proposal_data, dossier_pk):
+        """Get or Create a proposal model from raw data"""
+        proposal_display = '{} ({})'.format(proposal_data['title'].encode(
+            'utf-8'), proposal_data.get('report', '').encode('utf-8'))
+
+        if 'issue_type' not in proposal_data.keys():
+            logger.debug('This proposal data without issue_type: %s',
+                         proposal_data['epref'])
+            return
+
+        changed = False
+        try:
+            proposal = Proposal.objects.get(title=proposal_data['title'])
+        except Proposal.DoesNotExist:
+            proposal = Proposal(title=proposal_data['title'])
+            changed = True
+
+        data_map = dict(
+            title=proposal_data['title'],
+            datetime=_parse_date(proposal_data['ts']),
+            dossier_id=dossier_pk,
+            reference=proposal_data.get('report'),
+            kind=proposal_data.get('issue_type')
+        )
+
+        for position in ('For', 'Abstain', 'Against'):
+            position_data = proposal_data.get(position, {})
+            position_total = position_data.get('total', 0)
+
+            if isinstance(position_total, str) and position_total.isdigit():
+                position_total = int(position_total)
+
+            data_map['total_%s' % position.lower()] = position_total
+
+        for key, value in data_map.items():
+            if value != getattr(proposal, key, None):
+                setattr(proposal, key, value)
+                changed = True
+
+        if changed:
+            proposal.save()
+
+        responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
+
+        for receiver, response in responses:
+            if response is False:
+                logger.debug(
+                    'Skipping dossier %s', proposal_data.get(
+                        'epref', proposal_data['title']))
+                return
+
+        positions = ['For', 'Abstain', 'Against']
+        logger.info(
+            'Looking for votes in proposal {}'.format(proposal_display))
+        for position in positions:
+            for group_vote_data in proposal_data.get(
+                    position,
+                    {}).get(
+                    'groups',
+                    {}):
+                for vote_data in group_vote_data['votes']:
+                    if not isinstance(vote_data, dict):
+                        logger.error('Skipping vote data %s for proposal %s',
+                                     vote_data, proposal_data['_id'])
+                        continue
+
+                    representative_pk = self.get_representative(vote_data)
+
+                    if representative_pk is None:
+                        logger.error('Could not find mep for %s', vote_data)
+                        continue
+
+                    representative_name = vote_data.get('orig', '')
+
+                    changed = False
+                    try:
+                        vote = Vote.objects.get(
+                            representative_id=representative_pk,
+                            proposal_id=proposal.pk)
+                    except Vote.DoesNotExist:
+                        vote = Vote(proposal_id=proposal.pk,
+                            representative_id=representative_pk)
+                        changed = True
+
+                    if vote.position != position.lower():
+                        changed = True
+                        vote.position = position.lower()
+
+                    if vote.representative_name != representative_name:
+                        changed = True
+                        vote.representative_name = representative_name
+
+                    if changed:
+                        vote.save()
+                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
+                            vote.pk, representative_pk, proposal_data['title'],
+                            proposal.pk, position)
+
+        return proposal
+
+    def index_dossiers(self):
+        self.cache['dossiers'] = {
+            d[0]: d[1] for d in Dossier.objects.values_list('reference', 'pk')
+        }
+
+    def get_dossier(self, reference):
+        return self.cache['dossiers'].get(reference, None)
+
+    def index_representatives(self):
+        self.cache['meps'] = {int(l[0]): l[1] for l in
+            Representative.objects.values_list('remote_id', 'pk')}
+
+    def get_representative(self, vote_data):
+        if vote_data.get('ep_id', None) is None:
+            return
+        return self.cache['meps'].get(int(vote_data['ep_id']), None)
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    command = Command()
+    command.init_cache()
+
+    for vote_data in ijson.items(stream or sys.stdin, 'item'):
+        command.parse_vote_data(vote_data)

representatives_votes/contrib/parltrack/tests/dossiers_expected.json

+[
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.369Z",
+        "reference": "2012/2002(INI)",
+        "title": "Agenda for change: the future of EU development policy",
+        "text": "",
+        "created": "2015-12-13T10:11:31.369Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+        "fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 1
+},
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.378Z",
+        "reference": "2015/2132(BUD)",
+        "title": "2016 general budget: all sections",
+        "text": "",
+        "created": "2015-12-13T10:11:31.378Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
+        "fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 2
+},
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.388Z",
+        "reference": "2013/2857(DEA)",
+        "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+        "text": "",
+        "created": "2015-12-13T10:11:31.388Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)",
+        "fingerprint": "50745a1a6e47b8db097c55ef21a4f11fc1ef0d97"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 3
+},
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.398Z",
+        "reference": "2015/2623(DEA)",
+        "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+        "text": "",
+        "created": "2015-12-13T10:11:31.398Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)",
+        "fingerprint": "65fce7af2e020ff58849d1663f2c30ab0b1a35db"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 4
+},
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.408Z",
+        "reference": "2009/0051(COD)",
+        "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+        "text": "",
+        "created": "2015-12-13T10:11:31.408Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)",
+        "fingerprint": "f2abba201c10df8a7cec1d734f91b7988eec8260"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 5
+}
+]

representatives_votes/contrib/parltrack/tests/dossiers_fixture.json

+[
+    {
+        "meta": {
+            "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+            "updated": "2012-10-25T02:02:36.882000"
+        },
+        "procedure": {
+            "dossier_of_the_committee": "DEVE/7/08564",
+            "legal_basis": [
+                "Rules of Procedure of the European Parliament EP 048"
+            ],
+            "reference": "2012/2002(INI)",
+            "stage_reached": "Procedure completed",
+            "subject": [
+                "6.30 Development cooperation"
+            ],
+            "subtype": "Initiative",
+            "title": "Agenda for change: the future of EU development policy",
+            "type": "INI - Own-initiative procedure"
+        }
+    },
+    {
+        "meta": {
+            "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)&l=en",
+            "updated": "2015-11-18T02:23:18.933000"
+        },
+        "procedure": {
+            "dossier_of_the_committee": "BUDE/8/04900;BUDG/8/03789",
+            "reference": "2015/2132(BUD)",
+            "stage_reached": "Budgetary conciliation committee convened",
+            "subject": [
+                "8.70.56 2016 budget"
+            ],
+            "subtype": "Budget",
+            "title": "2016 general budget: all sections",
+            "type": "BUD - Budgetary procedure"
+        }
+    },
+    {
+        "meta": {
+            "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)&l=en",
+            "updated": "2014-12-06T03:56:21.811000"
+        },
+        "procedure": {
+            "dossier_of_the_committee": "PECH/7/14126",
+            "geographical_area": [
+                "Atlantic Ocean area"
+            ],
+            "reference": "2013/2857(DEA)",
+            "stage_reached": "Awaiting Council decision on delegated act",
+            "subject": [
+                "3.15.01 Fish stocks, conservation of fishery resources",
+                "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+                "3.15.15 Fisheries agreements and cooperation"
+            ],
+            "subtype": "Examination of delegated act",
+            "summary": [
+                "Supplementing"
+            ],
+            "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+            "type": "DEA - Delegated acts procedure"
+        }
+    },
+    {
+        "meta": {
+            "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)&l=en",
+            "updated": "2015-07-22T00:46:11.603000"
+        },
+        "procedure": {
+            "dossier_of_the_committee": "PECH/8/03020",
+            "reference": "2015/2623(DEA)",
+            "stage_reached": "Awaiting Council decision on delegated act",
+            "subject": [
+                "3.15.01 Fish stocks, conservation of fishery resources",
+                "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+                "3.15.15 Fisheries agreements and cooperation"
+            ],
+            "subtype": "Examination of delegated act",
+            "summary": [
+                "Supplementing"
+            ],
+            "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+            "type": "DEA - Delegated acts procedure"
+        }
+    },
+    {
+        "meta": {
+            "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)&l=en",
+            "updated": "2015-11-04T17:01:33.812000"
+        },
+        "procedure": {
+            "Modified legal basis": "Rules of Procedure of the European Parliament EP 150",
+            "dossier_of_the_committee": "PECH/7/00288",
+            "final": {
+                "title": "Regulation 2010/1236",
+                "url": "http://eur-lex.europa.eu/smartapi/cgi/sga_doc?smartapi!celexplus!prod!CELEXnumdoc&lg=EN&numdoc=32010R1236"
+            },
+            "instrument": "Regulation",
+            "legal_basis": [
+                "Treaty on the Functioning of the EU TFEU 043-p2"
+            ],
+            "reference": "2009/0051(COD)",
+            "stage_reached": "Procedure completed",
+            "subject": [
+                "3.15.01 Fish stocks, conservation of fishery resources",
+                "3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
+                "3.15.15 Fisheries agreements and cooperation"
+            ],
+            "subtype": "Legislation",
+            "summary": [
+                "Amended by",
+                "Repealing Regulation (EC) No 2791/1999"
+            ],
+            "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
+            "type": "COD - Ordinary legislative procedure (ex-codecision procedure)"
+        }
+    }
+]

representatives_votes/contrib/parltrack/tests/test_import.py

+import pytest
+import os
+import copy
+
+from django.core.serializers.json import Deserializer
+from django.core.management import call_command
+from representatives_votes.contrib.parltrack import import_dossiers
+from representatives_votes.contrib.parltrack import import_votes
+from representatives_votes.models import Dossier, Proposal, Vote
+from representatives.models import Representative
+from representatives.contrib import parltrack
+
+
+def _test_import(scenario, callback):
+    fixture = os.path.join(os.path.dirname(__file__),
+            '%s_fixture.json' % scenario)
+    expected = os.path.join(os.path.dirname(__file__),
+            '%s_expected.json' % scenario)
+
+    # Disable django auto fields
+    exclude = ('id', '_state', 'created', 'updated')
+
+    with open(fixture, 'r') as f:
+        callback(f)
+
+    with open(expected, 'r') as f:
+        for obj in Deserializer(f.read()):
+            compare = copy.copy(obj.object.__dict__)
+
+            for f in exclude:
+                if f in compare:
+                    compare.pop(f)
+
+            type(obj.object).objects.get(**compare)
+
+
+@pytest.mark.django_db
+def test_parltrack_import_dossiers():
+    _test_import('dossiers', import_dossiers.main)
+
+
+@pytest.mark.django_db
+def test_parltrack_import_votes():
+    for model in (Representative, Dossier, Proposal, Vote):
+        model.objects.all().delete()
+
+    call_command('loaddata', os.path.join(os.path.abspath(
+        parltrack.__path__[0]), 'tests', 'representatives_expected.json'))
+    call_command('loaddata', os.path.join(os.path.dirname(__file__),
+        'dossiers_expected.json'))
+
+    _test_import('votes', import_votes.main)

representatives_votes/contrib/parltrack/tests/votes_expected.json

+[
+{
+    "fields": {
+        "representative_name": "",
+        "position": "abstain",
+        "proposal": 1,
+        "representative": 2
+    },
+    "model": "representatives_votes.vote",
+    "pk": 1
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "abstain",
+        "proposal": 1,
+        "representative": 1
+    },
+    "model": "representatives_votes.vote",
+    "pk": 2
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "for",
+        "proposal": 2,
+        "representative": 2
+    },
+    "model": "representatives_votes.vote",
+    "pk": 3
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "for",
+        "proposal": 2,
+        "representative": 1
+    },
+    "model": "representatives_votes.vote",
+    "pk": 4
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "against",
+        "proposal": 3,
+        "representative": 2
+    },
+    "model": "representatives_votes.vote",
+    "pk": 5
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "against",
+