Merge pull request #14 from political-memory/njoyard-add-single-dossier-simpler

Add single dossier sync from parltrack

.travis.yml

@@ -7,7 +7,7 @@ python:
 before_install:
 - pip install codecov
 install:
-- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov django-responsediff
+- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov django-responsediff mock
 - pip install https://github.com/political-memory/django-representatives/archive/parltrack.tar.gz#egg=django-representatives
 - pip install -e .[api]
 script:

representatives_votes/contrib/parltrack/import_dossiers.py

 # coding: utf-8
 import logging
 import sys
+import urllib2
 
 import ijson
 import django
 from django.apps import apps
 
 from representatives_votes.models import Dossier
+from .import_votes import Command
 
 logger = logging.getLogger(__name__)
 
+BASEURL = 'http://parltrack.euwiki.org/'
 URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz'
 LOCAL_PATH = 'ep_dossiers.json.xz'
 
@@ -47,6 +50,27 @@ def parse_dossier_data(data):
         logger.info('Updated dossier %s', ref)
         dossier.save()
 
+    if 'votes' in data.keys() and 'epref' in data['votes']:
+        command = Command()
+        command.init_cache()
+        command.parse_vote_data(data['votes'])
+
+
+def sync_dossier(reference):
+    url = '%s/dossier/%s?format=json' % (BASEURL, reference)
+
+    logger.debug('Syncing dossier from %s' % url)
+    with urllib2.urlopen(url) as stream:
+        import_single(stream)
+
+
+def import_single(stream):
+    if not apps.ready:
+        django.setup()
+
+    for data in ijson.items(stream, ''):
+        parse_dossier_data(data)
+
 
 def main(stream=None):
     if not apps.ready:

representatives_votes/contrib/parltrack/tests/single_expected.json

+[
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.378Z",
+        "reference": "2015/2132(BUD)",
+        "title": "2016 general budget: all sections",
+        "text": "",
+        "created": "2015-12-13T10:11:31.378Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
+        "fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 1
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "for",
+        "proposal": 1,
+        "representative": 1
+    },
+    "model": "representatives_votes.vote",
+    "pk": 2
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "against",
+        "proposal": 1,
+        "representative": 2
+    },
+    "model": "representatives_votes.vote",
+    "pk": 3
+}
+]

representatives_votes/contrib/parltrack/tests/single_fixture.json

+{
+    "meta": {
+        "source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)&l=en",
+        "updated": "2015-11-18T02:23:18.933000"
+    },
+    "procedure": {
+        "dossier_of_the_committee": "BUDE/8/04900;BUDG/8/03789",
+        "reference": "2015/2132(BUD)",
+        "stage_reached": "Budgetary conciliation committee convened",
+        "subject": [
+            "8.70.56 2016 budget"
+        ],
+        "subtype": "Budget",
+        "title": "2016 general budget: all sections",
+        "type": "BUD - Budgetary procedure"
+    },
+    "votes": {
+        "For": {
+            "groups": [
+                {
+                    "group": "PPE",
+                    "votes": [
+                        {
+                            "ep_id": 2307,
+                            "name": "Pirker",
+                            "userid": 4611
+                        }
+                    ]
+                }
+            ],
+            "total": "1"
+        },
+        "Against": {
+            "groups": [
+                {
+                    "group": "PPE",
+                    "votes": [
+                        {
+                            "ep_id": 96673,
+                            "name": "Ludvigsson",
+                            "userid": 5860
+                        }
+                    ]
+                }
+            ],
+            "total": "1"
+        },
+        "_id": "56617a11ecc52ed712fc6b15",
+        "dossierid": "55a4580ed1d1c57c1e000002",
+        "epref": "2015/2132(BUD)",
+        "eptitle": "2016 general budget: all sections",
+        "issue_type": "Am 29",
+        "rapporteur": [
+            {
+                "name": "Jos\u00e9 Manuel Fernandes",
+                "ref": 96899
+            },
+            {
+                "name": " G\u00e9rard Deprez",
+                "ref": 1473
+            }
+        ],
+        "report": "A8-0298/2015",
+        "title": "A8-0298/2015 -  Jos\u00e9 Manuel Fernandes et  G\u00e9rard Deprez - Am 29",
+        "ts": "2015-10-28T13:00:12",
+        "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
+        "voteid": "59146"
+    }
+}

representatives_votes/contrib/parltrack/tests/sync_expected.json

+[
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.369Z",
+        "reference": "2012/2002(INI)",
+        "title": "Agenda for change: the future of EU development policy",
+        "text": "",
+        "created": "2015-12-13T10:11:31.369Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+        "fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 1
+}
+]

representatives_votes/contrib/parltrack/tests/sync_fixture.json

+[
+{
+    "fields": {
+        "updated": "2015-12-13T10:11:31.369Z",
+        "reference": "2012/2002(INI)",
+        "title": "initial title",
+        "text": "",
+        "created": "2015-12-13T10:11:31.369Z",
+        "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
+        "fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
+    },
+    "model": "representatives_votes.dossier",
+    "pk": 1
+}
+]

representatives_votes/contrib/parltrack/tests/test_import.py

-import pytest
-import os
 import copy
+import mock
+import os
+import pytest
 
 from django.core.serializers.json import Deserializer
 from django.core.management import call_command
+from django.test import TestCase
 from representatives_votes.contrib.parltrack import import_dossiers
 from representatives_votes.contrib.parltrack import import_votes
 from representatives_votes.models import Dossier, Proposal, Vote
@@ -50,3 +52,42 @@ def test_parltrack_import_votes():
         'dossiers_expected.json'))
 
     _test_import('votes', import_votes.main)
+
+
+class DossierTest(TestCase):
+    def setUp(self):
+        for model in (Representative, Dossier, Proposal, Vote):
+            model.objects.all().delete()
+
+    def test_parltrack_import_single_dossier(self):
+        call_command('loaddata', os.path.join(os.path.abspath(
+            representatives.__path__[0]), 'fixtures',
+            'representatives_test.json'))
+
+        with self.assertNumQueries(16):
+            _test_import('single', import_dossiers.import_single)
+
+    def test_parltrack_sync_dossier(self):
+        call_command('loaddata', os.path.join(os.path.abspath(
+            representatives.__path__[0]), 'fixtures',
+            'representatives_test.json'))
+        call_command('loaddata', os.path.join(os.path.dirname(__file__),
+            'sync_fixture.json'))
+
+        reference = '2012/2002(INI)'
+        mock_file = os.path.join(os.path.dirname(__file__),
+            'parltrack_dossier_%s.json' % reference.replace('/', '-'))
+        expected_url = '%s/dossier/%s?format=json' % (import_dossiers.BASEURL,
+            reference)
+
+        def callback(stream):
+            import_dossiers.sync_dossier(reference)
+
+        with mock.patch('urllib2.urlopen') as urlopen:
+            with open(mock_file, 'r') as mock_stream:
+                urlopen.return_value = mock_stream
+
+                with self.assertNumQueries(3):
+                    _test_import('sync', callback)
+
+            urlopen.assert_called_with(expected_url)