Commit d520f06d authored by Nicolas Joyard's avatar Nicolas Joyard Committed by Jamesie Pic

Add single dossier sync from parltrack

parent dc0323a7
......@@ -7,7 +7,7 @@ python:
before_install:
- pip install codecov
install:
- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov django-responsediff
- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov django-responsediff mock
- pip install https://github.com/political-memory/django-representatives/archive/parltrack.tar.gz#egg=django-representatives
- pip install -e .[api]
script:
......
# coding: utf-8
import logging
import sys
import urllib2
import ijson
import django
from django.apps import apps
from representatives_votes.models import Dossier
from .import_votes import Command
logger = logging.getLogger(__name__)
BASEURL = 'http://parltrack.euwiki.org/'
URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz'
LOCAL_PATH = 'ep_dossiers.json.xz'
......@@ -47,6 +50,27 @@ def parse_dossier_data(data):
logger.info('Updated dossier %s', ref)
dossier.save()
if 'votes' in data.keys() and 'epref' in data['votes']:
command = Command()
command.init_cache()
command.parse_vote_data(data['votes'])
def sync_dossier(reference):
url = '%s/dossier/%s?format=json' % (BASEURL, reference)
logger.debug('Syncing dossier from %s' % url)
with urllib2.urlopen(url) as stream:
import_single(stream)
def import_single(stream):
if not apps.ready:
django.setup()
for data in ijson.items(stream, ''):
parse_dossier_data(data)
def main(stream=None):
if not apps.ready:
......
[
{
"fields": {
"updated": "2015-12-13T10:11:31.378Z",
"reference": "2015/2132(BUD)",
"title": "2016 general budget: all sections",
"text": "",
"created": "2015-12-13T10:11:31.378Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
"fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
},
"model": "representatives_votes.dossier",
"pk": 1
},
{
"fields": {
"representative_name": "",
"position": "for",
"proposal": 1,
"representative": 1
},
"model": "representatives_votes.vote",
"pk": 2
},
{
"fields": {
"representative_name": "",
"position": "against",
"proposal": 1,
"representative": 2
},
"model": "representatives_votes.vote",
"pk": 3
}
]
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)&l=en",
"updated": "2015-11-18T02:23:18.933000"
},
"procedure": {
"dossier_of_the_committee": "BUDE/8/04900;BUDG/8/03789",
"reference": "2015/2132(BUD)",
"stage_reached": "Budgetary conciliation committee convened",
"subject": [
"8.70.56 2016 budget"
],
"subtype": "Budget",
"title": "2016 general budget: all sections",
"type": "BUD - Budgetary procedure"
},
"votes": {
"For": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 2307,
"name": "Pirker",
"userid": 4611
}
]
}
],
"total": "1"
},
"Against": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 96673,
"name": "Ludvigsson",
"userid": 5860
}
]
}
],
"total": "1"
},
"_id": "56617a11ecc52ed712fc6b15",
"dossierid": "55a4580ed1d1c57c1e000002",
"epref": "2015/2132(BUD)",
"eptitle": "2016 general budget: all sections",
"issue_type": "Am 29",
"rapporteur": [
{
"name": "Jos\u00e9 Manuel Fernandes",
"ref": 96899
},
{
"name": " G\u00e9rard Deprez",
"ref": 1473
}
],
"report": "A8-0298/2015",
"title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 29",
"ts": "2015-10-28T13:00:12",
"url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
"voteid": "59146"
}
}
[
{
"fields": {
"updated": "2015-12-13T10:11:31.369Z",
"reference": "2012/2002(INI)",
"title": "Agenda for change: the future of EU development policy",
"text": "",
"created": "2015-12-13T10:11:31.369Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
"fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
},
"model": "representatives_votes.dossier",
"pk": 1
}
]
[
{
"fields": {
"updated": "2015-12-13T10:11:31.369Z",
"reference": "2012/2002(INI)",
"title": "initial title",
"text": "",
"created": "2015-12-13T10:11:31.369Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
"fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
},
"model": "representatives_votes.dossier",
"pk": 1
}
]
import pytest
import os
import copy
import mock
import os
import pytest
from django.core.serializers.json import Deserializer
from django.core.management import call_command
from django.test import TestCase
from representatives_votes.contrib.parltrack import import_dossiers
from representatives_votes.contrib.parltrack import import_votes
from representatives_votes.models import Dossier, Proposal, Vote
......@@ -50,3 +52,42 @@ def test_parltrack_import_votes():
'dossiers_expected.json'))
_test_import('votes', import_votes.main)
class DossierTest(TestCase):
def setUp(self):
for model in (Representative, Dossier, Proposal, Vote):
model.objects.all().delete()
def test_parltrack_import_single_dossier(self):
call_command('loaddata', os.path.join(os.path.abspath(
representatives.__path__[0]), 'fixtures',
'representatives_test.json'))
with self.assertNumQueries(16):
_test_import('single', import_dossiers.import_single)
def test_parltrack_sync_dossier(self):
call_command('loaddata', os.path.join(os.path.abspath(
representatives.__path__[0]), 'fixtures',
'representatives_test.json'))
call_command('loaddata', os.path.join(os.path.dirname(__file__),
'sync_fixture.json'))
reference = '2012/2002(INI)'
mock_file = os.path.join(os.path.dirname(__file__),
'parltrack_dossier_%s.json' % reference.replace('/', '-'))
expected_url = '%s/dossier/%s?format=json' % (import_dossiers.BASEURL,
reference)
def callback(stream):
import_dossiers.sync_dossier(reference)
with mock.patch('urllib2.urlopen') as urlopen:
with open(mock_file, 'r') as mock_stream:
urlopen.return_value = mock_stream
with self.assertNumQueries(3):
_test_import('sync', callback)
urlopen.assert_called_with(expected_url)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment