Commit 04c8dffa authored by Jamesie Pic's avatar Jamesie Pic

Added parltrack import commands in contrib

Also:

- code was PEP8'ed
- tests were added,
- untested code was removed,
- setup.py was fixed and requirements.txt is gone,
parent b85d511e
[run]
omit = representatives_votes/tests/*
omit = representatives_votes/migrations/*
sudo: false
language: python
env:
- DJANGO="django>1.8,<1.9" DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
python:
- "2.7"
- "2.7"
before_install:
- pip install codecov
install:
- pip install django
- pip install -e git+https://github.com/political-memory/django-representatives.git#egg=representatives
- pip install -e .
- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov
- pip install https://github.com/political-memory/django-representatives/archive/parltrack.tar.gz#egg=django-representatives
- pip install -e .
script:
- test_project/manage.py migrate
- django-admin migrate
- flake8 representatives_votes/ --exclude migrations --ignore E128
- py.test
- cat representatives_votes/contrib/parltrack/tests/dossiers_fixture.json | parltrack_import_dossiers
- cat representatives_votes/contrib/parltrack/tests/votes_fixture.json | parltrack_import_votes
after_success:
- codecov
include *.rst *.txt README LICENSE AUTHORS CHANGELOG
recursive-include representatives_votes *.html *.css *.js *.py *.po *.mo *.json *.png *.gif
[![Build Status](https://travis-ci.org/political-memory/django-representatives-votes.svg?branch=travis)](https://travis-ci.org/political-memory/django-representatives-votes)
[![codecov.io](https://codecov.io/github/political-memory/django-representatives-votes/coverage.svg?branch=master)](https://codecov.io/github/political-memory/django-representatives-votes?branch=master)
[pytest]
DJANGO_SETTINGS_MODULE=representatives_votes.tests.settings
addopts = --cov=representatives_votes --create-db
# coding: utf-8
from django.contrib import admin
from .models import Dossier, Proposal, Vote
......@@ -10,8 +11,19 @@ class DossierAdmin(admin.ModelAdmin):
class ProposalAdmin(admin.ModelAdmin):
list_display = ('id', 'fingerprint', 'reference', 'dossier_reference', 'title', 'datetime', 'kind', 'total_abstain', 'total_against', 'total_for')
list_display = (
'id',
'fingerprint',
'reference',
'dossier_reference',
'title',
'datetime',
'kind',
'total_abstain',
'total_against',
'total_for')
search_fields = ('reference', 'dossier__reference', 'title', 'fingerprint')
def dossier_reference(self, obj):
return obj.dossier.reference
......@@ -31,7 +43,12 @@ class NoneMatchingFilter(admin.SimpleListFilter):
class VoteAdmin(admin.ModelAdmin):
list_display = ('id', 'proposal_reference', 'position', 'representative', 'representative_name')
list_display = (
'id',
'proposal_reference',
'position',
'representative',
'representative_name')
list_filter = (NoneMatchingFilter,)
def proposal_reference(self, obj):
......
# coding: utf-8
import logging
import sys
import ijson
import django
from django.apps import apps
from representatives_votes.models import Dossier
logger = logging.getLogger(__name__)
URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz'
LOCAL_PATH = 'ep_dossiers.json.xz'
def parse_dossier_data(data):
"""Parse data from parltarck dossier export (1 dossier) Update dossier
if it existed before, this function goal is to import and update a
dossier, not to import all parltrack data
"""
changed = False
ref = data['procedure']['reference']
logger.debug('Processing dossier %s', ref)
try:
dossier = Dossier.objects.get(reference=ref)
except Dossier.DoesNotExist:
dossier = Dossier(reference=ref)
logger.debug('Dossier did not exist')
changed = True
if dossier.title != data['procedure']['title']:
logger.debug('Title changed from "%s" to "%s"', dossier.title,
data['procedure']['title'])
dossier.title = data['procedure']['title']
changed = True
source = data['meta']['source'].replace('&l=en', '')
if dossier.link != source:
logger.debug('Source changed from "%s" to "%s"', dossier.link, source)
dossier.link = source
changed = True
if changed:
logger.info('Updated dossier %s', ref)
dossier.save()
def main(stream=None):
if not apps.ready:
django.setup()
for data in ijson.items(stream or sys.stdin, 'item'):
parse_dossier_data(data)
# coding: utf-8
import logging
import sys
from os.path import join
import django.dispatch
import ijson
import django
from django.apps import apps
from dateutil.parser import parse as date_parse
from django.db import transaction
from django.utils.timezone import make_aware as date_make_aware
from pytz import timezone as date_timezone
from representatives.models import Representative
from representatives_votes.models import Dossier, Proposal, Vote
logger = logging.getLogger(__name__)
vote_pre_import = django.dispatch.Signal(providing_args=['vote_data'])
def _parse_date(date_str):
return date_make_aware(
date_parse(date_str),
date_timezone('Europe/Brussels'))
JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION = join('/tmp', 'ep_votes.json')
class Command(object):
def init_cache(self):
self.cache = dict()
self.index_representatives()
self.index_dossiers()
def parse_vote_data(self, vote_data):
"""
Parse data from parltrack votes db dumps (1 proposal)
"""
if 'epref' not in vote_data.keys():
logger.debug('Could not import data without epref %s',
vote_data['title'])
return
dossier_pk = self.get_dossier(vote_data['epref'])
if not dossier_pk:
logger.debug('Cannot find dossier with remote id %s',
vote_data['epref'])
return
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
)
@transaction.atomic
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(proposal_data['title'].encode(
'utf-8'), proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
return
changed = False
try:
proposal = Proposal.objects.get(title=proposal_data['title'])
except Proposal.DoesNotExist:
proposal = Proposal(title=proposal_data['title'])
changed = True
data_map = dict(
title=proposal_data['title'],
datetime=_parse_date(proposal_data['ts']),
dossier_id=dossier_pk,
reference=proposal_data.get('report'),
kind=proposal_data.get('issue_type')
)
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
for receiver, response in responses:
if response is False:
logger.debug(
'Skipping dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
return proposal
def index_dossiers(self):
self.cache['dossiers'] = {
d[0]: d[1] for d in Dossier.objects.values_list('reference', 'pk')
}
def get_dossier(self, reference):
return self.cache['dossiers'].get(reference, None)
def index_representatives(self):
self.cache['meps'] = {int(l[0]): l[1] for l in
Representative.objects.values_list('remote_id', 'pk')}
def get_representative(self, vote_data):
if vote_data.get('ep_id', None) is None:
return
return self.cache['meps'].get(int(vote_data['ep_id']), None)
def main(stream=None):
if not apps.ready:
django.setup()
command = Command()
command.init_cache()
for vote_data in ijson.items(stream or sys.stdin, 'item'):
command.parse_vote_data(vote_data)
[
{
"fields": {
"updated": "2015-12-13T10:11:31.369Z",
"reference": "2012/2002(INI)",
"title": "Agenda for change: the future of EU development policy",
"text": "",
"created": "2015-12-13T10:11:31.369Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
"fingerprint": "9e2cccdc5f6d22afd008af8b5b55dc193c27c5d6"
},
"model": "representatives_votes.dossier",
"pk": 1
},
{
"fields": {
"updated": "2015-12-13T10:11:31.378Z",
"reference": "2015/2132(BUD)",
"title": "2016 general budget: all sections",
"text": "",
"created": "2015-12-13T10:11:31.378Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)",
"fingerprint": "e6856e0880e701c1022f23d595cc37a9a1cdcca8"
},
"model": "representatives_votes.dossier",
"pk": 2
},
{
"fields": {
"updated": "2015-12-13T10:11:31.388Z",
"reference": "2013/2857(DEA)",
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"text": "",
"created": "2015-12-13T10:11:31.388Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)",
"fingerprint": "50745a1a6e47b8db097c55ef21a4f11fc1ef0d97"
},
"model": "representatives_votes.dossier",
"pk": 3
},
{
"fields": {
"updated": "2015-12-13T10:11:31.398Z",
"reference": "2015/2623(DEA)",
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"text": "",
"created": "2015-12-13T10:11:31.398Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)",
"fingerprint": "65fce7af2e020ff58849d1663f2c30ab0b1a35db"
},
"model": "representatives_votes.dossier",
"pk": 4
},
{
"fields": {
"updated": "2015-12-13T10:11:31.408Z",
"reference": "2009/0051(COD)",
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"text": "",
"created": "2015-12-13T10:11:31.408Z",
"link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)",
"fingerprint": "f2abba201c10df8a7cec1d734f91b7988eec8260"
},
"model": "representatives_votes.dossier",
"pk": 5
}
]
[
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)",
"updated": "2012-10-25T02:02:36.882000"
},
"procedure": {
"dossier_of_the_committee": "DEVE/7/08564",
"legal_basis": [
"Rules of Procedure of the European Parliament EP 048"
],
"reference": "2012/2002(INI)",
"stage_reached": "Procedure completed",
"subject": [
"6.30 Development cooperation"
],
"subtype": "Initiative",
"title": "Agenda for change: the future of EU development policy",
"type": "INI - Own-initiative procedure"
}
},
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)&l=en",
"updated": "2015-11-18T02:23:18.933000"
},
"procedure": {
"dossier_of_the_committee": "BUDE/8/04900;BUDG/8/03789",
"reference": "2015/2132(BUD)",
"stage_reached": "Budgetary conciliation committee convened",
"subject": [
"8.70.56 2016 budget"
],
"subtype": "Budget",
"title": "2016 general budget: all sections",
"type": "BUD - Budgetary procedure"
}
},
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)&l=en",
"updated": "2014-12-06T03:56:21.811000"
},
"procedure": {
"dossier_of_the_committee": "PECH/7/14126",
"geographical_area": [
"Atlantic Ocean area"
],
"reference": "2013/2857(DEA)",
"stage_reached": "Awaiting Council decision on delegated act",
"subject": [
"3.15.01 Fish stocks, conservation of fishery resources",
"3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
"3.15.15 Fisheries agreements and cooperation"
],
"subtype": "Examination of delegated act",
"summary": [
"Supplementing"
],
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"type": "DEA - Delegated acts procedure"
}
},
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)&l=en",
"updated": "2015-07-22T00:46:11.603000"
},
"procedure": {
"dossier_of_the_committee": "PECH/8/03020",
"reference": "2015/2623(DEA)",
"stage_reached": "Awaiting Council decision on delegated act",
"subject": [
"3.15.01 Fish stocks, conservation of fishery resources",
"3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
"3.15.15 Fisheries agreements and cooperation"
],
"subtype": "Examination of delegated act",
"summary": [
"Supplementing"
],
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"type": "DEA - Delegated acts procedure"
}
},
{
"meta": {
"source": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)&l=en",
"updated": "2015-11-04T17:01:33.812000"
},
"procedure": {
"Modified legal basis": "Rules of Procedure of the European Parliament EP 150",
"dossier_of_the_committee": "PECH/7/00288",
"final": {
"title": "Regulation 2010/1236",
"url": "http://eur-lex.europa.eu/smartapi/cgi/sga_doc?smartapi!celexplus!prod!CELEXnumdoc&lg=EN&numdoc=32010R1236"
},
"instrument": "Regulation",
"legal_basis": [
"Treaty on the Functioning of the EU TFEU 043-p2"
],
"reference": "2009/0051(COD)",
"stage_reached": "Procedure completed",
"subject": [
"3.15.01 Fish stocks, conservation of fishery resources",
"3.15.07 Fisheries inspectorate, surveillance of fishing vessels and areas",
"3.15.15 Fisheries agreements and cooperation"
],
"subtype": "Legislation",
"summary": [
"Amended by",
"Repealing Regulation (EC) No 2791/1999"
],
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"type": "COD - Ordinary legislative procedure (ex-codecision procedure)"
}
}
]
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from django.core.management import call_command
from representatives_votes.contrib.parltrack import import_dossiers
from representatives_votes.contrib.parltrack import import_votes
from representatives_votes.models import Dossier, Proposal, Vote
from representatives.models import Representative
from representatives.contrib import parltrack
def _test_import(scenario, callback):
fixture = os.path.join(os.path.dirname(__file__),
'%s_fixture.json' % scenario)
expected = os.path.join(os.path.dirname(__file__),
'%s_expected.json' % scenario)
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated')
with open(fixture, 'r') as f:
callback(f)
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for f in exclude:
if f in compare:
compare.pop(f)
type(obj.object).objects.get(**compare)
@pytest.mark.django_db
def test_parltrack_import_dossiers():
_test_import('dossiers', import_dossiers.main)
@pytest.mark.django_db
def test_parltrack_import_votes():
for model in (Representative, Dossier, Proposal, Vote):
model.objects.all().delete()
call_command('loaddata', os.path.join(os.path.abspath(
parltrack.__path__[0]), 'tests', 'representatives_expected.json'))
call_command('loaddata', os.path.join(os.path.dirname(__file__),
'dossiers_expected.json'))
_test_import('votes', import_votes.main)
[
{
"fields": {
"representative_name": "",
"position": "abstain",
"proposal": 1,
"representative": 2
},
"model": "representatives_votes.vote",
"pk": 1
},
{
"fields": {
"representative_name": "",
"position": "abstain",
"proposal": 1,
"representative": 1
},
"model": "representatives_votes.vote",
"pk": 2
},
{
"fields": {
"representative_name": "",
"position": "for",
"proposal": 2,
"representative": 2
},
"model": "representatives_votes.vote",
"pk": 3