Skip to content

GitLab

memopol
memopol
Commit 7ef06aa5 authored by njoyard's avatar njoyard
Browse files
Options

Merge branch 'parltrack-committee-votes' into 'master' 

Import committee votes from parltrack



See merge request !170
parents ba69e530 17f6bb68
Hide whitespace changes
Inline Side-by-side
Showing with 235 additions and 79 deletions
bin/update_votes
View file @ 7ef06aa5
......@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
sleep 10
parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
sleep 10
francedata_download_pipe votes.json.gz francedata_import_votes
if [ "x$1" != "xnoscores" ]; then
......
src/memopol/tests/response_fixtures/RepresentativeListTest.test_csv.content
View file @ 7ef06aa5
François Asensi,fasensi@assemblee-nationale.fr,GDR,France [FR]
Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR]
Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
Paul BRANNEN,paul.brannen@europarl.europa.eu,SD,United Kingdom [GB]
......@@ -9,7 +9,7 @@ Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,Fran
Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
Albert DESS,albert.dess@europarl.europa.eu,EPP,Germany [DE]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
Daniel Fasquelle,dfasquelle@assemblee-nationale.fr,LR,France [FR]
......@@ -27,19 +27,19 @@ Werner LANGEN,werner.langen@europarl.europa.eu,EPP,Germany [DE]
Jo LEINEN,jo.leinen@europarl.europa.eu,SD,Germany [DE]
Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR]
Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
Arne LIETZ,arne.lietz@europarl.europa.eu,SD,Germany [DE]
Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Gesine MEISSNER,gesine.meissner@europarl.europa.eu,ALDE,Germany [DE]
Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR]
Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
Paul NUTTALL,paul.nuttall@europarl.europa.eu,EFDD,United Kingdom [GB]
Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]
Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR]
Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
André Santini,asantini@assemblee-nationale.fr,UDI,France [FR]
......
src/memopol/tests/response_fixtures/RepresentativeListTest.test_order_csv.content
View file @ 7ef06aa5
......@@ -7,10 +7,10 @@ Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR]
Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR]
Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
Claude Goasguen,cgoasguen@assemblee-nationale.fr,LR,France [FR]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
Philippe Vigier,pvigier@assemblee-nationale.fr,UDI,France [FR]
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
......@@ -18,7 +18,7 @@ Philippe Houillon,phouillon@assemblee-nationale.fr,LR,France [FR]
Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
Pascale Got,pgot@assemblee-nationale.fr,SER,France [FR]
Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR]
Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,France [FR]
Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
......@@ -28,9 +28,9 @@ Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR]
Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
Czesław HOC,czeslaw.hoc@europarl.europa.eu,ECR,Poland [PL]
Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]
......
src/memopol/tests/response_fixtures/RepresentativeListTest.test_search_csv.content
View file @ 7ef06aa5
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
Udo BULLMANN,udo.bullmann@europarl.europa.eu,SD,Germany [DE]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
María Teresa GIMÉNEZ BARBAT,teresa.gimenezbarbat@europarl.europa.eu,ALDE,Ireland [IE]
Sylvia-Yvonne KAUFMANN,sylvia-yvonne.kaufmann@europarl.europa.eu,SD,Germany [DE]
Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
Hermann WINKLER,hermann.winkler@europarl.europa.eu,EPP,Germany [DE]
src/memopol/views/representative_list.py
View file @ 7ef06aa5
......@@ -3,9 +3,10 @@
from core.views import GridListMixin, PaginationMixin, CSVDownloadMixin, \
ActiveLegislatureMixin, SortMixin
from django.db import models
from django.views import generic
from representatives.models import Representative
from representatives.models import Representative, Email
from ..filters import RepresentativeFilter
from .representative_mixin import RepresentativeViewMixin
......@@ -72,7 +73,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
def get_csv_results(self, context, **kwargs):
qs = self.get_queryset()
qs = qs.prefetch_related('email_set')
qs = qs.prefetch_related(
models.Prefetch(
'email_set',
queryset=Email.objects.order_by('email')
)
)
return [self.add_representative_country_and_main_mandate(r)
for r in qs]
......
src/representatives_votes/contrib/parltrack/import_votes.py
View file @ 7ef06aa5
......@@ -28,6 +28,8 @@ def _parse_date(date_str):
JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION = join('/tmp', 'ep_votes.json')
RE_COMVOTE_REF = re.compile(r'&reference=([^&]+)')
POSITION_MAP = {k: k for k in ('For', 'Against', 'Abstain')}
class Command(object):
......@@ -49,9 +51,14 @@ class Command(object):
"""
Parse data from parltrack votes db dumps (1 proposal)
"""
if 'epref' not in vote_data.keys():
keys = vote_data.keys()
if 'ep_ref' in keys:
vote_data['epref'] = vote_data['ep_ref']
elif 'epref' not in keys:
logger.debug('Could not import data without epref %s',
vote_data['title'])
vote_data.get('title',
vote_data.get('doc',
vote_data.get('url', '?'))))
return
dossier_pk = self.get_dossier(vote_data['epref'])
......@@ -61,17 +68,122 @@ class Command(object):
vote_data['epref'])
return
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
if 'committee' in vote_data:
return self.parse_committee_vote_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
)
else:
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
)
def parse_proposal_totals(self, data, position_map=POSITION_MAP):
totals = {}
for position in ('For', 'Abstain', 'Against'):
position_data = data.get(position_map[position], {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
totals['total_%s' % position.lower()] = position_total
return totals
def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
logger.info(
u'Looking for votes in proposal {}'.format(proposal.title))
for position in ('For', 'Abstain', 'Against'):
for group_vote_data in data.get(position_map[position],
{}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk,
proposal.title, proposal.pk, position)
@transaction.atomic
def parse_committee_vote_data(self, proposal_data, dossier_pk):
title = u'%s vote on %s' % (proposal_data['committee'],
proposal_data['doc'])
changed = False
try:
proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
except Proposal.DoesNotExist:
proposal = Proposal(title=title, dossier_id=dossier_pk)
changed = True
try:
ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
except:
logger.debug(u'Cannot find proposal reference for %s' % title)
return
data_map = dict(
datetime=_parse_date(proposal_data['ts']),
reference=ref,
kind='committee-vote'
)
position_map = {
'For': '+',
'Against': '-',
'Abstain': '0'
}
data_map.update(self.parse_proposal_totals(proposal_data,
position_map))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug(
u'Skipping votes for dossier %s', proposal_data.get(
'epref', title))
return
self.parse_proposal_votes(proposal, proposal_data, position_map)
@transaction.atomic
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(proposal_data['title'].encode(
'utf-8'), proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s',
proposal_data['epref'])
......@@ -92,14 +204,7 @@ class Command(object):
kind=proposal_data.get('issue_type')
)
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
data_map.update(self.parse_proposal_totals(proposal_data))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
......@@ -111,56 +216,11 @@ class Command(object):
if self.should_skip(proposal_data):
logger.debug(
'Skipping dossier %s', proposal_data.get(
u'Skipping votes for dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
self.parse_proposal_votes(proposal, proposal_data)
return proposal
......
src/representatives_votes/contrib/parltrack/tests/votes_expected.json
View file @ 7ef06aa5
[
{
"fields": {
"total_for": 1,
"description": "",
"reference": "TEST-COM-VOTE",
"title": "LIBE vote on Test Committee Vote",
"dossier": [
"2012/2002(INI)"
],
"kind": "committee-vote",
"datetime": "2012-10-23T16:31:10Z",
"total_against": 1,
"total_abstain": 0
},
"model": "representatives_votes.proposal"
},
{
"fields": {
"representative_name": "",
......@@ -163,5 +179,35 @@
},
"model": "representatives_votes.vote",
"pk": 11
},
{
"fields": {
"representative_name": "",
"position": "for",
"proposal": [
"LIBE vote on Test Committee Vote",
"2012/2002(INI)"
],
"representative": [
"hubert-pirker-1948-10-03"
]
},
"model": "representatives_votes.vote",
"pk": 12
},
{
"fields": {
"representative_name": "",
"position": "against",
"proposal": [
"LIBE vote on Test Committee Vote",
"2012/2002(INI)"
],
"representative": [
"olle-ludvigsson-1948-10-28"
]
},
"model": "representatives_votes.vote",
"pk": 13
}
]
src/representatives_votes/contrib/parltrack/tests/votes_fixture.json
View file @ 7ef06aa5
......@@ -281,5 +281,45 @@
"ts": "2015-10-28T13:01:09",
"url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
"voteid": "59151"
},
{
"url": "http://www.europarl.europa.eu/sides/getDoc.do?type=REPORT&mode=XML&reference=TEST-COM-VOTE&language=EN",
"doc": "Test Committee Vote",
"+": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 2307,
"name": "Pirker"
}
]
}
],
"total": "1"
},
"ep_ref": "2012/2002(INI)",
"-": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 96673,
"name": "Ludvigsson"
}
]
}
],
"total": "1"
},
"ts": "2012-10-23T18:31:10",
"0": {
"total": 0,
"groups": []
},
"committee": "LIBE",
"_id": "56617c15edd52ed712fc7901"
}
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment