Commit 7ef06aa5 authored by njoyard's avatar njoyard

Merge branch 'parltrack-committee-votes' into 'master'

Import committee votes from parltrack



See merge request !170
parents ba69e530 17f6bb68
Pipeline #193 passed with stages
in 10 minutes and 13 seconds
...@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes ...@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
sleep 10 sleep 10
parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
sleep 10
francedata_download_pipe votes.json.gz francedata_import_votes francedata_download_pipe votes.json.gz francedata_import_votes
if [ "x$1" != "xnoscores" ]; then if [ "x$1" != "xnoscores" ]; then
......
François Asensi,fasensi@assemblee-nationale.fr,GDR,France [FR] François Asensi,fasensi@assemblee-nationale.fr,GDR,France [FR]
Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR] Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR] Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR] Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
Paul BRANNEN,paul.brannen@europarl.europa.eu,SD,United Kingdom [GB] Paul BRANNEN,paul.brannen@europarl.europa.eu,SD,United Kingdom [GB]
...@@ -9,7 +9,7 @@ Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,Fran ...@@ -9,7 +9,7 @@ Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,Fran
Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR] Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL] Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
Albert DESS,albert.dess@europarl.europa.eu,EPP,Germany [DE] Albert DESS,albert.dess@europarl.europa.eu,EPP,Germany [DE]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR] Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR] Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR] Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
Daniel Fasquelle,dfasquelle@assemblee-nationale.fr,LR,France [FR] Daniel Fasquelle,dfasquelle@assemblee-nationale.fr,LR,France [FR]
...@@ -27,19 +27,19 @@ Werner LANGEN,werner.langen@europarl.europa.eu,EPP,Germany [DE] ...@@ -27,19 +27,19 @@ Werner LANGEN,werner.langen@europarl.europa.eu,EPP,Germany [DE]
Jo LEINEN,jo.leinen@europarl.europa.eu,SD,Germany [DE] Jo LEINEN,jo.leinen@europarl.europa.eu,SD,Germany [DE]
Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR] Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR] Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR] Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
Arne LIETZ,arne.lietz@europarl.europa.eu,SD,Germany [DE] Arne LIETZ,arne.lietz@europarl.europa.eu,SD,Germany [DE]
Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES] Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR] Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR] Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Gesine MEISSNER,gesine.meissner@europarl.europa.eu,ALDE,Germany [DE] Gesine MEISSNER,gesine.meissner@europarl.europa.eu,ALDE,Germany [DE]
Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR] Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE] Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
Paul NUTTALL,paul.nuttall@europarl.europa.eu,EFDD,United Kingdom [GB] Paul NUTTALL,paul.nuttall@europarl.europa.eu,EFDD,United Kingdom [GB]
Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB] Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]
Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL] Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR] François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR] Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR] Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR] Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
André Santini,asantini@assemblee-nationale.fr,UDI,France [FR] André Santini,asantini@assemblee-nationale.fr,UDI,France [FR]
......
...@@ -7,10 +7,10 @@ Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES] ...@@ -7,10 +7,10 @@ Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL] Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE] Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL] Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR] Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR] Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
Claude Goasguen,cgoasguen@assemblee-nationale.fr,LR,France [FR] Claude Goasguen,cgoasguen@assemblee-nationale.fr,LR,France [FR]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR] Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR] Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
Philippe Vigier,pvigier@assemblee-nationale.fr,UDI,France [FR] Philippe Vigier,pvigier@assemblee-nationale.fr,UDI,France [FR]
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR] Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
...@@ -18,7 +18,7 @@ Philippe Houillon,phouillon@assemblee-nationale.fr,LR,France [FR] ...@@ -18,7 +18,7 @@ Philippe Houillon,phouillon@assemblee-nationale.fr,LR,France [FR]
Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR] Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR] Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
Pascale Got,pgot@assemblee-nationale.fr,SER,France [FR] Pascale Got,pgot@assemblee-nationale.fr,SER,France [FR]
Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR] Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,France [FR] Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,France [FR]
Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR] Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR] Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
...@@ -28,9 +28,9 @@ Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR] ...@@ -28,9 +28,9 @@ Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR] Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR] François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR] Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR] Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR] Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR] Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
Czesław HOC,czeslaw.hoc@europarl.europa.eu,ECR,Poland [PL] Czesław HOC,czeslaw.hoc@europarl.europa.eu,ECR,Poland [PL]
Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE] Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB] Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]
......
Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR] Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
Udo BULLMANN,udo.bullmann@europarl.europa.eu,SD,Germany [DE] Udo BULLMANN,udo.bullmann@europarl.europa.eu,SD,Germany [DE]
Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR] Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
María Teresa GIMÉNEZ BARBAT,teresa.gimenezbarbat@europarl.europa.eu,ALDE,Ireland [IE] María Teresa GIMÉNEZ BARBAT,teresa.gimenezbarbat@europarl.europa.eu,ALDE,Ireland [IE]
Sylvia-Yvonne KAUFMANN,sylvia-yvonne.kaufmann@europarl.europa.eu,SD,Germany [DE] Sylvia-Yvonne KAUFMANN,sylvia-yvonne.kaufmann@europarl.europa.eu,SD,Germany [DE]
Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE] Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR] Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR] Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR] Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
Hermann WINKLER,hermann.winkler@europarl.europa.eu,EPP,Germany [DE] Hermann WINKLER,hermann.winkler@europarl.europa.eu,EPP,Germany [DE]
...@@ -3,9 +3,10 @@ ...@@ -3,9 +3,10 @@
from core.views import GridListMixin, PaginationMixin, CSVDownloadMixin, \ from core.views import GridListMixin, PaginationMixin, CSVDownloadMixin, \
ActiveLegislatureMixin, SortMixin ActiveLegislatureMixin, SortMixin
from django.db import models
from django.views import generic from django.views import generic
from representatives.models import Representative from representatives.models import Representative, Email
from ..filters import RepresentativeFilter from ..filters import RepresentativeFilter
from .representative_mixin import RepresentativeViewMixin from .representative_mixin import RepresentativeViewMixin
...@@ -72,7 +73,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin, ...@@ -72,7 +73,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
def get_csv_results(self, context, **kwargs): def get_csv_results(self, context, **kwargs):
qs = self.get_queryset() qs = self.get_queryset()
qs = qs.prefetch_related('email_set') qs = qs.prefetch_related(
models.Prefetch(
'email_set',
queryset=Email.objects.order_by('email')
)
)
return [self.add_representative_country_and_main_mandate(r) return [self.add_representative_country_and_main_mandate(r)
for r in qs] for r in qs]
......
...@@ -28,6 +28,8 @@ def _parse_date(date_str): ...@@ -28,6 +28,8 @@ def _parse_date(date_str):
JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz' JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION = join('/tmp', 'ep_votes.json') DESTINATION = join('/tmp', 'ep_votes.json')
RE_COMVOTE_REF = re.compile(r'&reference=([^&]+)')
POSITION_MAP = {k: k for k in ('For', 'Against', 'Abstain')}
class Command(object): class Command(object):
...@@ -49,9 +51,14 @@ class Command(object): ...@@ -49,9 +51,14 @@ class Command(object):
""" """
Parse data from parltrack votes db dumps (1 proposal) Parse data from parltrack votes db dumps (1 proposal)
""" """
if 'epref' not in vote_data.keys(): keys = vote_data.keys()
if 'ep_ref' in keys:
vote_data['epref'] = vote_data['ep_ref']
elif 'epref' not in keys:
logger.debug('Could not import data without epref %s', logger.debug('Could not import data without epref %s',
vote_data['title']) vote_data.get('title',
vote_data.get('doc',
vote_data.get('url', '?'))))
return return
dossier_pk = self.get_dossier(vote_data['epref']) dossier_pk = self.get_dossier(vote_data['epref'])
...@@ -61,17 +68,122 @@ class Command(object): ...@@ -61,17 +68,122 @@ class Command(object):
vote_data['epref']) vote_data['epref'])
return return
return self.parse_proposal_data( if 'committee' in vote_data:
proposal_data=vote_data, return self.parse_committee_vote_data(
dossier_pk=dossier_pk proposal_data=vote_data,
dossier_pk=dossier_pk
)
else:
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
)
def parse_proposal_totals(self, data, position_map=POSITION_MAP):
totals = {}
for position in ('For', 'Abstain', 'Against'):
position_data = data.get(position_map[position], {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
totals['total_%s' % position.lower()] = position_total
return totals
def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
logger.info(
u'Looking for votes in proposal {}'.format(proposal.title))
for position in ('For', 'Abstain', 'Against'):
for group_vote_data in data.get(position_map[position],
{}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk,
proposal.title, proposal.pk, position)
@transaction.atomic
def parse_committee_vote_data(self, proposal_data, dossier_pk):
title = u'%s vote on %s' % (proposal_data['committee'],
proposal_data['doc'])
changed = False
try:
proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
except Proposal.DoesNotExist:
proposal = Proposal(title=title, dossier_id=dossier_pk)
changed = True
try:
ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
except:
logger.debug(u'Cannot find proposal reference for %s' % title)
return
data_map = dict(
datetime=_parse_date(proposal_data['ts']),
reference=ref,
kind='committee-vote'
) )
position_map = {
'For': '+',
'Against': '-',
'Abstain': '0'
}
data_map.update(self.parse_proposal_totals(proposal_data,
position_map))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug(
u'Skipping votes for dossier %s', proposal_data.get(
'epref', title))
return
self.parse_proposal_votes(proposal, proposal_data, position_map)
@transaction.atomic @transaction.atomic
def parse_proposal_data(self, proposal_data, dossier_pk): def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data""" """Get or Create a proposal model from raw data"""
proposal_display = '{} ({})'.format(proposal_data['title'].encode(
'utf-8'), proposal_data.get('report', '').encode('utf-8'))
if 'issue_type' not in proposal_data.keys(): if 'issue_type' not in proposal_data.keys():
logger.debug('This proposal data without issue_type: %s', logger.debug('This proposal data without issue_type: %s',
proposal_data['epref']) proposal_data['epref'])
...@@ -92,14 +204,7 @@ class Command(object): ...@@ -92,14 +204,7 @@ class Command(object):
kind=proposal_data.get('issue_type') kind=proposal_data.get('issue_type')
) )
for position in ('For', 'Abstain', 'Against'): data_map.update(self.parse_proposal_totals(proposal_data))
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
for key, value in data_map.items(): for key, value in data_map.items():
if value != getattr(proposal, key, None): if value != getattr(proposal, key, None):
...@@ -111,56 +216,11 @@ class Command(object): ...@@ -111,56 +216,11 @@ class Command(object):
if self.should_skip(proposal_data): if self.should_skip(proposal_data):
logger.debug( logger.debug(
'Skipping dossier %s', proposal_data.get( u'Skipping votes for dossier %s', proposal_data.get(
'epref', proposal_data['title'])) 'epref', proposal_data['title']))
return return
positions = ['For', 'Abstain', 'Against'] self.parse_proposal_votes(proposal, proposal_data)
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
return proposal return proposal
......
[ [
{
"fields": {
"total_for": 1,
"description": "",
"reference": "TEST-COM-VOTE",
"title": "LIBE vote on Test Committee Vote",
"dossier": [
"2012/2002(INI)"
],
"kind": "committee-vote",
"datetime": "2012-10-23T16:31:10Z",
"total_against": 1,
"total_abstain": 0
},
"model": "representatives_votes.proposal"
},
{ {
"fields": { "fields": {
"representative_name": "", "representative_name": "",
...@@ -163,5 +179,35 @@ ...@@ -163,5 +179,35 @@
}, },
"model": "representatives_votes.vote", "model": "representatives_votes.vote",
"pk": 11 "pk": 11
},
{
"fields": {
"representative_name": "",
"position": "for",
"proposal": [
"LIBE vote on Test Committee Vote",
"2012/2002(INI)"
],
"representative": [
"hubert-pirker-1948-10-03"
]
},
"model": "representatives_votes.vote",
"pk": 12
},
{
"fields": {
"representative_name": "",
"position": "against",
"proposal": [
"LIBE vote on Test Committee Vote",
"2012/2002(INI)"
],
"representative": [
"olle-ludvigsson-1948-10-28"
]
},
"model": "representatives_votes.vote",
"pk": 13
} }
] ]
...@@ -281,5 +281,45 @@ ...@@ -281,5 +281,45 @@
"ts": "2015-10-28T13:01:09", "ts": "2015-10-28T13:01:09",
"url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml", "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
"voteid": "59151" "voteid": "59151"
},
{
"url": "http://www.europarl.europa.eu/sides/getDoc.do?type=REPORT&mode=XML&reference=TEST-COM-VOTE&language=EN",
"doc": "Test Committee Vote",
"+": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 2307,
"name": "Pirker"
}
]
}
],
"total": "1"
},
"ep_ref": "2012/2002(INI)",
"-": {
"groups": [
{
"group": "PPE",
"votes": [
{
"ep_id": 96673,
"name": "Ludvigsson"
}
]
}
],
"total": "1"
},
"ts": "2012-10-23T18:31:10",
"0": {
"total": 0,
"groups": []
},
"committee": "LIBE",
"_id": "56617c15edd52ed712fc7901"
} }
] ]
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment