Commit c061527a authored by Nicolas Joyard's avatar Nicolas Joyard

Import committee votes from parltrack

parent ba69e530
......@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
sleep 10
parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
sleep 10
francedata_download_pipe votes.json.gz francedata_import_votes
if [ "x$1" != "xnoscores" ]; then
......
......@@ -28,6 +28,8 @@ def _parse_date(date_str):
JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
DESTINATION = join('/tmp', 'ep_votes.json')
RE_COMVOTE_REF = re.compile(r'&reference=([^&]+)')
POSITION_MAP = {k: k for k in ('For', 'Against', 'Abstain')}
class Command(object):
......@@ -61,11 +63,118 @@ class Command(object):
vote_data['epref'])
return
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
if 'committee' in vote_data:
return self.parse_committee_vote_data(
vote_data=vote_data,
dossier_pk=dossier_pk
)
else:
return self.parse_proposal_data(
proposal_data=vote_data,
dossier_pk=dossier_pk
)
def parse_proposal_totals(self, data, position_map=POSITION_MAP):
totals = {}
for position in ('For', 'Abstain', 'Against'):
position_data = data.get(position_map[position], {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
totals['total_%s' % position.lower()] = position_total
return totals
def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
logger.info(
u'Looking for votes in proposal {}'.format(proposal.title))
for position in ('For', 'Abstain', 'Against'):
for group_vote_data in data.get(position, {}).get('groups', {}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, data['title'],
proposal.pk, position)
@transaction.atomic
def parse_committee_vote_data(self, proposal_data, dossier_pk):
title = u'{} vote on {}' % (proposal_data['committee'],
proposal_data['doc'])
changed = False
try:
proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
except Proposal.DoesNotExist:
proposal = Proposal(title=title, dossier_id=dossier_pk)
changed = True
try:
ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
except:
logger.debug(u'Cannot find proposal reference for %s' % title)
return
data_map = dict(
datetime=_parse_date(proposal_data['ts']),
reference=ref,
kind='Committee vote'
)
position_map = {
'For': '+',
'Against': '-',
'Abstain': '0'
}
data_map.update(self.parse_proposal_totals(proposal_data,
position_map))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
setattr(proposal, key, value)
changed = True
if changed:
proposal.save()
if self.should_skip(proposal_data):
logger.debug(
u'Skipping votes for dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
self.parse_proposal_votes(proposal, proposal_data, position_map)
@transaction.atomic
def parse_proposal_data(self, proposal_data, dossier_pk):
"""Get or Create a proposal model from raw data"""
......@@ -92,14 +201,7 @@ class Command(object):
kind=proposal_data.get('issue_type')
)
for position in ('For', 'Abstain', 'Against'):
position_data = proposal_data.get(position, {})
position_total = position_data.get('total', 0)
if isinstance(position_total, str) and position_total.isdigit():
position_total = int(position_total)
data_map['total_%s' % position.lower()] = position_total
data_map.update(self.parse_proposal_totals(proposal_data))
for key, value in data_map.items():
if value != getattr(proposal, key, None):
......@@ -111,56 +213,11 @@ class Command(object):
if self.should_skip(proposal_data):
logger.debug(
'Skipping dossier %s', proposal_data.get(
u'Skipping votes for dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
'Looking for votes in proposal {}'.format(proposal_display))
for position in positions:
for group_vote_data in proposal_data.get(
position,
{}).get(
'groups',
{}):
for vote_data in group_vote_data['votes']:
if not isinstance(vote_data, dict):
logger.error('Skipping vote data %s for proposal %s',
vote_data, proposal_data['_id'])
continue
representative_pk = self.get_representative(vote_data)
if representative_pk is None:
logger.error('Could not find mep for %s', vote_data)
continue
representative_name = vote_data.get('orig', '')
changed = False
try:
vote = Vote.objects.get(
representative_id=representative_pk,
proposal_id=proposal.pk)
except Vote.DoesNotExist:
vote = Vote(proposal_id=proposal.pk,
representative_id=representative_pk)
changed = True
if vote.position != position.lower():
changed = True
vote.position = position.lower()
if vote.representative_name != representative_name:
changed = True
vote.representative_name = representative_name
if changed:
vote.save()
logger.debug('Save vote %s for MEP %s on %s #%s to %s',
vote.pk, representative_pk, proposal_data['title'],
proposal.pk, position)
self.parse_proposal_votes(proposal, proposal_data)
return proposal
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment