Import committee votes from parltrack

bin/update_votes

@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
 
 sleep 10
 
+parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
+
+sleep 10
+
 francedata_download_pipe votes.json.gz francedata_import_votes
 
 if [ "x$1" != "xnoscores" ]; then

src/representatives_votes/contrib/parltrack/import_votes.py

@@ -28,6 +28,8 @@ def _parse_date(date_str):
 
 JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
 DESTINATION = join('/tmp', 'ep_votes.json')
+RE_COMVOTE_REF = re.compile(r'&reference=([^&]+)')
+POSITION_MAP = {k: k for k in ('For', 'Against', 'Abstain')}
 
 
 class Command(object):
@@ -61,11 +63,118 @@ class Command(object):
                          vote_data['epref'])
             return
 
-        return self.parse_proposal_data(
-            proposal_data=vote_data,
-            dossier_pk=dossier_pk
+        if 'committee' in vote_data:
+            return self.parse_committee_vote_data(
+                vote_data=vote_data,
+                dossier_pk=dossier_pk
+            )
+        else:
+            return self.parse_proposal_data(
+                proposal_data=vote_data,
+                dossier_pk=dossier_pk
+            )
+
+    def parse_proposal_totals(self, data, position_map=POSITION_MAP):
+        totals = {}
+
+        for position in ('For', 'Abstain', 'Against'):
+            position_data = data.get(position_map[position], {})
+            position_total = position_data.get('total', 0)
+
+            if isinstance(position_total, str) and position_total.isdigit():
+                position_total = int(position_total)
+
+            totals['total_%s' % position.lower()] = position_total
+
+        return totals
+
+    def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
+        logger.info(
+            u'Looking for votes in proposal {}'.format(proposal.title))
+
+        for position in ('For', 'Abstain', 'Against'):
+            for group_vote_data in data.get(position, {}).get('groups', {}):
+                for vote_data in group_vote_data['votes']:
+                    if not isinstance(vote_data, dict):
+                        logger.error('Skipping vote data %s for proposal %s',
+                                     vote_data, data['_id'])
+                        continue
+
+                    representative_pk = self.get_representative(vote_data)
+
+                    if representative_pk is None:
+                        logger.error('Could not find mep for %s', vote_data)
+                        continue
+
+                    changed = False
+                    try:
+                        vote = Vote.objects.get(
+                            representative_id=representative_pk,
+                            proposal_id=proposal.pk)
+                    except Vote.DoesNotExist:
+                        vote = Vote(proposal_id=proposal.pk,
+                                    representative_id=representative_pk)
+                        changed = True
+
+                    if vote.position != position.lower():
+                        changed = True
+                        vote.position = position.lower()
+
+                    if changed:
+                        vote.save()
+                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
+                                     vote.pk, representative_pk, data['title'],
+                                     proposal.pk, position)
+
+    @transaction.atomic
+    def parse_committee_vote_data(self, proposal_data, dossier_pk):
+        title = u'{} vote on {}' % (proposal_data['committee'],
+                                    proposal_data['doc'])
+        changed = False
+
+        try:
+            proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
+        except Proposal.DoesNotExist:
+            proposal = Proposal(title=title, dossier_id=dossier_pk)
+            changed = True
+
+        try:
+            ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
+        except:
+            logger.debug(u'Cannot find proposal reference for %s' % title)
+            return
+
+        data_map = dict(
+            datetime=_parse_date(proposal_data['ts']),
+            reference=ref,
+            kind='Committee vote'
         )
 
+        position_map = {
+            'For': '+',
+            'Against': '-',
+            'Abstain': '0'
+        }
+
+        data_map.update(self.parse_proposal_totals(proposal_data,
+                                                   position_map))
+
+        for key, value in data_map.items():
+            if value != getattr(proposal, key, None):
+                setattr(proposal, key, value)
+                changed = True
+
+        if changed:
+            proposal.save()
+
+        if self.should_skip(proposal_data):
+            logger.debug(
+                u'Skipping votes for dossier %s', proposal_data.get(
+                    'epref', proposal_data['title']))
+            return
+
+        self.parse_proposal_votes(proposal, proposal_data, position_map)
+
     @transaction.atomic
     def parse_proposal_data(self, proposal_data, dossier_pk):
         """Get or Create a proposal model from raw data"""
@@ -92,14 +201,7 @@ class Command(object):
             kind=proposal_data.get('issue_type')
         )
 
-        for position in ('For', 'Abstain', 'Against'):
-            position_data = proposal_data.get(position, {})
-            position_total = position_data.get('total', 0)
-
-            if isinstance(position_total, str) and position_total.isdigit():
-                position_total = int(position_total)
-
-            data_map['total_%s' % position.lower()] = position_total
+        data_map.update(self.parse_proposal_totals(proposal_data))
 
         for key, value in data_map.items():
             if value != getattr(proposal, key, None):
@@ -111,56 +213,11 @@ class Command(object):
 
         if self.should_skip(proposal_data):
             logger.debug(
-                'Skipping dossier %s', proposal_data.get(
+                u'Skipping votes for dossier %s', proposal_data.get(
                     'epref', proposal_data['title']))
             return
 
-        positions = ['For', 'Abstain', 'Against']
-        logger.info(
-            'Looking for votes in proposal {}'.format(proposal_display))
-        for position in positions:
-            for group_vote_data in proposal_data.get(
-                    position,
-                    {}).get(
-                    'groups',
-                    {}):
-                for vote_data in group_vote_data['votes']:
-                    if not isinstance(vote_data, dict):
-                        logger.error('Skipping vote data %s for proposal %s',
-                                     vote_data, proposal_data['_id'])
-                        continue
-
-                    representative_pk = self.get_representative(vote_data)
-
-                    if representative_pk is None:
-                        logger.error('Could not find mep for %s', vote_data)
-                        continue
-
-                    representative_name = vote_data.get('orig', '')
-
-                    changed = False
-                    try:
-                        vote = Vote.objects.get(
-                            representative_id=representative_pk,
-                            proposal_id=proposal.pk)
-                    except Vote.DoesNotExist:
-                        vote = Vote(proposal_id=proposal.pk,
-                            representative_id=representative_pk)
-                        changed = True
-
-                    if vote.position != position.lower():
-                        changed = True
-                        vote.position = position.lower()
-
-                    if vote.representative_name != representative_name:
-                        changed = True
-                        vote.representative_name = representative_name
-
-                    if changed:
-                        vote.save()
-                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
-                            vote.pk, representative_pk, proposal_data['title'],
-                            proposal.pk, position)
+        self.parse_proposal_votes(proposal, proposal_data)
 
         return proposal