Merge branch 'parltrack-committee-votes' into 'master'

Import committee votes from parltrack See merge request !170

Merge branch 'parltrack-committee-votes' into 'master'
Import committee votes from parltrack See merge request !170
7ef06aa5 · njoyard · ba69e530 · 17f6bb68 · 7ef06aa5 · 7ef06aa5
Commit 7ef06aa5 authored Oct 11, 2016 by njoyard
8 changed files
--- a/bin/update_votes
+++ b/bin/update_votes
@@ -7,6 +7,10 @@ parltrack_download_pipe ep_votes.json.xz parltrack_import_votes

 sleep 10

+parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
+
+sleep 10
+
 francedata_download_pipe votes.json.gz francedata_import_votes

 if [ "x$1" != "xnoscores" ]; then

--- a/src/memopol/tests/response_fixtures/RepresentativeListTest.test_csv.content
+++ b/src/memopol/tests/response_fixtures/RepresentativeListTest.test_csv.content
 François Asensi,fasensi@assemblee-nationale.fr,GDR,France [FR]
-Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR]
+Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
 Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
 Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
 Paul BRANNEN,paul.brannen@europarl.europa.eu,SD,United Kingdom [GB]
@@ -9,7 +9,7 @@ Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,Fran
 Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
 Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
 Albert DESS,albert.dess@europarl.europa.eu,EPP,Germany [DE]
-Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
+Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
 Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
 Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
 Daniel Fasquelle,dfasquelle@assemblee-nationale.fr,LR,France [FR]
@@ -27,19 +27,19 @@ Werner LANGEN,werner.langen@europarl.europa.eu,EPP,Germany [DE]
 Jo LEINEN,jo.leinen@europarl.europa.eu,SD,Germany [DE]
 Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
 Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
-Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR]
+Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
 Arne LIETZ,arne.lietz@europarl.europa.eu,SD,Germany [DE]
 Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
-Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
+Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
 Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
 Gesine MEISSNER,gesine.meissner@europarl.europa.eu,ALDE,Germany [DE]
-Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR]
+Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
 Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
 Paul NUTTALL,paul.nuttall@europarl.europa.eu,EFDD,United Kingdom [GB]
 Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]
 Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
 François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
-Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR]
+Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
 Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
 Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
 André Santini,asantini@assemblee-nationale.fr,UDI,France [FR]

--- a/src/memopol/tests/response_fixtures/RepresentativeListTest.test_order_csv.content
+++ b/src/memopol/tests/response_fixtures/RepresentativeListTest.test_order_csv.content
@@ -7,10 +7,10 @@ Verónica LOPE FONTAGNÉ,veronica.lopefontagne@europarl.europa.eu,EPP,Spain [ES]
 Esther de LANGE,esther.delange@europarl.europa.eu,EPP,Netherlands [NL]
 Angelika NIEBLER,angelika.niebler@europarl.europa.eu,EPP,Germany [DE]
 Mirosław PIOTROWSKI,miroslaw.piotrowski@europarl.europa.eu,ECR,Poland [PL]
-Thierry Benoit,"tbenoit@assemblee-nationale.fr, contact@thierry-benoit.fr",UDI,France [FR]
-Pierre Lequiller,"plequiller@assemblee-nationale.fr, pierre.lequiller@gmail.com",LR,France [FR]
+Thierry Benoit,"contact@thierry-benoit.fr, tbenoit@assemblee-nationale.fr",UDI,France [FR]
+Pierre Lequiller,"pierre.lequiller@gmail.com, plequiller@assemblee-nationale.fr",LR,France [FR]
 Claude Goasguen,cgoasguen@assemblee-nationale.fr,LR,France [FR]
-Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
+Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
 Pierre Lellouche,plellouche@assemblee-nationale.fr,LR,France [FR]
 Philippe Vigier,pvigier@assemblee-nationale.fr,UDI,France [FR]
 Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
@@ -18,7 +18,7 @@ Philippe Houillon,phouillon@assemblee-nationale.fr,LR,France [FR]
 Dominique Dord,ddord@assemblee-nationale.fr,LR,France [FR]
 Jean-Michel Couve,"couve.jean-michel@wanadoo.fr, jmcouve@assemblee-nationale.fr",LR,France [FR]
 Pascale Got,pgot@assemblee-nationale.fr,SER,France [FR]
-Alain Moyne-Bressand,"moyne-bressand.alain@wanadoo.fr, amoyne-bressand@assemblee-nationale.fr",LR,France [FR]
+Alain Moyne-Bressand,"amoyne-bressand@assemblee-nationale.fr, moyne-bressand.alain@wanadoo.fr",LR,France [FR]
 Jean-Louis Christ,"christjl@wanadoo.fr, jlchrist@assemblee-nationale.fr",LR,France [FR]
 Olivier Dussopt,odussopt@assemblee-nationale.fr,SER,France [FR]
 Jean-Claude Bouchet,"jcbouchet@assemblee-nationale.fr, jcbouchet@jcbouchet.fr",LR,France [FR]
@@ -28,9 +28,9 @@ Franck Reynier,freynier@assemblee-nationale.fr,UDI,France [FR]
 Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
 François Pupponi,fpupponi@assemblee-nationale.fr,SER,France [FR]
 Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
-Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
+Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
 Annick Lepetit,"alepetit@assemblee-nationale.fr, annick-lepetit@wanadoo.fr",SER,France [FR]
-Jean-Luc Reitzer,"jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr, jeanluc.reitzer@wanadoo.fr",LR,France [FR]
+Jean-Luc Reitzer,"jeanluc.reitzer@wanadoo.fr, jlr.depute@orange.fr, jlreitzer@assemblee-nationale.fr",LR,France [FR]
 Czesław HOC,czeslaw.hoc@europarl.europa.eu,ECR,Poland [PL]
 Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
 Patrick O'FLYNN,patrick.oflynn@europarl.europa.eu,EFDD,United Kingdom [GB]

--- a/src/memopol/tests/response_fixtures/RepresentativeListTest.test_search_csv.content
+++ b/src/memopol/tests/response_fixtures/RepresentativeListTest.test_search_csv.content
 Marcel Bonnot,"marcelbonnot@yahoo.fr, mbonnot@assemblee-nationale.fr",LR,France [FR]
 Udo BULLMANN,udo.bullmann@europarl.europa.eu,SD,Germany [DE]
-Marc Dolez,"mdolez@assemblee-nationale.fr, contact@marc-dolez.org",GDR,France [FR]
+Marc Dolez,"contact@marc-dolez.org, mdolez@assemblee-nationale.fr",GDR,France [FR]
 María Teresa GIMÉNEZ BARBAT,teresa.gimenezbarbat@europarl.europa.eu,ALDE,Ireland [IE]
 Sylvia-Yvonne KAUFMANN,sylvia-yvonne.kaufmann@europarl.europa.eu,SD,Germany [DE]
 Dietmar KÖSTER,dietmar.koester@europarl.europa.eu,SD,Germany [DE]
-Jacqueline Maquet,"j-maquet@orange.fr, jmaquet@assemblee-nationale.fr",SER,France [FR]
+Jacqueline Maquet,"jmaquet@assemblee-nationale.fr, j-maquet@orange.fr",SER,France [FR]
 Philippe Martin,pmartin@assemblee-nationale.fr,SER,France [FR]
 Marcel Rogemont,"marcel.rogemont@wanadoo.fr, mrogemont@assemblee-nationale.fr",SER,France [FR]
 Hermann WINKLER,hermann.winkler@europarl.europa.eu,EPP,Germany [DE]
--- a/src/memopol/views/representative_list.py
+++ b/src/memopol/views/representative_list.py
@@ -3,9 +3,10 @@
 from core.views import GridListMixin, PaginationMixin, CSVDownloadMixin, \
    ActiveLegislatureMixin, SortMixin

+from django.db import models
 from django.views import generic

-from representatives.models import Representative
+from representatives.models import Representative, Email

 from ..filters import RepresentativeFilter
 from .representative_mixin import RepresentativeViewMixin
@@ -72,7 +73,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,

    def get_csv_results(self, context, **kwargs):
        qs = self.get_queryset()
-        qs = qs.prefetch_related('email_set')
+        qs = qs.prefetch_related(
+            models.Prefetch(
+                'email_set',
+                queryset=Email.objects.order_by('email')
+            )
+        )
        return [self.add_representative_country_and_main_mandate(r)
                for r in qs]


--- a/src/representatives_votes/contrib/parltrack/import_votes.py
+++ b/src/representatives_votes/contrib/parltrack/import_votes.py
@@ -28,6 +28,8 @@ def _parse_date(date_str):

 JSON_URL = 'http://parltrack.euwiki.org/dumps/ep_votes.json.xz'
 DESTINATION = join('/tmp', 'ep_votes.json')
+RE_COMVOTE_REF = re.compile(r'&reference=([^&]+)')
+POSITION_MAP = {k: k for k in ('For', 'Against', 'Abstain')}


 class Command(object):
@@ -49,9 +51,14 @@ class Command(object):
        """
        Parse data from parltrack votes db dumps (1 proposal)
        """
-        if 'epref' not in vote_data.keys():
+        keys = vote_data.keys()
+        if 'ep_ref' in keys:
+            vote_data['epref'] = vote_data['ep_ref']
+        elif 'epref' not in keys:
            logger.debug('Could not import data without epref %s',
-                vote_data['title'])
+                vote_data.get('title',
+                              vote_data.get('doc',
+                                            vote_data.get('url', '?'))))
            return

        dossier_pk = self.get_dossier(vote_data['epref'])
@@ -61,17 +68,122 @@ class Command(object):
                         vote_data['epref'])
            return

-        return self.parse_proposal_data(
-            proposal_data=vote_data,
-            dossier_pk=dossier_pk
+        if 'committee' in vote_data:
+            return self.parse_committee_vote_data(
+                proposal_data=vote_data,
+                dossier_pk=dossier_pk
+            )
+        else:
+            return self.parse_proposal_data(
+                proposal_data=vote_data,
+                dossier_pk=dossier_pk
+            )
+
+    def parse_proposal_totals(self, data, position_map=POSITION_MAP):
+        totals = {}
+
+        for position in ('For', 'Abstain', 'Against'):
+            position_data = data.get(position_map[position], {})
+            position_total = position_data.get('total', 0)
+
+            if isinstance(position_total, str) and position_total.isdigit():
+                position_total = int(position_total)
+
+            totals['total_%s' % position.lower()] = position_total
+
+        return totals
+
+    def parse_proposal_votes(self, proposal, data, position_map=POSITION_MAP):
+        logger.info(
+            u'Looking for votes in proposal {}'.format(proposal.title))
+
+        for position in ('For', 'Abstain', 'Against'):
+            for group_vote_data in data.get(position_map[position],
+                                            {}).get('groups', {}):
+                for vote_data in group_vote_data['votes']:
+                    if not isinstance(vote_data, dict):
+                        logger.error('Skipping vote data %s for proposal %s',
+                                     vote_data, data['_id'])
+                        continue
+
+                    representative_pk = self.get_representative(vote_data)
+
+                    if representative_pk is None:
+                        logger.error('Could not find mep for %s', vote_data)
+                        continue
+
+                    changed = False
+                    try:
+                        vote = Vote.objects.get(
+                            representative_id=representative_pk,
+                            proposal_id=proposal.pk)
+                    except Vote.DoesNotExist:
+                        vote = Vote(proposal_id=proposal.pk,
+                                    representative_id=representative_pk)
+                        changed = True
+
+                    if vote.position != position.lower():
+                        changed = True
+                        vote.position = position.lower()
+
+                    if changed:
+                        vote.save()
+                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
+                                     vote.pk, representative_pk,
+                                     proposal.title, proposal.pk, position)
+
+    @transaction.atomic
+    def parse_committee_vote_data(self, proposal_data, dossier_pk):
+        title = u'%s vote on %s' % (proposal_data['committee'],
+                                    proposal_data['doc'])
+        changed = False
+
+        try:
+            proposal = Proposal.objects.get(title=title, dossier_id=dossier_pk)
+        except Proposal.DoesNotExist:
+            proposal = Proposal(title=title, dossier_id=dossier_pk)
+            changed = True
+
+        try:
+            ref = RE_COMVOTE_REF.search(proposal_data['url']).group(1)
+        except:
+            logger.debug(u'Cannot find proposal reference for %s' % title)
+            return
+
+        data_map = dict(
+            datetime=_parse_date(proposal_data['ts']),
+            reference=ref,
+            kind='committee-vote'
        )

+        position_map = {
+            'For': '+',
+            'Against': '-',
+            'Abstain': '0'
+        }
+
+        data_map.update(self.parse_proposal_totals(proposal_data,
+                                                   position_map))
+
+        for key, value in data_map.items():
+            if value != getattr(proposal, key, None):
+                setattr(proposal, key, value)
+                changed = True
+
+        if changed:
+            proposal.save()
+
+        if self.should_skip(proposal_data):
+            logger.debug(
+                u'Skipping votes for dossier %s', proposal_data.get(
+                    'epref', title))
+            return
+
+        self.parse_proposal_votes(proposal, proposal_data, position_map)
+
    @transaction.atomic
    def parse_proposal_data(self, proposal_data, dossier_pk):
        """Get or Create a proposal model from raw data"""
-        proposal_display = '{} ({})'.format(proposal_data['title'].encode(
-            'utf-8'), proposal_data.get('report', '').encode('utf-8'))
-
        if 'issue_type' not in proposal_data.keys():
            logger.debug('This proposal data without issue_type: %s',
                         proposal_data['epref'])
@@ -92,14 +204,7 @@ class Command(object):
            kind=proposal_data.get('issue_type')
        )

-        for position in ('For', 'Abstain', 'Against'):
-            position_data = proposal_data.get(position, {})
-            position_total = position_data.get('total', 0)
-
-            if isinstance(position_total, str) and position_total.isdigit():
-                position_total = int(position_total)
-
-            data_map['total_%s' % position.lower()] = position_total
+        data_map.update(self.parse_proposal_totals(proposal_data))

        for key, value in data_map.items():
            if value != getattr(proposal, key, None):
@@ -111,56 +216,11 @@ class Command(object):

        if self.should_skip(proposal_data):
            logger.debug(
-                'Skipping dossier %s', proposal_data.get(
+                u'Skipping votes for dossier %s', proposal_data.get(
                    'epref', proposal_data['title']))
            return

-        positions = ['For', 'Abstain', 'Against']
-        logger.info(
-            'Looking for votes in proposal {}'.format(proposal_display))
-        for position in positions:
-            for group_vote_data in proposal_data.get(
-                    position,
-                    {}).get(
-                    'groups',
-                    {}):
-                for vote_data in group_vote_data['votes']:
-                    if not isinstance(vote_data, dict):
-                        logger.error('Skipping vote data %s for proposal %s',
-                                     vote_data, proposal_data['_id'])
-                        continue
-
-                    representative_pk = self.get_representative(vote_data)
-
-                    if representative_pk is None:
-                        logger.error('Could not find mep for %s', vote_data)
-                        continue
-
-                    representative_name = vote_data.get('orig', '')
-
-                    changed = False
-                    try:
-                        vote = Vote.objects.get(
-                            representative_id=representative_pk,
-                            proposal_id=proposal.pk)
-                    except Vote.DoesNotExist:
-                        vote = Vote(proposal_id=proposal.pk,
-                            representative_id=representative_pk)
-                        changed = True
-
-                    if vote.position != position.lower():
-                        changed = True
-                        vote.position = position.lower()
-
-                    if vote.representative_name != representative_name:
-                        changed = True
-                        vote.representative_name = representative_name
-
-                    if changed:
-                        vote.save()
-                        logger.debug('Save vote %s for MEP %s on %s #%s to %s',
-                            vote.pk, representative_pk, proposal_data['title'],
-                            proposal.pk, position)
+        self.parse_proposal_votes(proposal, proposal_data)

        return proposal


--- a/src/representatives_votes/contrib/parltrack/tests/votes_expected.json
+++ b/src/representatives_votes/contrib/parltrack/tests/votes_expected.json
 [
+{
+    "fields": {
+        "total_for": 1,
+        "description": "",
+        "reference": "TEST-COM-VOTE",
+        "title": "LIBE vote on Test Committee Vote",
+        "dossier": [
+            "2012/2002(INI)"
+        ],
+        "kind": "committee-vote",
+        "datetime": "2012-10-23T16:31:10Z",
+        "total_against": 1,
+        "total_abstain": 0
+    },
+    "model": "representatives_votes.proposal"
+},
 {
    "fields": {
        "representative_name": "",
@@ -163,5 +179,35 @@
    },
    "model": "representatives_votes.vote",
    "pk": 11
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "for",
+        "proposal": [
+            "LIBE vote on Test Committee Vote",
+            "2012/2002(INI)"
+        ],
+        "representative": [
+            "hubert-pirker-1948-10-03"
+        ]
+    },
+    "model": "representatives_votes.vote",
+    "pk": 12
+},
+{
+    "fields": {
+        "representative_name": "",
+        "position": "against",
+        "proposal": [
+            "LIBE vote on Test Committee Vote",
+            "2012/2002(INI)"
+        ],
+        "representative": [
+            "olle-ludvigsson-1948-10-28"
+        ]
+    },
+    "model": "representatives_votes.vote",
+    "pk": 13
 }
 ]
--- a/src/representatives_votes/contrib/parltrack/tests/votes_fixture.json
+++ b/src/representatives_votes/contrib/parltrack/tests/votes_fixture.json
@@ -281,5 +281,45 @@
        "ts": "2015-10-28T13:01:09",
        "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml",
        "voteid": "59151"
+    },
+    {
+        "url": "http://www.europarl.europa.eu/sides/getDoc.do?type=REPORT&mode=XML&reference=TEST-COM-VOTE&language=EN",
+        "doc": "Test Committee Vote",
+        "+": {
+            "groups": [
+                {
+                    "group": "PPE",
+                    "votes": [
+                        {
+                            "ep_id": 2307,
+                            "name": "Pirker"
+                        }
+                    ]
+                }
+            ],
+            "total": "1"
+        },
+        "ep_ref": "2012/2002(INI)",
+        "-": {
+            "groups": [
+                {
+                    "group": "PPE",
+                    "votes": [
+                        {
+                            "ep_id": 96673,
+                            "name": "Ludvigsson"
+                        }
+                    ]
+                }
+            ],
+            "total": "1"
+        },
+        "ts": "2012-10-23T18:31:10",
+        "0": {
+            "total": 0,
+            "groups": []
+        },
+        "committee": "LIBE",
+        "_id": "56617c15edd52ed712fc7901"
    }
 ]