Merge django-representatives-votes

Note: it **must** be kept loosely coupled with other modules in this
repository. It was merged here just to remove the burden of having
several repositories to manage: volunteer time is precious.

.coveragerc

@@ -6,3 +6,5 @@ omit =
 	src/representatives/migrations/*
 	src/representatives/contrib/francedata/tests/*
 	src/representatives/contrib/parltrack/tests/*
+    src/representatives_votes/tests/*
+    src/representatives_votes/migrations/*

.travis.yml

 sudo: false
+<<<<<<< HEAD
 env:
   global:
   - DJANGO_DEBUG=True

src/representatives_votes/admin.py

+# coding: utf-8
+
+from django.contrib import admin
+
+from .models import Dossier, Document, Proposal, Vote
+
+
+class DossierAdmin(admin.ModelAdmin):
+    list_display = ('id', 'reference', 'title')
+    search_fields = ('reference', 'title')
+
+
+class DocumentAdmin(admin.ModelAdmin):
+    list_display = ('dossier_reference', 'kind', 'title', 'link')
+    search_fields = ('reference', 'dossier__reference', 'title')
+
+    def dossier_reference(self, obj):
+        return obj.dossier.reference
+
+
+class ProposalAdmin(admin.ModelAdmin):
+    list_display = (
+        'reference',
+        'dossier_reference',
+        'title',
+        'kind')
+    search_fields = ('reference', 'dossier__reference', 'title')
+
+    def dossier_reference(self, obj):
+        return obj.dossier.reference
+
+
+class NoneMatchingFilter(admin.SimpleListFilter):
+    title = 'Representative'
+    parameter_name = 'representative'
+
+    def lookups(self, request, model_admin):
+        return [('None', 'Unknown')]
+
+    def queryset(self, request, queryset):
+        if self.value() == 'None':
+            return queryset.filter(representative=None)
+        else:
+            return queryset
+
+
+class VoteAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'proposal_reference',
+        'position',
+        'representative',
+        'representative_name')
+    list_filter = (NoneMatchingFilter,)
+
+    def proposal_reference(self, obj):
+        return obj.proposal.reference
+
+admin.site.register(Dossier, DossierAdmin)
+admin.site.register(Document, DocumentAdmin)
+admin.site.register(Proposal, ProposalAdmin)
+admin.site.register(Vote, VoteAdmin)

src/representatives_votes/api.py

+from .models import (
+    Dossier,
+    Proposal,
+    Vote
+)
+
+from rest_framework import (
+    filters,
+    viewsets,
+)
+
+from representatives.api import DefaultWebPagination
+
+from representatives_votes.serializers import (
+    DossierDetailSerializer,
+    DossierSerializer,
+    ProposalDetailSerializer,
+    ProposalSerializer,
+    VoteSerializer,
+)
+
+
+class DossierViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows dossiers to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Dossier.objects.all()
+    serializer_class = DossierSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'title': ['exact', 'icontains'],
+        'reference': ['exact', 'icontains'],
+    }
+
+    search_fields = (
+        'title',
+        'reference',
+        'text',
+        'proposals__title'
+    )
+
+    ordering_fields = ['reference']
+
+    def retrieve(self, request, pk=None):
+        self.serializer_class = DossierDetailSerializer
+        self.queryset = self.queryset.prefetch_related('proposals',
+                                                       'documents')
+        return super(DossierViewSet, self).retrieve(request, pk)
+
+
+class ProposalViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows proposals to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Proposal.objects.all()
+    serializer_class = ProposalSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'title': ['exact', 'icontains'],
+        'description': ['icontains'],
+        'reference': ['exact', 'icontains'],
+        'datetime': ['exact', 'gte', 'lte'],
+        'kind': ['exact'],
+    }
+
+    search_fields = (
+        'title',
+        'reference',
+        'dossier__title',
+        'dossier__reference'
+    )
+
+    ordering_fields = ['reference']
+
+    def retrieve(self, request, pk=None):
+        self.serializer_class = ProposalDetailSerializer
+        return super(ProposalViewSet, self).retrieve(request, pk)
+
+
+class VoteViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows proposals to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Vote.objects.select_related('representative', 'proposal')
+    serializer_class = VoteSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'position': ['exact'],
+        'representative_name': ['exact', 'icontains'],
+        'representative': ['exact']
+    }

src/representatives_votes/contrib/francedata/import_dossiers.py

+# coding: utf-8
+
+import sys
+import ijson
+import logging
+import re
+
+import django
+from django.apps import apps
+from django.db import transaction
+
+from representatives.contrib.francedata.import_representatives import \
+    ensure_chambers
+from representatives.models import Chamber
+from representatives_votes.models import Document, Dossier
+
+logger = logging.getLogger(__name__)
+
+
+def extract_reference(url):
+    m = re.search(r'/dossier-legislatif/([^./]+)\.html', url)
+    if m:
+        return m.group(1)
+
+    m = re.search(r'/(\d+)/dossiers/([^./]+)\.asp', url)
+    if m:
+        return '%s/%s' % (m.group(1), m.group(2))
+
+    m = re.search(r'/dossiers/([^./]+)\.asp', url)
+    if m:
+        return m.group(1)
+
+    return None
+
+
+def find_dossier(data):
+    '''
+    Find dossier with reference matching either 'ref_an' or 'ref_sen',
+    create it if not found.  Ensure its reference is 'ref_an' if both fields
+    are present.
+    '''
+
+    changed = False
+    dossier = None
+    reffield = None
+
+    for field in [k for k in ('ref_an', 'ref_sen') if k in data]:
+        try:
+            dossier = Dossier.objects.get(reference=data[field])
+            reffield = field
+            break
+        except Dossier.DoesNotExist:
+            pass
+
+    if dossier is None:
+        reffield = 'ref_an' if 'ref_an' in data else 'ref_sen'
+        dossier = Dossier(reference=data[reffield])
+        logger.debug('Created dossier %s' % data[reffield])
+        changed = True
+
+    if 'ref_an' in data and reffield != 'ref_an':
+        logger.debug('Changed dossier reference to %s' % data['ref_an'])
+        dossier.reference = data['ref_an']
+        changed = True
+
+    return dossier, changed
+
+
+def handle_document(dossier, chamber, url):
+    doc_changed = False
+    try:
+        doc = Document.objects.get(chamber=chamber, dossier=dossier,
+                                   kind='procedure-file')
+    except Document.DoesNotExist:
+        doc = Document(chamber=chamber, dossier=dossier, kind='procedure-file')
+        logger.debug('Created %s document for dossier %s' %
+            (chamber.abbreviation, dossier.title))
+        doc_changed = True
+
+    if doc.link != url:
+        logger.debug('Changing %s url from %s to %s' %
+            (chamber.abbreviation, doc.link, url))
+        doc.link = url
+        doc_changed = True
+
+    if doc_changed:
+        doc.save()
+
+
+def parse_dossier_data(data, an, sen):
+    if 'url_an' in data:
+        ref_an = extract_reference(data['url_an'])
+        if ref_an is None:
+            logger.warn('No reference for dossier %s' % data['url_an'])
+            return
+        else:
+            data['ref_an'] = ref_an
+
+    if 'url_sen' in data:
+        ref_sen = extract_reference(data['url_sen'])
+        if ref_sen is None:
+            logger.warn('No reference for dossier %s' % data['url_sen'])
+            return
+        else:
+            data['ref_sen'] = ref_sen
+
+    dossier, changed = find_dossier(data)
+
+    thisref = data['ref_an' if data['chambre'] == 'AN' else 'ref_sen']
+
+    title = data['titre']
+    if dossier.reference == thisref and dossier.title != title:
+        logger.debug('Changed dossier title to %s' % title)
+        dossier.title = title
+        changed = True
+
+    with transaction.atomic():
+        if changed:
+            logger.debug('Saved dossier %s' % dossier.reference)
+            dossier.save()
+
+        if 'url_an' in data:
+            handle_document(dossier, an, data['url_an'])
+
+        if 'url_sen' in data:
+            handle_document(dossier, sen, data['url_sen'])
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    ensure_chambers()
+    an = Chamber.objects.get(abbreviation='AN')
+    sen = Chamber.objects.get(abbreviation='SEN')
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        parse_dossier_data(data, an, sen)

src/representatives_votes/contrib/francedata/import_scrutins.py

+# coding: utf-8
+
+from datetime import datetime
+import ijson
+import logging
+from pytz import timezone as date_timezone
+import sys
+
+import django
+from django.apps import apps
+from django.utils.timezone import make_aware as date_make_aware
+
+from representatives_votes.models import Dossier, Proposal
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_date(date_str):
+    return date_make_aware(
+        datetime.strptime(date_str, "%Y-%m-%d"),
+        date_timezone('Europe/Paris')
+    )
+
+
+def _get_unique_title(proposal_pk, candidate):
+    title = candidate
+
+    try:
+        exists = Proposal.objects.get(title=title)
+    except Proposal.DoesNotExist:
+        exists = None
+
+    if exists and exists.pk != proposal_pk:
+        num = 1
+        while exists and exists.pk != proposal_pk:
+            title = '%s (%d)' % (candidate, num)
+
+            try:
+                exists = Proposal.objects.get(title=title)
+            except Proposal.DoesNotExist:
+                exists = None
+
+            num = num + 1
+
+        logger.debug('Made unique title %s' % title)
+
+    return title
+
+
+class ScrutinImporter:
+    dossiers = {}
+
+    def get_dossier(self, url):
+        if url not in self.dossiers:
+            try:
+                self.dossiers[url] = Dossier.objects.get(documents__link=url)
+            except Dossier.DoesNotExist:
+                return None
+
+        return self.dossiers[url]
+
+    def parse_scrutin_data(self, data):
+        ref = data['url']
+
+        if 'dossier_url' not in data:
+            logger.debug('Cannot create proposal without dossier')
+            return
+
+        dossier = self.get_dossier(data['dossier_url'])
+        if dossier is None:
+            logger.debug('Cannot create proposal for unknown dossier %s'
+                         % data['dossier_url'])
+            return
+
+        changed = False
+        try:
+            proposal = Proposal.objects.get(reference=ref)
+        except Proposal.DoesNotExist:
+            proposal = Proposal(reference=ref, total_for=0, total_against=0,
+                                total_abstain=0)
+            logger.debug('Created proposal %s' % ref)
+            changed = True
+
+        values = dict(
+            title=_get_unique_title(proposal.pk, data["objet"]),
+            datetime=_parse_date(data["date"]),
+            dossier_id=dossier.pk,
+            kind='dossier'
+        )
+
+        for key, value in values.items():
+            if value != getattr(proposal, key, None):
+                logger.debug('Changed proposal %s to %s' % (key, value))
+                setattr(proposal, key, value)
+                changed = True
+
+        if changed:
+            logger.debug('Updated proposal %s' % ref)
+            proposal.save()
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    importer = ScrutinImporter()
+
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        importer.parse_scrutin_data(data)

src/representatives_votes/contrib/francedata/import_votes.py

+# coding: utf-8
+
+import ijson
+import logging
+import sys
+
+import django
+from django.apps import apps
+from django.utils.text import slugify
+
+from representatives_votes.models import Proposal, Representative, Vote
+
+logger = logging.getLogger(__name__)
+
+
+class VotesImporter:
+    deputes_slug = None
+    deputes_rid = None
+    scrutins = None
+    touched = []
+
+    positions = dict(
+        pour="for",
+        contre="against",
+        abstention="abstain"
+    )
+
+    def get_depute_by_name(self, prenom, nom):
+        if self.deputes_slug is None:
+            self.deputes_slug = {
+                slugify(r[0]): r[1] for r in
+                Representative.objects.values_list('full_name', 'pk')
+            }
+
+        full = (u'%s %s' % (prenom, nom)).replace(u' ', ' ')
+        return self.deputes_slug.get(slugify(full), None)
+
+    def get_depute_by_url(self, url):
+        if self.deputes_rid is None:
+            self.deputes_rid = {
+                r[0]: r[1] for r in
+                Representative.objects.prefetch_related('website_set')
+                .filter(website__kind__in=['AN', 'SEN'])
+                .values_list('website__url', 'pk')
+            }
+
+        return self.deputes_rid.get(url, None)
+
+    def get_scrutin(self, ref):
+        if self.scrutins is None:
+            self.scrutins = {
+                s[0]: s[1] for s in Proposal.objects.values_list('reference',
+                                                                 'pk')
+            }
+
+        return self.scrutins.get(ref, None)
+
+    def parse_vote_data(self, data):
+        scrutin = self.get_scrutin(data['scrutin_url'])
+        if scrutin is None:
+            logger.debug('Cannot import vote for unknown scrutin %s'
+                         % data['scrutin_url'])
+            return
+
+        if 'parl_url' in data:
+            repdesc = data['parl_url']
+            depute = self.get_depute_by_url(data['parl_url'])
+        else:
+            repdesc = '%s %s' % (data['prenom'], data['nom'])
+            depute = self.get_depute_by_name(data['prenom'], data['nom'])
+
+        if depute is None:
+            logger.debug('Cannot import vote by unknown rep %s' % repdesc)
+            return
+
+        if not data['division'].lower() in self.positions:
+            logger.debug('Cannot import vote for invalid position %s'
+                         % data['division'])
+            return
+        position = self.positions[data['division'].lower()]
+
+        changed = False
+        try:
+            vote = Vote.objects.get(representative_id=depute,
+                                    proposal_id=scrutin)
+        except Vote.DoesNotExist:
+            vote = Vote(representative_id=depute, proposal_id=scrutin)
+            logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
+            changed = True
+
+        if vote.position != position:
+            logger.debug('Changed vote position to %s' % position)
+            changed = True
+            vote.position = position
+
+        if changed:
+            logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
+            self.touched.append(scrutin)
+            vote.save()
+
+    def update_totals(self):
+        proposals = [Proposal.objects.get(pk=pk) for pk in self.touched]
+
+        for proposal in proposals:
+            changed = False
+
+            for pos in self.positions.values():
+                count = Vote.objects.filter(proposal_id=proposal.pk,
+                                            position=pos).count()
+
+                if getattr(proposal, 'total_%s' % pos, None) != count:
+                    logger.debug('Changed %s count for proposal %s to %s' % (
+                        pos, proposal.pk, count))
+                    setattr(proposal, 'total_%s' % pos, count)
+                    changed = True
+
+            if changed:
+                logger.debug('Updated proposal %s' % proposal.pk)
+                proposal.save()
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    importer = VotesImporter()
+
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        importer.parse_vote_data(data)
+
+    importer.update_totals()

src/representatives_votes/contrib/francedata/tests/dossiers_expected.json

+[
+{
+    "fields" : {
+        "abbreviation": "AN",
+        "country": 1095,
+        "name": "Assembl\u00e9e nationale"
+    },
+    "model": "representatives.chamber",
+    "pk": 2
+},
+{
+    "fields": {
+        "abbreviation": "SEN",