src/representatives_votes/admin.py

+# coding: utf-8
+
+from django.contrib import admin
+
+import nested_admin
+
+from representatives_recommendations.models import Recommendation
+from representatives_recommendations.forms import RecommendationForm
+
+from .models import Dossier, Document, Proposal, Vote
+
+
+class DocumentAdmin(admin.ModelAdmin):
+    list_display = ('dossier_reference', 'kind', 'title', 'link')
+    search_fields = ('reference', 'dossier__reference', 'title')
+
+    def dossier_reference(self, obj):
+        return obj.dossier.reference
+
+
+class RecommendationStackedInline(nested_admin.NestedStackedInline):
+    model = Recommendation
+    min_num = 1
+    max_num = 1
+    fields = ('title', 'description',
+            'recommendation', 'weight',
+            'id', 'proposal',)
+    form = RecommendationForm
+
+
+class ProposalAdmin(nested_admin.NestedModelAdmin):
+    list_display = (
+        'reference',
+        'dossier_reference',
+        'title',
+        'kind')
+    search_fields = ('reference', 'dossier__reference', 'title')
+    inlines = [
+        RecommendationStackedInline,
+    ]
+
+    def dossier_reference(self, obj):
+        return obj.dossier.reference
+
+
+class ProposalStackedInline(nested_admin.NestedStackedInline):
+    model = Proposal
+    extra = 0
+    can_delete = False
+    min_num = 0
+    readonly_fields = ('title', 'description',)
+    fields = ('title', 'description',)
+    show_change_link = True
+    inlines = [
+        RecommendationStackedInline,
+    ]
+
+
+class DossierAdmin(nested_admin.NestedModelAdmin):
+    list_display = ('id', 'reference', 'title')
+    search_fields = ('reference', 'title')
+    inlines = [
+        ProposalStackedInline,
+    ]
+
+
+class NoneMatchingFilter(admin.SimpleListFilter):
+    title = 'Representative'
+    parameter_name = 'representative'
+
+    def lookups(self, request, model_admin):
+        return [('None', 'Unknown')]
+
+    def queryset(self, request, queryset):
+        if self.value() == 'None':
+            return queryset.filter(representative=None)
+        else:
+            return queryset
+
+
+class VoteAdmin(admin.ModelAdmin):
+    list_display = (
+        'id',
+        'proposal_reference',
+        'position',
+        'representative',
+        'representative_name')
+    list_filter = (NoneMatchingFilter,)
+
+    def proposal_reference(self, obj):
+        return obj.proposal.reference
+
+
+admin.site.register(Dossier, DossierAdmin)
+admin.site.register(Document, DocumentAdmin)
+admin.site.register(Proposal, ProposalAdmin)
+admin.site.register(Vote, VoteAdmin)

src/representatives_votes/api.py

+from django.db import models
+
+from .models import (
+    Document,
+    Dossier,
+    Proposal,
+    Vote
+)
+
+from rest_framework import (
+    filters,
+    viewsets,
+)
+
+from representatives.api import DefaultWebPagination
+
+from representatives_votes.serializers import (
+    DossierDetailSerializer,
+    DossierSerializer,
+    ProposalDetailSerializer,
+    ProposalSerializer,
+    VoteSerializer,
+)
+
+
+class DossierViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows dossiers to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Dossier.objects.order_by('id')
+    serializer_class = DossierSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'title': ['exact', 'icontains'],
+        'reference': ['exact', 'icontains'],
+    }
+
+    search_fields = (
+        'title',
+        'reference',
+        'text',
+        'proposals__title'
+    )
+
+    def retrieve(self, request, pk=None):
+        self.serializer_class = DossierDetailSerializer
+        self.queryset = self.queryset.prefetch_related(
+            models.Prefetch(
+                'proposals',
+                queryset=Proposal.objects.order_by('id')
+            ),
+            models.Prefetch(
+                'documents',
+                queryset=Document.objects.order_by('id')
+            )
+        )
+        return super(DossierViewSet, self).retrieve(request, pk)
+
+
+class ProposalViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows proposals to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Proposal.objects.order_by('id')
+    serializer_class = ProposalSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'title': ['exact', 'icontains'],
+        'description': ['icontains'],
+        'reference': ['exact', 'icontains'],
+        'datetime': ['exact', 'gte', 'lte'],
+        'kind': ['exact'],
+    }
+
+    search_fields = (
+        'title',
+        'reference',
+        'dossier__title',
+        'dossier__reference'
+    )
+
+    def retrieve(self, request, pk=None):
+        self.serializer_class = ProposalDetailSerializer
+        self.queryset = self.queryset.prefetch_related(
+            models.Prefetch(
+                'votes',
+                queryset=Vote.objects.order_by('representative_id')
+            )
+        )
+        return super(ProposalViewSet, self).retrieve(request, pk)
+
+
+class VoteViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint that allows proposals to be viewed.
+    """
+
+    pagination_class = DefaultWebPagination
+    queryset = Vote.objects.select_related('representative', 'proposal') \
+                           .order_by('proposal_id', 'representative__slug')
+    serializer_class = VoteSerializer
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter
+    )
+
+    filter_fields = {
+        'position': ['exact'],
+        'representative_name': ['exact', 'icontains'],
+        'representative': ['exact']
+    }

src/representatives_votes/contrib/francedata/import_dossiers.py

+# coding: utf-8
+
+import sys
+import ijson
+import logging
+import re
+
+import django
+from django.apps import apps
+from django.db import transaction
+
+from representatives.contrib.francedata.import_representatives import \
+    ensure_chambers
+from representatives.models import Chamber
+from representatives_votes.models import Document, Dossier
+
+logger = logging.getLogger(__name__)
+
+
+def extract_reference(url):
+    m = re.search(r'/dossier-legislatif/([^./]+)\.html', url)
+    if m:
+        return m.group(1)
+
+    m = re.search(r'/(\d+)/dossiers/([^./]+)\.asp', url)
+    if m:
+        return '%s/%s' % (m.group(1), m.group(2))
+
+    m = re.search(r'/dossiers/([^./]+)\.asp', url)
+    if m:
+        return m.group(1)
+
+    return None
+
+
+def find_dossier(data):
+    '''
+    Find dossier with reference matching either 'ref_an' or 'ref_sen',
+    create it if not found.  Ensure its reference is 'ref_an' if both fields
+    are present.
+    '''
+
+    changed = False
+    dossier = None
+    reffield = None
+
+    for field in [k for k in ('ref_an', 'ref_sen') if k in data]:
+        try:
+            dossier = Dossier.objects.get(reference=data[field])
+            reffield = field
+            break
+        except Dossier.DoesNotExist:
+            pass
+
+    if dossier is None:
+        reffield = 'ref_an' if 'ref_an' in data else 'ref_sen'
+        dossier = Dossier(reference=data[reffield])
+        logger.debug('Created dossier %s' % data[reffield])
+        changed = True
+
+    if 'ref_an' in data and reffield != 'ref_an':
+        logger.debug('Changed dossier reference to %s' % data['ref_an'])
+        dossier.reference = data['ref_an']
+        changed = True
+
+    return dossier, changed
+
+
+def handle_document(dossier, chamber, url):
+    doc_changed = False
+    try:
+        doc = Document.objects.get(chamber=chamber, dossier=dossier,
+                                   kind='procedure-file')
+    except Document.DoesNotExist:
+        doc = Document(chamber=chamber, dossier=dossier, kind='procedure-file')
+        logger.debug('Created %s document for dossier %s' %
+            (chamber.abbreviation, dossier.title))
+        doc_changed = True
+
+    if doc.link != url:
+        logger.debug('Changing %s url from %s to %s' %
+            (chamber.abbreviation, doc.link, url))
+        doc.link = url
+        doc_changed = True
+
+    if doc_changed:
+        doc.save()
+
+
+def parse_dossier_data(data, an, sen):
+    if 'url_an' in data:
+        ref_an = extract_reference(data['url_an'])
+        if ref_an is None:
+            logger.warn('No reference for dossier %s' % data['url_an'])
+            return
+        else:
+            data['ref_an'] = ref_an
+
+    if 'url_sen' in data:
+        ref_sen = extract_reference(data['url_sen'])
+        if ref_sen is None:
+            logger.warn('No reference for dossier %s' % data['url_sen'])
+            return
+        else:
+            data['ref_sen'] = ref_sen
+
+    dossier, changed = find_dossier(data)
+
+    thisref = data['ref_an' if data['chambre'] == 'AN' else 'ref_sen']
+
+    title = data['titre']
+    if dossier.reference == thisref and dossier.title != title:
+        logger.debug('Changed dossier title to %s' % title)
+        dossier.title = title
+        changed = True
+
+    with transaction.atomic():
+        if changed:
+            logger.debug('Saved dossier %s' % dossier.reference)
+            dossier.save()
+
+        if 'url_an' in data:
+            handle_document(dossier, an, data['url_an'])
+
+        if 'url_sen' in data:
+            handle_document(dossier, sen, data['url_sen'])
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    ensure_chambers()
+    an = Chamber.objects.get(abbreviation='AN')
+    sen = Chamber.objects.get(abbreviation='SEN')
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        try:
+            parse_dossier_data(data, an, sen)
+        except Exception:
+            logger.exception('error trying to import dossier %s', str(data))

src/representatives_votes/contrib/francedata/import_scrutins.py

+# coding: utf-8
+
+from datetime import datetime
+import ijson
+import logging
+from pytz import timezone as date_timezone
+import sys
+
+import django
+from django.apps import apps
+from django.utils.timezone import make_aware as date_make_aware
+
+from representatives_votes.models import Dossier, Proposal
+
+logger = logging.getLogger(__name__)
+
+
+def _parse_date(date_str):
+    return date_make_aware(
+        datetime.strptime(date_str, "%Y-%m-%d"),
+        date_timezone('Europe/Paris')
+    )
+
+
+def _get_unique_title(proposal_pk, candidate):
+    title = candidate
+
+    try:
+        exists = Proposal.objects.get(title=title)
+    except Proposal.DoesNotExist:
+        exists = None
+
+    if exists and exists.pk != proposal_pk:
+        num = 1
+        while exists and exists.pk != proposal_pk:
+            title = '%s (%d)' % (candidate, num)
+
+            try:
+                exists = Proposal.objects.get(title=title)
+            except Proposal.DoesNotExist:
+                exists = None
+
+            num = num + 1
+
+        logger.debug('Made unique title %s' % title)
+
+    return title
+
+
+class ScrutinImporter:
+    dossiers = {}
+
+    def get_dossier(self, url):
+        if url not in self.dossiers:
+            try:
+                self.dossiers[url] = Dossier.objects.get(documents__link=url)
+            except Dossier.DoesNotExist:
+                return None
+
+        return self.dossiers[url]
+
+    def parse_scrutin_data(self, data):
+        ref = data['url']
+
+        if 'dossier_url' not in data:
+            logger.debug('Cannot create proposal without dossier')
+            return
+
+        dossier = self.get_dossier(data['dossier_url'])
+        if dossier is None:
+            logger.debug('Cannot create proposal for unknown dossier %s'
+                         % data['dossier_url'])
+            return
+
+        changed = False
+        try:
+            proposal = Proposal.objects.get(reference=ref)
+        except Proposal.DoesNotExist:
+            proposal = Proposal(reference=ref, total_for=0, total_against=0,
+                                total_abstain=0)
+            logger.debug('Created proposal %s' % ref)
+            changed = True
+
+        values = dict(
+            title=_get_unique_title(proposal.pk, data["objet"]),
+            datetime=_parse_date(data["date"]),
+            dossier_id=dossier.pk,
+            kind='dossier'
+        )
+
+        for key, value in values.items():
+            if value != getattr(proposal, key, None):
+                logger.debug('Changed proposal %s to %s' % (key, value))
+                setattr(proposal, key, value)
+                changed = True
+
+        if changed:
+            logger.debug('Updated proposal %s' % ref)
+            proposal.save()
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    importer = ScrutinImporter()
+
+    for data in ijson.items(stream or sys.stdin, 'item'):
+        try:
+            importer.parse_scrutin_data(data)
+        except Exception:
+            logger.exception('error trying to import scrutin %s', str(data))