Commit d92d62d9 authored by Jamesie Pic's avatar Jamesie Pic

Merge django-representatives-votes

Note: it **must** be kept loosely coupled with other modules in this
repository. It was merged here just to remove the burden of having
several repositories to manage: volunteer time is precious.
parents 7a387ae8 fb88c4fe
......@@ -6,3 +6,5 @@ omit =
src/representatives/migrations/*
src/representatives/contrib/francedata/tests/*
src/representatives/contrib/parltrack/tests/*
src/representatives_votes/tests/*
src/representatives_votes/migrations/*
sudo: false
<<<<<<< HEAD
env:
global:
- DJANGO_DEBUG=True
......
# coding: utf-8
from django.contrib import admin
from .models import Dossier, Document, Proposal, Vote
class DossierAdmin(admin.ModelAdmin):
list_display = ('id', 'reference', 'title')
search_fields = ('reference', 'title')
class DocumentAdmin(admin.ModelAdmin):
list_display = ('dossier_reference', 'kind', 'title', 'link')
search_fields = ('reference', 'dossier__reference', 'title')
def dossier_reference(self, obj):
return obj.dossier.reference
class ProposalAdmin(admin.ModelAdmin):
list_display = (
'reference',
'dossier_reference',
'title',
'kind')
search_fields = ('reference', 'dossier__reference', 'title')
def dossier_reference(self, obj):
return obj.dossier.reference
class NoneMatchingFilter(admin.SimpleListFilter):
title = 'Representative'
parameter_name = 'representative'
def lookups(self, request, model_admin):
return [('None', 'Unknown')]
def queryset(self, request, queryset):
if self.value() == 'None':
return queryset.filter(representative=None)
else:
return queryset
class VoteAdmin(admin.ModelAdmin):
list_display = (
'id',
'proposal_reference',
'position',
'representative',
'representative_name')
list_filter = (NoneMatchingFilter,)
def proposal_reference(self, obj):
return obj.proposal.reference
admin.site.register(Dossier, DossierAdmin)
admin.site.register(Document, DocumentAdmin)
admin.site.register(Proposal, ProposalAdmin)
admin.site.register(Vote, VoteAdmin)
from .models import (
Dossier,
Proposal,
Vote
)
from rest_framework import (
filters,
viewsets,
)
from representatives.api import DefaultWebPagination
from representatives_votes.serializers import (
DossierDetailSerializer,
DossierSerializer,
ProposalDetailSerializer,
ProposalSerializer,
VoteSerializer,
)
class DossierViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows dossiers to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Dossier.objects.all()
serializer_class = DossierSerializer
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter
)
filter_fields = {
'title': ['exact', 'icontains'],
'reference': ['exact', 'icontains'],
}
search_fields = (
'title',
'reference',
'text',
'proposals__title'
)
ordering_fields = ['reference']
def retrieve(self, request, pk=None):
self.serializer_class = DossierDetailSerializer
self.queryset = self.queryset.prefetch_related('proposals',
'documents')
return super(DossierViewSet, self).retrieve(request, pk)
class ProposalViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows proposals to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Proposal.objects.all()
serializer_class = ProposalSerializer
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter
)
filter_fields = {
'title': ['exact', 'icontains'],
'description': ['icontains'],
'reference': ['exact', 'icontains'],
'datetime': ['exact', 'gte', 'lte'],
'kind': ['exact'],
}
search_fields = (
'title',
'reference',
'dossier__title',
'dossier__reference'
)
ordering_fields = ['reference']
def retrieve(self, request, pk=None):
self.serializer_class = ProposalDetailSerializer
return super(ProposalViewSet, self).retrieve(request, pk)
class VoteViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows proposals to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Vote.objects.select_related('representative', 'proposal')
serializer_class = VoteSerializer
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter
)
filter_fields = {
'position': ['exact'],
'representative_name': ['exact', 'icontains'],
'representative': ['exact']
}
# coding: utf-8
import sys
import ijson
import logging
import re
import django
from django.apps import apps
from django.db import transaction
from representatives.contrib.francedata.import_representatives import \
ensure_chambers
from representatives.models import Chamber
from representatives_votes.models import Document, Dossier
logger = logging.getLogger(__name__)
def extract_reference(url):
m = re.search(r'/dossier-legislatif/([^./]+)\.html', url)
if m:
return m.group(1)
m = re.search(r'/(\d+)/dossiers/([^./]+)\.asp', url)
if m:
return '%s/%s' % (m.group(1), m.group(2))
m = re.search(r'/dossiers/([^./]+)\.asp', url)
if m:
return m.group(1)
return None
def find_dossier(data):
'''
Find dossier with reference matching either 'ref_an' or 'ref_sen',
create it if not found. Ensure its reference is 'ref_an' if both fields
are present.
'''
changed = False
dossier = None
reffield = None
for field in [k for k in ('ref_an', 'ref_sen') if k in data]:
try:
dossier = Dossier.objects.get(reference=data[field])
reffield = field
break
except Dossier.DoesNotExist:
pass
if dossier is None:
reffield = 'ref_an' if 'ref_an' in data else 'ref_sen'
dossier = Dossier(reference=data[reffield])
logger.debug('Created dossier %s' % data[reffield])
changed = True
if 'ref_an' in data and reffield != 'ref_an':
logger.debug('Changed dossier reference to %s' % data['ref_an'])
dossier.reference = data['ref_an']
changed = True
return dossier, changed
def handle_document(dossier, chamber, url):
doc_changed = False
try:
doc = Document.objects.get(chamber=chamber, dossier=dossier,
kind='procedure-file')
except Document.DoesNotExist:
doc = Document(chamber=chamber, dossier=dossier, kind='procedure-file')
logger.debug('Created %s document for dossier %s' %
(chamber.abbreviation, dossier.title))
doc_changed = True
if doc.link != url:
logger.debug('Changing %s url from %s to %s' %
(chamber.abbreviation, doc.link, url))
doc.link = url
doc_changed = True
if doc_changed:
doc.save()
def parse_dossier_data(data, an, sen):
if 'url_an' in data:
ref_an = extract_reference(data['url_an'])
if ref_an is None:
logger.warn('No reference for dossier %s' % data['url_an'])
return
else:
data['ref_an'] = ref_an
if 'url_sen' in data:
ref_sen = extract_reference(data['url_sen'])
if ref_sen is None:
logger.warn('No reference for dossier %s' % data['url_sen'])
return
else:
data['ref_sen'] = ref_sen
dossier, changed = find_dossier(data)
thisref = data['ref_an' if data['chambre'] == 'AN' else 'ref_sen']
title = data['titre']
if dossier.reference == thisref and dossier.title != title:
logger.debug('Changed dossier title to %s' % title)
dossier.title = title
changed = True
with transaction.atomic():
if changed:
logger.debug('Saved dossier %s' % dossier.reference)
dossier.save()
if 'url_an' in data:
handle_document(dossier, an, data['url_an'])
if 'url_sen' in data:
handle_document(dossier, sen, data['url_sen'])
def main(stream=None):
if not apps.ready:
django.setup()
ensure_chambers()
an = Chamber.objects.get(abbreviation='AN')
sen = Chamber.objects.get(abbreviation='SEN')
for data in ijson.items(stream or sys.stdin, 'item'):
parse_dossier_data(data, an, sen)
# coding: utf-8
from datetime import datetime
import ijson
import logging
from pytz import timezone as date_timezone
import sys
import django
from django.apps import apps
from django.utils.timezone import make_aware as date_make_aware
from representatives_votes.models import Dossier, Proposal
logger = logging.getLogger(__name__)
def _parse_date(date_str):
return date_make_aware(
datetime.strptime(date_str, "%Y-%m-%d"),
date_timezone('Europe/Paris')
)
def _get_unique_title(proposal_pk, candidate):
title = candidate
try:
exists = Proposal.objects.get(title=title)
except Proposal.DoesNotExist:
exists = None
if exists and exists.pk != proposal_pk:
num = 1
while exists and exists.pk != proposal_pk:
title = '%s (%d)' % (candidate, num)
try:
exists = Proposal.objects.get(title=title)
except Proposal.DoesNotExist:
exists = None
num = num + 1
logger.debug('Made unique title %s' % title)
return title
class ScrutinImporter:
dossiers = {}
def get_dossier(self, url):
if url not in self.dossiers:
try:
self.dossiers[url] = Dossier.objects.get(documents__link=url)
except Dossier.DoesNotExist:
return None
return self.dossiers[url]
def parse_scrutin_data(self, data):
ref = data['url']
if 'dossier_url' not in data:
logger.debug('Cannot create proposal without dossier')
return
dossier = self.get_dossier(data['dossier_url'])
if dossier is None:
logger.debug('Cannot create proposal for unknown dossier %s'
% data['dossier_url'])
return
changed = False
try:
proposal = Proposal.objects.get(reference=ref)
except Proposal.DoesNotExist:
proposal = Proposal(reference=ref, total_for=0, total_against=0,
total_abstain=0)
logger.debug('Created proposal %s' % ref)
changed = True
values = dict(
title=_get_unique_title(proposal.pk, data["objet"]),
datetime=_parse_date(data["date"]),
dossier_id=dossier.pk,
kind='dossier'
)
for key, value in values.items():
if value != getattr(proposal, key, None):
logger.debug('Changed proposal %s to %s' % (key, value))
setattr(proposal, key, value)
changed = True
if changed:
logger.debug('Updated proposal %s' % ref)
proposal.save()
def main(stream=None):
if not apps.ready:
django.setup()
importer = ScrutinImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
importer.parse_scrutin_data(data)
# coding: utf-8
import ijson
import logging
import sys
import django
from django.apps import apps
from django.utils.text import slugify
from representatives_votes.models import Proposal, Representative, Vote
logger = logging.getLogger(__name__)
class VotesImporter:
deputes_slug = None
deputes_rid = None
scrutins = None
touched = []
positions = dict(
pour="for",
contre="against",
abstention="abstain"
)
def get_depute_by_name(self, prenom, nom):
if self.deputes_slug is None:
self.deputes_slug = {
slugify(r[0]): r[1] for r in
Representative.objects.values_list('full_name', 'pk')
}
full = (u'%s %s' % (prenom, nom)).replace(u' ', ' ')
return self.deputes_slug.get(slugify(full), None)
def get_depute_by_url(self, url):
if self.deputes_rid is None:
self.deputes_rid = {
r[0]: r[1] for r in
Representative.objects.prefetch_related('website_set')
.filter(website__kind__in=['AN', 'SEN'])
.values_list('website__url', 'pk')
}
return self.deputes_rid.get(url, None)
def get_scrutin(self, ref):
if self.scrutins is None:
self.scrutins = {
s[0]: s[1] for s in Proposal.objects.values_list('reference',
'pk')
}
return self.scrutins.get(ref, None)
def parse_vote_data(self, data):
scrutin = self.get_scrutin(data['scrutin_url'])
if scrutin is None:
logger.debug('Cannot import vote for unknown scrutin %s'
% data['scrutin_url'])
return
if 'parl_url' in data:
repdesc = data['parl_url']
depute = self.get_depute_by_url(data['parl_url'])
else:
repdesc = '%s %s' % (data['prenom'], data['nom'])
depute = self.get_depute_by_name(data['prenom'], data['nom'])
if depute is None:
logger.debug('Cannot import vote by unknown rep %s' % repdesc)
return
if not data['division'].lower() in self.positions:
logger.debug('Cannot import vote for invalid position %s'
% data['division'])
return
position = self.positions[data['division'].lower()]
changed = False
try:
vote = Vote.objects.get(representative_id=depute,
proposal_id=scrutin)
except Vote.DoesNotExist:
vote = Vote(representative_id=depute, proposal_id=scrutin)
logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
changed = True
if vote.position != position:
logger.debug('Changed vote position to %s' % position)
changed = True
vote.position = position
if changed:
logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
self.touched.append(scrutin)
vote.save()
def update_totals(self):
proposals = [Proposal.objects.get(pk=pk) for pk in self.touched]
for proposal in proposals:
changed = False
for pos in self.positions.values():
count = Vote.objects.filter(proposal_id=proposal.pk,
position=pos).count()
if getattr(proposal, 'total_%s' % pos, None) != count:
logger.debug('Changed %s count for proposal %s to %s' % (
pos, proposal.pk, count))
setattr(proposal, 'total_%s' % pos, count)
changed = True
if changed:
logger.debug('Updated proposal %s' % proposal.pk)
proposal.save()
def main(stream=None):
if not apps.ready:
django.setup()
importer = VotesImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
importer.parse_vote_data(data)
importer.update_totals()
[
{
"fields" : {
"abbreviation": "AN",
"country": 1095,
"name": "Assembl\u00e9e nationale"
},
"model": "representatives.chamber",
"pk": 2
},
{
"fields": {
"abbreviation": "SEN",
"country": 1095,
"name": "S\u00e9nat"
},
"model": "representatives.chamber",
"pk": 3
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.303Z",
"title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"reference": "14/liberte_maires_rythmes_scolaires_premier_degre",
"created": "2016-07-07T20:23:24.302Z"
},
"model": "representatives_votes.dossier",
"pk": 1
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.365Z",
"title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"reference": "14/action_publique_territoriale_metropoles",
"created": "2016-07-07T20:23:24.332Z"
},
"model": "representatives_votes.dossier",
"pk": 2
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.410Z",
"title": "Protection de l'enfant",
"reference": "ppl13-799",
"created": "2016-07-07T20:23:24.410Z"
},
"model": "representatives_votes.dossier",
"pk": 3
},
{
"fields": {
"updated": "2016-07-07T20:23:24.307Z",
"title": "",
"dossier": 1,
"created": "2016-07-07T20:23:24.307Z",
"kind": "procedure-file",
"chamber": 2,
"link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp"
},
"model": "representatives_votes.document",
"pk": 1
},
{
"fields": {
"updated": "2016-07-07T20:23:24.335Z",
"title": "",
"dossier": 2,
"created": "2016-07-07T20:23:24.335Z",
"kind": "procedure-file",
"chamber": 3,
"link": "http://www.senat.fr/dossier-legislatif/pjl12-495.html"
},
"model": "representatives_votes.document",
"pk": 2
},
{
"fields": {
"updated": "2016-07-07T20:23:24.371Z",
"title": "",
"dossier": 2,
"created": "2016-07-07T20:23:24.371Z",
"kind": "procedure-file",
"chamber": 2,
"link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp"
},
"model": "representatives_votes.document",
"pk": 3
},
{
"fields": {
"updated": "2016-07-07T20:23:24.415Z",
"title": "",
"dossier":