Commit 7a387ae8 authored by Jamesie Pic's avatar Jamesie Pic

Import representatives module

Note: it **must** be kept loosely coupled with other modules in this
repository. It was merged here just to remove the burden of having
several repositories to manage: volunteer time is precious.
parents 813d0336 9a75eb6b
......@@ -2,3 +2,7 @@
omit =
src/representatives_positions/contrib/*
src/representatives_recommendations/contrib/*
src/representatives/tests/*
src/representatives/migrations/*
src/representatives/contrib/francedata/tests/*
src/representatives/contrib/parltrack/tests/*
# coding: utf-8
# This file is part of compotista.
#
# compotista is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# compotista is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.contrib import admin
from .models import (Address, Constituency, Country, Email, Group, Mandate,
Phone, Representative, WebSite)
class EmailInline(admin.TabularInline):
model = Email
extra = 0
class WebsiteInline(admin.TabularInline):
model = WebSite
extra = 0
class AdressInline(admin.StackedInline):
model = Address
extra = 0
class PhoneInline(admin.TabularInline):
model = Phone
extra = 0
class MandateInline(admin.StackedInline):
model = Mandate
extra = 0
class RepresentativeAdmin(admin.ModelAdmin):
list_display = ('id', 'full_name', 'gender', 'birth_place')
search_fields = ('first_name', 'last_name', 'birth_place')
list_filter = ('gender', )
inlines = [
PhoneInline,
EmailInline,
WebsiteInline,
AdressInline,
MandateInline
]
class GroupAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'abbreviation', 'kind')
list_filter = ('kind',)
class MandateAdmin(admin.ModelAdmin):
list_display = (
'id',
'representative',
'group',
'role',
'constituency',
'begin_date',
'end_date')
search_fields = ('representative', 'group', 'constituency')
admin.site.register(Representative, RepresentativeAdmin)
admin.site.register(Country)
admin.site.register(Mandate, MandateAdmin)
admin.site.register(Group, GroupAdmin)
admin.site.register(Constituency)
from django.db import models
from rest_framework import (
filters,
pagination,
renderers,
viewsets,
)
from rql_filter.backend import RQLFilterBackend
from representatives.serializers import (
ChamberSerializer,
ConstituencySerializer,
CountrySerializer,
GroupSerializer,
MandateSerializer,
RepresentativeDetailSerializer,
RepresentativeSerializer,
)
from .models import (
Address,
Chamber,
Constituency,
Country,
Group,
Mandate,
Phone,
Representative,
)
class DefaultWebPagination(pagination.PageNumberPagination):
default_web_page_size = 10
def get_page_size(self, request):
web = isinstance(request.accepted_renderer,
renderers.BrowsableAPIRenderer)
size = pagination.PageNumberPagination.get_page_size(self, request)
if web and not size:
return self.default_web_page_size
return size
class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows representatives to be viewed.
"""
queryset = Representative.objects.all()
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'active': ['exact'],
'slug': ['exact', 'icontains'],
'id': ['exact'],
'first_name': ['exact', 'icontains'],
'last_name': ['exact', 'icontains'],
'full_name': ['exact', 'icontains'],
'gender': ['exact'],
'birth_place': ['exact'],
'birth_date': ['exact', 'gte', 'lte'],
}
search_fields = ('first_name', 'last_name', 'slug')
ordering_fields = ('id', 'birth_date', 'last_name', 'full_name')
pagination_class = DefaultWebPagination
def get_queryset(self):
qs = super(RepresentativeViewSet, self).get_queryset()
qs = qs.prefetch_related(
'email_set',
'website_set',
models.Prefetch(
'address_set',
queryset=Address.objects.select_related('country')
),
models.Prefetch(
'phone_set',
queryset=Phone.objects.select_related('address__country')
),
'mandates',
)
return qs
def list(self, request):
self.serializer_class = RepresentativeSerializer
return super(RepresentativeViewSet, self).list(request)
def retrieve(self, request, pk=None):
self.serializer_class = RepresentativeDetailSerializer
return super(RepresentativeViewSet, self).retrieve(request, pk)
class MandateViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows mandates to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Mandate.objects.select_related('representative')
serializer_class = MandateSerializer
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'id': ['exact'],
'group__name': ['exact', 'icontains'],
'group__abbreviation': ['exact'],
}
search_fields = ('group__name', 'group__abbreviation')
class ConstituencyViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Constituency.objects.all()
serializer_class = ConstituencySerializer
filter_backends = (
RQLFilterBackend,
)
class GroupViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Group.objects.all()
serializer_class = GroupSerializer
filter_backends = (
RQLFilterBackend,
)
class ChamberViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Chamber.objects.all()
serializer_class = ChamberSerializer
filter_backends = (
RQLFilterBackend,
)
class CountryViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Country.objects.all()
serializer_class = CountrySerializer
filter_backends = (
RQLFilterBackend,
)
[
{
"twitter": "bernardroman59",
"profession": "Avocat, administrateur territorial",
"url_nosdeputes_api": "https://www.nosdeputes.fr/bernard-roman/json",
"nb_mandats": 2,
"chambre": "AN",
"id": 153,
"groupes_parlementaires": [
{
"responsabilite": {
"organisme": "Groupe d'amitié france-mexique",
"fonction": "vice-président"
}
},
{
"responsabilite": {
"organisme": "Groupe d'amitié france-argentine",
"fonction": "vice-président"
}
}
],
"sexe": "H",
"lieu_naissance": "Lille (Nord)",
"anciens_autres_mandats": [
{
"mandat": "Nord-Pas-de-Calais / Conseil régional / Membre / 17/03/1986 / 01/04/1989"
},
{
"mandat": "Nord / Conseil général / Membre / 03/10/1988 / 27/03/1994"
},
{
"mandat": "Lille (Nord) / Conseil municipal / Adjoint au Maire / 19/03/2001 / 17/04/2004"
},
{
"mandat": "Nord / Conseil général / Membre / 28/03/1994 / 31/07/1997"
},
{
"mandat": "Lille (Nord) / Conseil municipal / Adjoint au Maire / 14/03/1983 / 19/03/1989"
},
{
"mandat": "Lille (Nord) / Conseil municipal / Adjoint au Maire / 20/03/1989 / 18/06/1995"
},
{
"mandat": "Lille (Nord) / Conseil municipal / Adjoint au Maire / 19/06/1995 / 18/03/2001"
},
{
"mandat": "Nord-Pas-de-Calais / Conseil régional / Vice-président / 29/03/2004 / 14/03/2010"
}
],
"mandat_debut": "2012-06-20",
"emails": [
{
"email": "contact@bernard-roman.org"
},
{
"email": "broman@assemblee-nationale.fr"
}
],
"responsabilites_extra_parlementaires": [],
"anciens_mandats": [
{
"mandat": "20/06/2007 / 19/06/2012 / fin de législature"
},
{
"mandat": "20/06/2012 / / "
},
{
"mandat": "01/06/1997 / 18/06/2002 / fin de législature"
},
{
"mandat": "19/06/2002 / 19/06/2007 / fin de législature"
}
],
"groupe": {
"organisme": "Socialiste, républicain et citoyen",
"fonction": "membre"
},
"nom": "Bernard Roman",
"sites_web": [
{
"site": "http://www.bernard-roman.net"
},
{
"site": "https://twitter.com/bernardroman59"
}
],
"groupe_sigle": "SRC",
"nom_circo": "Nord",
"parti_ratt_financier": "Parti socialiste",
"place_en_hemicycle": "424",
"autres_mandats": [
{
"mandat": "Nord-Pas-de-Calais / Conseil régional / vice-président"
}
],
"url_nosdeputes": "https://www.nosdeputes.fr/bernard-roman",
"url_an": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611",
"adresses": [
{
"geo": {
"geometry": {
"type": "Point",
"coordinates": [
3.06785,
50.623596
]
},
"type": "Feature",
"properties": {
"city": "Lille",
"citycode": "59350",
"street": "Rue d'Arras",
"name": "20 Rue d'Arras",
"housenumber": "20",
"label": "20 Rue d'Arras 59000 Lille",
"score": 0.30006938349007306,
"postcode": "59000",
"context": "59, Nord, Nord-Pas-de-Calais",
"type": "housenumber",
"id": "ADRNIVX_0000000331894641"
}
},
"tel": "03 20 52 09 20",
"fax": "03 28 54 01 37",
"adresse": "Permanence, 165 Rue d'Arras, 59000 Lille Téléphone : 03 20 52 09 20 Télécopie : 03 28 54 01 37"
},
{
"geo": {
"geometry": {
"type": "Point",
"coordinates": [
3.077254,
50.628633
]
},
"type": "Feature",
"properties": {
"city": "Lille",
"citycode": "59350",
"name": "Avenue du Président Hoover",
"label": "Avenue du Président Hoover 59000 Lille",
"score": 0.4689764705882353,
"postcode": "59000",
"context": "59, Nord, Nord-Pas-de-Calais",
"type": "street",
"id": "59350_XXXX_32cc9a"
}
},
"adresse": "Conseil régional, 151 Avenue du Président Hoover 59555 Lille cedex"
},
{
"adresse": "Assemblée nationale, 126 Rue de l'Université, 75355 Paris 07 SP"
}
],
"date_naissance": "1952-07-15",
"slug": "bernard-roman",
"id_an": "2611",
"photo_url": "http://www.nosdeputes.fr/depute/photo/bernard-roman",
"prenom": "Bernard",
"num_deptmt": "59",
"nom_de_famille": "Roman",
"num_circo": 1,
"responsabilites": [
{
"responsabilite": {
"organisme": "Bureau de l'assemblée nationale",
"fonction": "questeur"
}
},
{
"responsabilite": {
"organisme": "Commission des lois constitutionnelles, de la législation et de l'administration générale de la république",
"fonction": "membre"
}
},
{
"responsabilite": {
"organisme": "Délégation chargée de la communication et de la presse",
"fonction": "membre"
}
}
]
},
{
"mandat_debut": "2004-09-26",
"mandat_fin": "2012-01-01",
"twitter": "dassouline",
"profession": "Professeur d'histoire-géographie",
"url_nossenateurs_api": "http://www.nossenateurs.fr/david-assouline/json",
"chambre": "SEN",
"nb_mandats": 3,
"id": 44,
"groupes_parlementaires": [
{
"responsabilite": {
"organisme": "Groupe France-Japon",
"fonction": "président"
}
}
],
"sexe": "H",
"lieu_naissance": "non disponible",
"anciens_autres_mandats": [],
"url_nossenateurs": "http://www.nossenateurs.fr/david-assouline",
"emails": [
{
"email": "d.assouline@senat.fr"
}
],
"responsabilites_extra_parlementaires": [
{
"responsabilite": {
"organisme": "Centre national du cinéma et de l'image animée",
"fonction": "membre"
}
},
{
"responsabilite": {
"organisme": "Commission de modernisation de la diffusion audiovisuelle",
"fonction": "membre"
}
},
{
"responsabilite": {
"organisme": "Section française de l'Assemblée parlementaire de la francophonie (A.P.F)",
"fonction": "membre"
}
}
],
"anciens_mandats": [
{
"mandat": "25/09/2011 / / "
},
{
"mandat": "26/09/2004 / 25/09/2011 / "
}
],
"url_institution": "http://www.senat.fr/senateur/assouline_david04059m.html",
"parti_ratt_financier": "Parti socialiste",
"groupe": {
"organisme": "Socialiste et républicain",
"fonction": "membre"
},
"nom": "David Assouline",
"sites_web": [
{
"site": "http://www.david-assouline.net"
},
{
"site": "https://twitter.com/dassouline"
},
{
"site": "https://www.facebook.com/DavAssouline"
},
{
"site": "http://david-assouline.net"
}
],
"groupe_sigle": "SOC",
"nom_circo": "Paris",
"id_institution": "assouline_david04059m",
"place_en_hemicycle": "114",
"autres_mandats": [
{
"mandat": "Paris (20ème arrondissement) / conseiller"
}
],
"adresses": [],
"date_naissance": "1959-06-16",
"slug": "david-assouline",
"photo_url": "http://www.nossenateurs.fr/senateur/photo/david-assouline",
"prenom": "David",
"num_deptmt": "75",
"nom_de_famille": "Assouline",
"num_circo": "non disponible",
"responsabilites": [
{
"responsabilite": {
"organisme": "Commission de la culture, de l'éducation et de la communication",
"fonction": "vice-président"
}
}
]
}
]
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.contrib.francedata import import_representatives
@pytest.mark.django_db
def test_francedata_import_representatives():
inputjson = os.path.join(os.path.dirname(__file__),
'representatives_input.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(inputjson, 'r') as f:
import_representatives.main(f)
missing = []
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for field in exclude:
if field in compare:
compare.pop(field)
try:
type(obj.object).objects.get(**compare)
except:
missing.append(compare)
assert len(missing) is 0
assert Representative.objects.count() == 2
# coding: utf-8
class DelegationHelper:
'''
Helper class for building committees/delegations from rep json data
given dicts for equivalences and abbreviations
'''
def __init__(self, equivs, abbrevs, committees=True):
self.equivs = equivs