Commit 13187632 authored by luxcem's avatar luxcem

test

parent c6a3f754
......@@ -49,7 +49,7 @@ class MandateInline(admin.StackedInline):
class RepresentativeAdmin(admin.ModelAdmin):
list_display = ('full_name', 'gender', 'birth_place')
list_display = ('id', 'full_name', 'gender', 'birth_place')
search_fields = ('first_name', 'last_name', 'birth_place')
list_filter = ('gender', )
inlines = [
......@@ -62,7 +62,7 @@ class RepresentativeAdmin(admin.ModelAdmin):
class MandateAdmin(admin.ModelAdmin):
list_display = ('representative', 'group', 'role', 'constituency', 'begin_date', 'end_date')
list_display = ('id', 'representative', 'group', 'role', 'constituency', 'begin_date', 'end_date')
search_fields = ('representative', 'group', 'constituency')
......
......@@ -19,40 +19,25 @@
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
import json
import ijson
import pyprind
from django.core.management.base import BaseCommand
from django.conf import settings
from urllib2 import urlopen
from representatives.tasks import import_representatives_from_compotista
from representatives.models import Representative
from representatives.utils import import_a_representative
class Command(BaseCommand):
def handle(self, *args, **options):
Representative.objects.all().delete()
self.compotista_server = getattr(settings,
'COMPOTISTA_SERVER',
'http://compotista.mm.staz.be')
url = self.compotista_server + '/export/latest/'
print('Import representatives from %s' % url)
"""
Command to import representative from a compotista server first
command call should not be in parallel (group and counstituency
would be created in double or triple), next calls could be or not in
parallel
"""
def add_arguments(self, parser):
parser.add_argument('--parallel', action='store_true', default=False)
parser.add_argument('--nocelery', action='store_true', default=False)
bar = pyprind.ProgBar(self.get_number_of_meps())
resource = urlopen(url)
for i, representative in enumerate(ijson.items(resource, 'item')):
representative = import_a_representative(representative)
representative_id = '{} - {}'.format(i, representative.full_name.encode('utf-8'))
bar.update(item_id = representative_id)
print(bar)
def get_number_of_meps(self):
response = urlopen(self.compotista_server + '/api/representatives/')
return int(json.load(response).get('count'))
def handle(self, *args, **options):
if options['nocelery']:
import_representatives_from_compotista(options['parallel'])
else:
import_representatives_from_compotista.delay(options['parallel'])
......@@ -2,6 +2,7 @@
from __future__ import unicode_literals
from django.db import models, migrations
import uuidfield.fields
class Migration(migrations.Migration):
......@@ -27,7 +28,6 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Constituency',
......@@ -35,9 +35,6 @@ class Migration(migrations.Migration):
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(max_length=255)),
],
options={
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Country',
......@@ -46,21 +43,17 @@ class Migration(migrations.Migration):
('name', models.CharField(max_length=255)),
('code', models.CharField(max_length=2)),
],
options={
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Email',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('email', models.EmailField(max_length=75)),
('email', models.EmailField(max_length=254)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
options={
'abstract': False,
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Group',
......@@ -70,9 +63,6 @@ class Migration(migrations.Migration):
('abbreviation', models.CharField(max_length=10, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
options={
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Mandate',
......@@ -85,9 +75,6 @@ class Migration(migrations.Migration):
('constituency', models.ForeignKey(related_name='mandates', to='representatives.Constituency', null=True)),
('group', models.ForeignKey(related_name='mandates', to='representatives.Group', null=True)),
],
options={
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Phone',
......@@ -100,12 +87,11 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
bases=(models.Model,),
),
migrations.CreateModel(
name='Representative',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('id', uuidfield.fields.UUIDField(max_length=32, serialize=False, primary_key=True)),
('slug', models.SlugField(max_length=100)),
('remote_id', models.CharField(unique=True, max_length=255)),
('first_name', models.CharField(max_length=255, null=True, blank=True)),
......@@ -118,9 +104,6 @@ class Migration(migrations.Migration):
('photo', models.CharField(max_length=512, null=True)),
('active', models.BooleanField(default=False)),
],
options={
},
bases=(models.Model,),
),
migrations.CreateModel(
name='WebSite',
......@@ -133,36 +116,30 @@ class Migration(migrations.Migration):
options={
'abstract': False,
},
bases=(models.Model,),
),
migrations.AddField(
model_name='phone',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
preserve_default=True,
),
migrations.AddField(
model_name='mandate',
name='representative',
field=models.ForeignKey(related_name='mandates', to='representatives.Representative'),
preserve_default=True,
),
migrations.AddField(
model_name='email',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
preserve_default=True,
),
migrations.AddField(
model_name='address',
name='country',
field=models.ForeignKey(to='representatives.Country'),
preserve_default=True,
),
migrations.AddField(
model_name='address',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
preserve_default=True,
),
]
......@@ -19,7 +19,13 @@
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
import uuid
from datetime import datetime
from django.db import models
from django.utils.functional import cached_property
from uuidfield import UUIDField
class Country(models.Model):
......@@ -27,32 +33,54 @@ class Country(models.Model):
code = models.CharField(max_length=2)
def __unicode__(self):
return "%s [%s]" % (self.name, self.code)
return u'{} [{}]'.format(self.name, self.code)
class Representative(models.Model):
"""
Base model for representatives
"""
GENDER = (
(0, "N/A"),
(1, "F"),
(2, "M"),
)
# We use a UUIDField instead of an autogenerated integer id
# Thid allow better sync capabilities (primary keys are
# consistents accross various servers).
# We use a namespaced UUID, that means that a same representative
# (same name and same remote_id) will always have the same pk
# https://github.com/dcramer/django-uuidfield
id = UUIDField(
primary_key=True,
version=5
)
slug = models.SlugField(max_length=100)
remote_id = models.CharField(max_length=255, unique=True)
first_name = models.CharField(max_length=255, blank=True, null=True)
last_name = models.CharField(max_length=255, blank=True, null=True)
full_name = models.CharField(max_length=255)
GENDER = (
(0, "N/A"),
(1, "F"),
(2, "M"),
)
gender = models.SmallIntegerField(choices=GENDER, default=0)
birth_place = models.CharField(max_length=255, blank=True, null=True)
birth_date = models.DateField(blank=True, null=True)
cv = models.TextField(blank=True, null=True)
photo = models.CharField(max_length=512, null=True)
active = models.BooleanField(default=False)
def save(self, *args, **kwargs):
if self.id == None:
self.id = uuid.uuid5(
uuid.UUID('6e987b9b-d98b-4c3b-9829-8cd0bea327dd'),
'{}{}'.format(self.slug, self.remote_id)
)
super(Representative, self).save(*args, **kwargs)
def __unicode__(self):
return self.full_name
return u'{} ({})'.format(self.full_name.decode('utf-8'), self.remote_id)
def gender_as_str(self):
genders = {0: 'N/A', 1: 'F', 2: 'M'}
return genders[self.gender]
......@@ -106,6 +134,10 @@ class Group(models.Model):
abbreviation = models.CharField(max_length=10, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
def __unicode__(self):
return unicode(self.name)
......@@ -115,9 +147,13 @@ class Constituency(models.Model):
An authority for which a representative has a mandate
"""
name = models.CharField(max_length=255)
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
def __unicode__(self):
return self.name
return unicode(self.name)
class Mandate(models.Model):
......@@ -135,3 +171,14 @@ class Mandate(models.Model):
end_date = models.DateField(blank=True, null=True)
link = models.URLField()
@property
def active(self):
return self.end_date >= datetime.now().date()
def __unicode__(self):
return u'Mandate : {representative},{role} {group} for {constituency}'.format(
representative=self.representative,
role=(u' {} of'.format(self.role) if self.role else u''),
constituency=self.constituency,
group=self.group
)
......@@ -18,11 +18,11 @@
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.db import transaction
import representatives.models as models
from rest_framework import serializers
from django.db import transaction
class CountrySerializer(serializers.ModelSerializer):
class Meta:
......@@ -108,6 +108,7 @@ class RepresentativeMandateSerializer(MandateSerializer):
class RepresentativeSerializer(serializers.ModelSerializer):
id = serializers.UUIDField(format='hex', read_only=True)
class Meta:
model = models.Representative
fields = (
......@@ -141,28 +142,34 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
)
# Nested creation is not implemented yet in DRF, it sucks
# We made an intensive use of get_or_create to avoid recreating representatives
# The idea here is to truncate all models except representatives and recreate them
# every import
# TODO : fix this code when it will be implemented
@transaction.atomic
def create(self, validated_data):
"""
Nested creation is not implemented yet in DRF, it sucks We made an
intensive use of get_or_create to avoid recreating
representatives The idea here is to truncate all models except
representatives and recreate them every import.
TODO : fix this code when it will be implemented
"""
contacts_data = validated_data.pop('contacts')
mandates_data = validated_data.pop('mandates')
representative = models.Representative.objects.create(**validated_data)
representative = models.Representative.objects.update_or_create(
**validated_data
)
self._create_mandates(mandates_data, representative)
self._create_contacts(contacts_data, representative)
return representative
def _create_contacts(self, contacts_data, representative):
for contact_data in contacts_data['emails']:
contact_data['representative'] = representative
contact = models.Email.objects.create(**contact_data)
models.Email.objects.create(**contact_data)
for contact_data in contacts_data['websites']:
contact_data['representative'] = representative
contact = models.WebSite.objects.create(**contact_data)
models.WebSite.objects.create(**contact_data)
for contact_data in contacts_data['address']:
country, _ = models.Country.objects.get_or_create(
......@@ -176,8 +183,7 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
for phone_data in phone_set:
phone_data['representative'] = representative
phone_data['address'] = contact
models.Phone.objects.create(**phone_data)
models.Phone.objects.create(**phone_data)
def _create_mandates(self, mandates_data, representative):
for mandate_data in mandates_data:
......
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from __future__ import absolute_import
from django.conf import settings
import ijson
from celery import shared_task
from urllib2 import urlopen
from representatives.models import Representative
from representatives.serializers import RepresentativeDetailSerializer
@shared_task
def import_a_representative(data, verbose=False):
'''
Import a representative from a serialized
Python datatypes
'''
serializer = RepresentativeDetailSerializer(data=data)
if serializer.is_valid():
return serializer.save()
else:
# print(data)
raise Exception(serializer.errors)
@shared_task
def import_representatives_from_compotista(delay=False):
# Clean data before import
Representative.objects.all().delete()
compotista_server = getattr(settings,
'COMPOTISTA_SERVER',
'http://compotista.mm.staz.be')
url = compotista_server + '/export/latest/'
res = urlopen(url)
for representative in ijson.items(res, 'item'):
if delay:
representative = import_a_representative.delay(representative)
else:
representative = import_a_representative(representative)
@shared_task
def export_a_representative(representative):
'''
Export a representative to a serialized
Python datatypes
'''
serialized = RepresentativeDetailSerializer(representative)
return serialized.data
@shared_task
def export_representatives(filters={}):
return [export_a_representative.delay(representative) for representative in Representative.objects.filter(**filters)]
......@@ -20,34 +20,3 @@
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from representatives.models import Representative
from representatives.serializers import RepresentativeDetailSerializer
# Import a representative
def import_a_representative(data, verbose=False):
serializer = RepresentativeDetailSerializer(data=data)
if serializer.is_valid():
representative = serializer.save()
if verbose:
print(representative)
return representative
else:
print(data)
raise Exception(serializer.errors)
def import_representatives(data, verbose=False):
return [import_a_representative(r_data, verbose) for r_data in data]
# Export
def export_a_representative(representative):
serialized = RepresentativeDetailSerializer(representative)
return serialized.data
def export_representatives(filters={}):
return [export_a_representative(representative) for representative in Representative.objects.filter(**filters)]
def export_all_representatives():
return export_representatives()
def export_active_representatives():
return export_representatives({'active': True})
django>=1.7,<1.8
djangorestframework
ijson
pyprind
celery
django-uuidfield
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment