Commit ab756151 authored by luxcem's avatar luxcem

update models

parent b1af064a
......@@ -34,10 +34,9 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--parallel', action='store_true', default=False)
parser.add_argument('--nocelery', action='store_true', default=False)
def handle(self, *args, **options):
if options['nocelery']:
if not options['parallel']:
import_representatives_from_compotista(options['parallel'])
else:
import_representatives_from_compotista.delay(options['parallel'])
......@@ -19,44 +19,95 @@
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
import uuid
import hashlib
from datetime import datetime
from django.db import models
from django.utils.functional import cached_property
from django.utils.encoding import smart_str
from uuidfield import UUIDField
class TimeStampedModel(models.Model):
"""
An abstract base class model that provides self-updating
``created`` and ``modified`` fields.
"""
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
class HashableModel(models.Model):
"""
An abstract base class model that provides a fingerprint
field
"""
fingerprint = models.CharField(
max_length=40,
unique=True,
)
class Meta:
abstract = True
def calculate_hash(self):
fingerprint = hashlib.sha1()
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
if field.is_relation:
fingerprint.update(
getattr(self, field_name).fingerprint
)
else:
fingerprint.update(
smart_str(getattr(self, field_name))
)
self.fingerprint = fingerprint.hexdigest()
return self.fingerprint
def get_hash_str(self):
string = ''
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
if field.is_relation:
string += getattr(self, field_name).fingerprint
else:
string += smart_str(getattr(self, field_name))
return string
def save(self, *args, **kwargs):
self.calculate_hash()
super(HashableModel, self).save(*args, **kwargs)
class Country(models.Model):
name = models.CharField(max_length=255)
code = models.CharField(max_length=2)
@property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
fingerprint.update(smart_str(self.code))
return fingerprint.hexdigest()
def __unicode__(self):
return u'{} [{}]'.format(self.name, self.code)
class Representative(models.Model):
class Representative(HashableModel, TimeStampedModel):
"""
Base model for representatives
"""
# We use a UUIDField instead of an autogenerated integer id
# Thid allow better sync capabilities (primary keys are
# consistents accross various servers).
# We use a namespaced UUID, that means that a same representative
# (same name and same remote_id) will always have the same pk
# https://github.com/dcramer/django-uuidfield
id = UUIDField(
primary_key=True,
version=5
)
slug = models.SlugField(max_length=100)
remote_id = models.CharField(max_length=255, unique=True)
first_name = models.CharField(max_length=255, blank=True, null=True)
last_name = models.CharField(max_length=255, blank=True, null=True)
first_name = models.CharField(max_length=255, blank=True, default='')
last_name = models.CharField(max_length=255, blank=True, default='')
full_name = models.CharField(max_length=255)
GENDER = (
(0, "N/A"),
......@@ -64,19 +115,13 @@ class Representative(models.Model):
(2, "M"),
)
gender = models.SmallIntegerField(choices=GENDER, default=0)
birth_place = models.CharField(max_length=255, blank=True, null=True)
birth_place = models.CharField(max_length=255, blank=True, default='')
birth_date = models.DateField(blank=True, null=True)
cv = models.TextField(blank=True, null=True)
cv = models.TextField(blank=True, default='')
photo = models.CharField(max_length=512, null=True)
active = models.BooleanField(default=False)
def save(self, *args, **kwargs):
if self.id == None:
self.id = uuid.uuid5(
uuid.UUID('6e987b9b-d98b-4c3b-9829-8cd0bea327dd'),
'{}{}'.format(self.slug, self.remote_id)
)
super(Representative, self).save(*args, **kwargs)
hashable_fields = ['remote_id']
def __unicode__(self):
return u'{} ({})'.format(self.full_name.decode('utf-8'), self.remote_id)
......@@ -87,7 +132,7 @@ class Representative(models.Model):
# Contact related models
class Contact(models.Model):
class Contact(TimeStampedModel):
representative = models.ForeignKey(Representative)
class Meta:
......@@ -96,44 +141,49 @@ class Contact(models.Model):
class Email(Contact):
email = models.EmailField()
kind = models.CharField(max_length=255, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, default='')
class WebSite(Contact):
url = models.CharField(max_length=2048, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
url = models.CharField(max_length=2048, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
class Address(Contact):
country = models.ForeignKey(Country)
city = models.CharField(max_length=255, blank=True, null=True)
street = models.CharField(max_length=255, blank=True, null=True)
number = models.CharField(max_length=255, blank=True, null=True)
postcode = models.CharField(max_length=255, blank=True, null=True)
floor = models.CharField(max_length=255, blank=True, null=True)
office_number = models.CharField(max_length=255, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
name = models.CharField(max_length=255, blank=True, null=True)
location = models.CharField(max_length=255, blank=True, null=True) # TODO Find standard for storage in charfield
city = models.CharField(max_length=255, blank=True, default='')
street = models.CharField(max_length=255, blank=True, default='')
number = models.CharField(max_length=255, blank=True, default='')
postcode = models.CharField(max_length=255, blank=True, default='')
floor = models.CharField(max_length=255, blank=True, default='')
office_number = models.CharField(max_length=255, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
name = models.CharField(max_length=255, blank=True, default='')
location = models.CharField(max_length=255, blank=True, default='')
class Phone(Contact):
number = models.CharField(max_length=255, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
number = models.CharField(max_length=255, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
address = models.ForeignKey(Address, null=True, related_name='phones')
# Mandate related models
class Group(models.Model):
class Group(TimeStampedModel):
"""
An entity represented by a representative through a mandate
An entity represented by a representative through a mandate
"""
name = models.CharField(max_length=255)
abbreviation = models.CharField(max_length=10, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
abbreviation = models.CharField(max_length=10, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
@cached_property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
fingerprint.update(smart_str(self.abbreviation))
fingerprint.update(smart_str(self.kind))
return fingerprint.hexdigest()
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
......@@ -142,12 +192,18 @@ class Group(models.Model):
return unicode(self.name)
class Constituency(models.Model):
class Constituency(TimeStampedModel):
"""
An authority for which a representative has a mandate
An authority for which a representative has a mandate
"""
name = models.CharField(max_length=255)
@cached_property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
return fingerprint.hexdigest()
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
......@@ -156,21 +212,23 @@ class Constituency(models.Model):
return unicode(self.name)
class Mandate(models.Model):
class Mandate(HashableModel, TimeStampedModel):
group = models.ForeignKey(Group, null=True, related_name='mandates')
constituency = models.ForeignKey(Constituency, null=True, related_name='mandates')
representative = models.ForeignKey(Representative, related_name='mandates')
role = models.CharField(
max_length=25,
blank=True,
null=True,
default='',
help_text="Eg.: president of a political group at the European Parliament"
)
begin_date = models.DateField(blank=True, null=True)
end_date = models.DateField(blank=True, null=True)
link = models.URLField()
hashable_fields = ['group', 'constituency', 'role',
'begin_date', 'end_date', 'representative']
@property
def active(self):
return self.end_date >= datetime.now().date()
......
......@@ -29,11 +29,13 @@ class CountrySerializer(serializers.ModelSerializer):
model = models.Country
fields = ('name', 'code')
class EmailSerializer(serializers.ModelSerializer):
class Meta:
model = models.Email
fields = ('email', 'kind')
class WebsiteSerializer(serializers.ModelSerializer):
class Meta:
model = models.WebSite
......@@ -45,6 +47,7 @@ class WebsiteSerializer(serializers.ModelSerializer):
'''
return value
class PhoneSerializer(serializers.ModelSerializer):
class Meta:
model = models.Phone
......@@ -53,12 +56,17 @@ class PhoneSerializer(serializers.ModelSerializer):
def validate_phone(self, value):
return value
class AddressSerializer(serializers.ModelSerializer):
country = CountrySerializer()
phones = PhoneSerializer(many=True)
class Meta:
model = models.Address
fields = ('country', 'city', 'street', 'number', 'postcode', 'floor', 'office_number', 'kind', 'phones')
fields = ('country', 'city', 'street',
'number', 'postcode', 'floor',
'office_number', 'kind', 'phones',
)
class ContactField(serializers.Serializer):
emails = EmailSerializer(many=True)
......@@ -80,7 +88,10 @@ class MandateSerializer(serializers.ModelSerializer):
short_id = serializers.CharField(source='group.abbreviation', allow_null=True)
kind = serializers.CharField(source='group.kind')
constituency = serializers.CharField(source='constituency.name')
# def validate_fingerprint(self, value):
# return value
class Meta:
model = models.Mandate
fields = (
......@@ -94,6 +105,7 @@ class MandateSerializer(serializers.ModelSerializer):
'representative',
'begin_date',
'end_date',
'fingerprint',
)
......@@ -108,7 +120,6 @@ class RepresentativeMandateSerializer(MandateSerializer):
class RepresentativeSerializer(serializers.ModelSerializer):
id = serializers.UUIDField(format='hex', read_only=True)
class Meta:
model = models.Representative
fields = (
......@@ -123,6 +134,7 @@ class RepresentativeSerializer(serializers.ModelSerializer):
'birth_date',
'photo',
'active',
'fingerprint'
)
......@@ -154,22 +166,46 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
contacts_data = validated_data.pop('contacts')
mandates_data = validated_data.pop('mandates')
representative = models.Representative.objects.update_or_create(
representative = models.Representative.objects.create(
**validated_data
)
self._create_mandates(mandates_data, representative)
self._create_contacts(contacts_data, representative)
return representative
def update(self, instance, validated_data):
contacts_data = validated_data.pop('contacts')
mandates_data = validated_data.pop('mandates')
for attr, value in validated_data.iteritems():
setattr(instance, attr, value)
instance.save()
self._create_mandates(mandates_data, instance)
self._create_contacts(contacts_data, instance)
return instance
def touch_model(self, model, **data):
'''
This method create or look up a model with the given data
it saves the given model if it exists, updating its
updated field
'''
instance, created = model.objects.get_or_create(**data)
if not created:
instance.save()
return (instance, created)
def _create_contacts(self, contacts_data, representative):
for contact_data in contacts_data['emails']:
contact_data['representative'] = representative
models.Email.objects.create(**contact_data)
self.touch_model(model=models.Email, **contact_data)
for contact_data in contacts_data['websites']:
contact_data['representative'] = representative
models.WebSite.objects.create(**contact_data)
self.touch_model(model=models.WebSite, **contact_data)
for contact_data in contacts_data['address']:
country, _ = models.Country.objects.get_or_create(
......@@ -178,22 +214,27 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
phone_set = contact_data.pop('phones')
contact_data['representative'] = representative
contact_data['country'] = country
contact = models.Address.objects.create(**contact_data)
contact, _ = self.touch_model(model=models.Address, **contact_data)
for phone_data in phone_set:
phone_data['representative'] = representative
phone_data['address'] = contact
models.Phone.objects.create(**phone_data)
self.touch_model(model=models.Phone, **phone_data)
def _create_mandates(self, mandates_data, representative):
for mandate_data in mandates_data:
constituency, _ = models.Constituency.objects.get_or_create(
for mandate_data in mandates_data:
# serializer = MandateSerializer(data=mandate_data)
constituency, _ = self.touch_model(model=models.Constituency,
**mandate_data.pop('constituency')
)
group, _ = models.Group.objects.get_or_create(
group, _ = self.touch_model(model=models.Group,
**mandate_data.pop('group')
)
mandate_data['representative'] = representative
mandate_data['constituency'] = constituency
mandate_data['group'] = group
models.Mandate.objects.create(**mandate_data)
models.Mandate.objects.update_or_create(
fingerprint=mandate_data['fingerprint'],
defaults=mandate_data
)
......@@ -22,13 +22,13 @@
from __future__ import absolute_import
from django.conf import settings
from django.utils import timezone
import ijson
from celery import shared_task
from urllib2 import urlopen
from representatives.models import Representative
from representatives.models import Representative, Group, Constituency, Mandate, Address, Phone, Email, WebSite
from representatives.serializers import RepresentativeDetailSerializer
......@@ -38,7 +38,18 @@ def import_a_representative(data, verbose=False):
Import a representative from a serialized
Python datatypes
'''
serializer = RepresentativeDetailSerializer(data=data)
try:
representative = Representative.objects.get(
fingerprint=data['fingerprint']
)
serializer = RepresentativeDetailSerializer(
instance=representative,
data=data
)
except:
serializer = RepresentativeDetailSerializer(data=data)
if serializer.is_valid():
return serializer.save()
else:
......@@ -47,13 +58,11 @@ def import_a_representative(data, verbose=False):
@shared_task
def import_representatives_from_compotista(delay=False):
# Clean data before import
Representative.objects.all().delete()
def import_representatives_from_compotista(delay=False):
compotista_server = getattr(settings,
'COMPOTISTA_SERVER',
'http://compotista.mm.staz.be')
import_start_datetime = timezone.now()
url = compotista_server + '/export/latest/'
res = urlopen(url)
for representative in ijson.items(res, 'item'):
......@@ -61,6 +70,10 @@ def import_representatives_from_compotista(delay=False):
representative = import_a_representative.delay(representative)
else:
representative = import_a_representative(representative)
for model in [Representative, Group, Constituency,
Mandate, Address, Phone, Email, WebSite]:
model.objects.filter(updated__lt=import_start_datetime).delete()
@shared_task
......@@ -74,5 +87,10 @@ def export_a_representative(representative):
@shared_task
def export_representatives(filters={}):
return [export_a_representative.delay(representative) for representative in Representative.objects.filter(**filters)]
def export_representatives(delay=False, **filters):
if delay:
return [export_a_representative.delay(representative)
for representative in Representative.objects.filter(**filters)]
else:
return [export_a_representative(representative)
for representative in Representative.objects.filter(**filters)]
# coding: utf-8
# This file is part of compotista.
#
# compotista is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# compotista is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment