Commit 53cf0e6b authored by luxcem's avatar luxcem

better sync models

parent b1af064a
......@@ -34,10 +34,9 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--parallel', action='store_true', default=False)
parser.add_argument('--nocelery', action='store_true', default=False)
def handle(self, *args, **options):
if options['nocelery']:
if not options['parallel']:
import_representatives_from_compotista(options['parallel'])
else:
import_representatives_from_compotista.delay(options['parallel'])
......@@ -11,31 +11,6 @@ class Migration(migrations.Migration):
]
operations = [
migrations.CreateModel(
name='Address',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('city', models.CharField(max_length=255, null=True, blank=True)),
('street', models.CharField(max_length=255, null=True, blank=True)),
('number', models.CharField(max_length=255, null=True, blank=True)),
('postcode', models.CharField(max_length=255, null=True, blank=True)),
('floor', models.CharField(max_length=255, null=True, blank=True)),
('office_number', models.CharField(max_length=255, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('name', models.CharField(max_length=255, null=True, blank=True)),
('location', models.CharField(max_length=255, null=True, blank=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Constituency',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(max_length=255)),
],
),
migrations.CreateModel(
name='Country',
fields=[
......@@ -43,103 +18,5 @@ class Migration(migrations.Migration):
('name', models.CharField(max_length=255)),
('code', models.CharField(max_length=2)),
],
),
migrations.CreateModel(
name='Email',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('email', models.EmailField(max_length=254)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Group',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(max_length=255)),
('abbreviation', models.CharField(max_length=10, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
),
migrations.CreateModel(
name='Mandate',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('role', models.CharField(help_text=b'Eg.: president of a political group at the European Parliament', max_length=25, null=True, blank=True)),
('begin_date', models.DateField(null=True, blank=True)),
('end_date', models.DateField(null=True, blank=True)),
('link', models.URLField()),
('constituency', models.ForeignKey(related_name='mandates', to='representatives.Constituency', null=True)),
('group', models.ForeignKey(related_name='mandates', to='representatives.Group', null=True)),
],
),
migrations.CreateModel(
name='Phone',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('number', models.CharField(max_length=255, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('address', models.ForeignKey(related_name='phones', to='representatives.Address', null=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Representative',
fields=[
('id', uuidfield.fields.UUIDField(max_length=32, serialize=False, primary_key=True)),
('slug', models.SlugField(max_length=100)),
('remote_id', models.CharField(unique=True, max_length=255)),
('first_name', models.CharField(max_length=255, null=True, blank=True)),
('last_name', models.CharField(max_length=255, null=True, blank=True)),
('full_name', models.CharField(max_length=255)),
('gender', models.SmallIntegerField(default=0, choices=[(0, b'N/A'), (1, b'F'), (2, b'M')])),
('birth_place', models.CharField(max_length=255, null=True, blank=True)),
('birth_date', models.DateField(null=True, blank=True)),
('cv', models.TextField(null=True, blank=True)),
('photo', models.CharField(max_length=512, null=True)),
('active', models.BooleanField(default=False)),
],
),
migrations.CreateModel(
name='WebSite',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('url', models.CharField(max_length=2048, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('representative', models.ForeignKey(to='representatives.Representative')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='phone',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
migrations.AddField(
model_name='mandate',
name='representative',
field=models.ForeignKey(related_name='mandates', to='representatives.Representative'),
),
migrations.AddField(
model_name='email',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
migrations.AddField(
model_name='address',
name='country',
field=models.ForeignKey(to='representatives.Country'),
),
migrations.AddField(
model_name='address',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
)
]
......@@ -28,6 +28,7 @@ def unload_fixture(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('representatives', '0001_initial'),
]
operations = [
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('representatives', '0002_fixtures'),
]
operations = [
migrations.CreateModel(
name='Address',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('city', models.CharField(max_length=255, null=True, blank=True)),
('street', models.CharField(max_length=255, null=True, blank=True)),
('number', models.CharField(max_length=255, null=True, blank=True)),
('postcode', models.CharField(max_length=255, null=True, blank=True)),
('floor', models.CharField(max_length=255, null=True, blank=True)),
('office_number', models.CharField(max_length=255, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('name', models.CharField(max_length=255, null=True, blank=True)),
('location', models.CharField(max_length=255, null=True, blank=True)),
('country', models.ForeignKey(to='representatives.Country')),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Constituency',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=255)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Email',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('email', models.EmailField(max_length=254)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Group',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('name', models.CharField(max_length=255)),
('abbreviation', models.CharField(max_length=10, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Mandate',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('fingerprint', models.CharField(unique=True, max_length=40)),
('role', models.CharField(default=b'', help_text=b'Eg.: president of a political group at the European Parliament', max_length=25, blank=True)),
('begin_date', models.DateField(null=True, blank=True)),
('end_date', models.DateField(null=True, blank=True)),
('link', models.URLField()),
('constituency', models.ForeignKey(related_name='mandates', to='representatives.Constituency', null=True)),
('group', models.ForeignKey(related_name='mandates', to='representatives.Group', null=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Phone',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('number', models.CharField(max_length=255, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('address', models.ForeignKey(related_name='phones', to='representatives.Address', null=True)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='Representative',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('fingerprint', models.CharField(unique=True, max_length=40)),
('slug', models.SlugField(max_length=100)),
('remote_id', models.CharField(unique=True, max_length=255)),
('first_name', models.CharField(max_length=255, null=True, blank=True)),
('last_name', models.CharField(max_length=255, null=True, blank=True)),
('full_name', models.CharField(max_length=255)),
('gender', models.SmallIntegerField(default=0, choices=[(0, b'N/A'), (1, b'F'), (2, b'M')])),
('birth_place', models.CharField(max_length=255, null=True, blank=True)),
('birth_date', models.DateField(null=True, blank=True)),
('cv', models.TextField(null=True, blank=True)),
('photo', models.CharField(max_length=512, null=True)),
('active', models.BooleanField(default=False)),
],
options={
'abstract': False,
},
),
migrations.CreateModel(
name='WebSite',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('url', models.CharField(max_length=2048, null=True, blank=True)),
('kind', models.CharField(max_length=255, null=True, blank=True)),
('representative', models.ForeignKey(to='representatives.Representative')),
],
options={
'abstract': False,
},
),
migrations.AddField(
model_name='phone',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
migrations.AddField(
model_name='mandate',
name='representative',
field=models.ForeignKey(related_name='mandates', to='representatives.Representative'),
),
migrations.AddField(
model_name='email',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
migrations.AddField(
model_name='address',
name='representative',
field=models.ForeignKey(to='representatives.Representative'),
),
]
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
import hashlib
from django.db import models
from django.utils.encoding import smart_str
class TimeStampedModel(models.Model):
"""
An abstract base class model that provides self-updating
``created`` and ``modified`` fields.
"""
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
class HashableModel(models.Model):
"""
An abstract base class model that provides a fingerprint
field
"""
fingerprint = models.CharField(
max_length=40,
unique=True,
)
class Meta:
abstract = True
def calculate_hash(self):
fingerprint = hashlib.sha1()
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
if field.is_relation:
fingerprint.update(
getattr(self, field_name).fingerprint
)
else:
fingerprint.update(
smart_str(getattr(self, field_name))
)
self.fingerprint = fingerprint.hexdigest()
return self.fingerprint
def get_hash_str(self):
string = ''
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
if field.is_relation:
string += getattr(self, field_name).fingerprint
else:
string += smart_str(getattr(self, field_name))
return string
def save(self, *args, **kwargs):
self.calculate_hash()
super(HashableModel, self).save(*args, **kwargs)
......@@ -19,40 +19,36 @@
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
import uuid
import hashlib
from datetime import datetime
from django.db import models
from django.utils.functional import cached_property
from django.utils.encoding import smart_str
from uuidfield import UUIDField
from memopol_utils.mixins import HashableModel, TimeStampedModel
class Country(models.Model):
name = models.CharField(max_length=255)
code = models.CharField(max_length=2)
@property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
fingerprint.update(smart_str(self.code))
return fingerprint.hexdigest()
def __unicode__(self):
return u'{} [{}]'.format(self.name, self.code)
class Representative(models.Model):
class Representative(HashableModel, TimeStampedModel):
"""
Base model for representatives
"""
# We use a UUIDField instead of an autogenerated integer id
# Thid allow better sync capabilities (primary keys are
# consistents accross various servers).
# We use a namespaced UUID, that means that a same representative
# (same name and same remote_id) will always have the same pk
# https://github.com/dcramer/django-uuidfield
id = UUIDField(
primary_key=True,
version=5
)
slug = models.SlugField(max_length=100)
remote_id = models.CharField(max_length=255, unique=True)
first_name = models.CharField(max_length=255, blank=True, null=True)
......@@ -70,13 +66,7 @@ class Representative(models.Model):
photo = models.CharField(max_length=512, null=True)
active = models.BooleanField(default=False)
def save(self, *args, **kwargs):
if self.id == None:
self.id = uuid.uuid5(
uuid.UUID('6e987b9b-d98b-4c3b-9829-8cd0bea327dd'),
'{}{}'.format(self.slug, self.remote_id)
)
super(Representative, self).save(*args, **kwargs)
hashable_fields = ['remote_id']
def __unicode__(self):
return u'{} ({})'.format(self.full_name.decode('utf-8'), self.remote_id)
......@@ -87,7 +77,7 @@ class Representative(models.Model):
# Contact related models
class Contact(models.Model):
class Contact(TimeStampedModel):
representative = models.ForeignKey(Representative)
class Meta:
......@@ -97,7 +87,7 @@ class Contact(models.Model):
class Email(Contact):
email = models.EmailField()
kind = models.CharField(max_length=255, blank=True, null=True)
class WebSite(Contact):
url = models.CharField(max_length=2048, blank=True, null=True)
......@@ -116,24 +106,34 @@ class Address(Contact):
name = models.CharField(max_length=255, blank=True, null=True)
location = models.CharField(max_length=255, blank=True, null=True) # TODO Find standard for storage in charfield
'''
hashable_fields = ['country', 'city', 'street', 'number',
'postcode', 'floor', 'office_number',
'kind', 'name', 'location', 'representative']
'''
class Phone(Contact):
number = models.CharField(max_length=255, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
address = models.ForeignKey(Address, null=True, related_name='phones')
# Mandate related models
class Group(models.Model):
class Group(TimeStampedModel):
"""
An entity represented by a representative through a mandate
An entity represented by a representative through a mandate
"""
name = models.CharField(max_length=255)
abbreviation = models.CharField(max_length=10, blank=True, null=True)
kind = models.CharField(max_length=255, blank=True, null=True)
@cached_property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
fingerprint.update(smart_str(self.abbreviation))
fingerprint.update(smart_str(self.kind))
return fingerprint.hexdigest()
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
......@@ -142,12 +142,18 @@ class Group(models.Model):
return unicode(self.name)
class Constituency(models.Model):
class Constituency(TimeStampedModel):
"""
An authority for which a representative has a mandate
An authority for which a representative has a mandate
"""
name = models.CharField(max_length=255)
@cached_property
def fingerprint(self):
fingerprint = hashlib.sha1()
fingerprint.update(smart_str(self.name))
return fingerprint.hexdigest()
@cached_property
def active(self):
return self.mandates.filter(end_date__gte=datetime.now()).exists()
......@@ -156,21 +162,23 @@ class Constituency(models.Model):
return unicode(self.name)
class Mandate(models.Model):
class Mandate(HashableModel, TimeStampedModel):
group = models.ForeignKey(Group, null=True, related_name='mandates')
constituency = models.ForeignKey(Constituency, null=True, related_name='mandates')
representative = models.ForeignKey(Representative, related_name='mandates')
role = models.CharField(
max_length=25,
blank=True,
null=True,
default='',
help_text="Eg.: president of a political group at the European Parliament"
)
begin_date = models.DateField(blank=True, null=True)
end_date = models.DateField(blank=True, null=True)
link = models.URLField()
hashable_fields = ['group', 'constituency', 'role',
'begin_date', 'end_date', 'representative']
@property
def active(self):
return self.end_date >= datetime.now().date()
......
......@@ -29,11 +29,13 @@ class CountrySerializer(serializers.ModelSerializer):
model = models.Country
fields = ('name', 'code')
class EmailSerializer(serializers.ModelSerializer):
class Meta:
model = models.Email
fields = ('email', 'kind')
class WebsiteSerializer(serializers.ModelSerializer):</