Commit c07c5b19 authored by luxcem's avatar luxcem

idempotent import (big words)

parent 0c951424
......@@ -19,24 +19,23 @@
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.core.management.base import BaseCommand
from django.conf import settings
from urllib2 import urlopen
from django.utils.six import BytesIO
from rest_framework.parsers import JSONParser
import ijson
from representatives.utils import import_representatives
from representatives.utils import import_a_representative
class Command(BaseCommand):
def handle(self, *args, **options):
compotista_server = getattr(settings,
'REPRESENTATIVES_COMPOTISTA_SERVER',
'COMPOTISTA_SERVER',
'http://compotista.mm.staz.be')
url = compotista_server + "/latest/"
url = compotista_server + '/export/latest/'
print('Import representatives from %s' % url)
stream = BytesIO(urlopen(url))
import_representatives(JSONParser().parse(stream))
resource = urlopen(url)
for representative in ijson.items(resource, 'item'):
import_a_representative(representative)
......@@ -23,8 +23,7 @@ import sys
import json
from django.core.management.base import BaseCommand
from representatives.utils import import_representatives_from_format
class Command(BaseCommand):
def handle(self, *args, **options):
import_representatives_from_format(json.load(sys.stdin))
# TODO
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('representatives', '0010_auto_20150527_1401'),
]
operations = [
migrations.AlterField(
model_name='mandate',
name='representative',
field=models.ForeignKey(related_name='mandates', to='representatives.Representative'),
preserve_default=True,
),
migrations.AlterField(
model_name='phone',
name='address',
field=models.ForeignKey(related_name='phones', to='representatives.Address', null=True),
preserve_default=True,
),
]
......@@ -92,7 +92,7 @@ class Address(Contact):
class Phone(Contact):
number = models.CharField(max_length=255)
kind = models.CharField(max_length=255, blank=True, null=True)
address = models.ForeignKey(Address, null=True)
address = models.ForeignKey(Address, null=True, related_name='phones')
# Mandate related models
......@@ -123,7 +123,7 @@ class Constituency(models.Model):
class Mandate(models.Model):
group = models.ForeignKey(Group, null=True)
constituency = models.ForeignKey(Constituency, null=True)
representative = models.ForeignKey(Representative)
representative = models.ForeignKey(Representative, related_name='mandates')
role = models.CharField(
max_length=25,
blank=True,
......
......@@ -38,14 +38,23 @@ class WebsiteSerializer(serializers.ModelSerializer):
model = models.WebSite
fields = ('url', 'kind')
def validate_url(self, value):
'''
Don’t validate url, because it could break import of not proper formed url
'''
return value
class PhoneSerializer(serializers.ModelSerializer):
class Meta:
model = models.Phone
fields = ('number', 'kind')
def validate_phone(self, value):
return value
class AddressSerializer(serializers.ModelSerializer):
country = CountrySerializer()
phones = PhoneSerializer(many=True, source='phone_set')
phones = PhoneSerializer(many=True)
class Meta:
model = models.Address
fields = ('country', 'city', 'street', 'number', 'postcode', 'floor', 'office_number', 'kind', 'phones')
......@@ -123,53 +132,51 @@ class RepresentativeHyperLinkedSerializer(RepresentativeSerializer):
class RepresentativeDetailSerializer(RepresentativeSerializer):
contacts = ContactField()
mandates = RepresentativeMandateSerializer(many=True, source='mandate_set')
mandates = RepresentativeMandateSerializer(many=True)
class Meta(RepresentativeSerializer.Meta):
fields = RepresentativeSerializer.Meta.fields + (
'cv',
'contacts',
'mandates'
)
# Nested creation is not implemented yet in DRF, it sucks
# We made an intensive use of get_or_create to avoid recreating objects
# every import
# TODO : fix this code when it will be implemented
@transaction.atomic
def create(self, validated_data):
contacts_data = validated_data.pop('contacts')
mandates_data = validated_data.pop('mandate_set')
representative = models.Representative.objects.create(**validated_data)
mandates_data = validated_data.pop('mandates')
representative, _ = models.Representative.objects.get_or_create(**validated_data)
self._create_mandates(mandates_data, representative)
self._create_contacts(contacts_data, representative)
return representative
def _create_contacts(self, contacts_data, representative):
for contact_data in contacts_data['emails']:
contact = models.Email(**contact_data)
contact.representative = representative
contact.save()
contact_data['representative'] = representative
contact, _ = models.Email.objects.get_or_create(**contact_data)
for contact_data in contacts_data['websites']:
contact = models.WebSite(**contact_data)
contact.representative = representative
contact.save()
contact_data['representative'] = representative
contact, _ = models.WebSite.objects.get_or_create(**contact_data)
for contact_data in contacts_data['address']:
country, _ = models.Country.objects.get_or_create(
**contact_data.pop('country')
)
phone_set = contact_data.pop('phone_set')
contact = models.Address(**contact_data)
contact.country = country
contact.representative = representative
contact.save()
phone_set = contact_data.pop('phones')
contact_data['representative'] = representative
contact_data['country'] = country
contact, _ = models.Address.objects.get_or_create(**contact_data)
for phone_data in phone_set:
phone = models.Phone(**phone_data)
phone.address = contact
phone.representative = representative
phone.save()
phone_data['representative'] = representative
phone_data['address'] = contact
phone, _ = models.Phone.objects.get_or_create(**phone_data)
def _create_mandates(self, mandates_data, representative):
for mandate_data in mandates_data:
......@@ -179,8 +186,7 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
group, _ = models.Group.objects.get_or_create(
**mandate_data.pop('group')
)
mandate = models.Mandate(**mandate_data)
mandate.representative = representative
mandate.constituency = constituency
mandate.group = group
mandate.save()
mandate_data['representative'] = representative
mandate_data['constituency'] = constituency
mandate_data['group'] = group
mandate, _ = models.Mandate.objects.get_or_create(**mandate_data)
......@@ -25,8 +25,11 @@ from representatives.serializers import RepresentativeDetailSerializer
# Import a representative
def import_a_representative(data):
serializer = RepresentativeDetailSerializer(data=data)
serializer.is_valid()
return serializer.save()
if serializer.is_valid():
representative = serializer.save()
print(representative)
return representative
print(serializer.errors)
def import_representatives(data):
return [import_a_representative(r_data) for r_data in data]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment