update representatives

representatives/management/commands/import_representatives_from_compotista.py

@@ -33,10 +33,10 @@ class Command(BaseCommand):
     """
 
     def add_arguments(self, parser):
-        parser.add_argument('--parallel', action='store_true', default=False)
+        parser.add_argument('--celery', action='store_true', default=False)
 
     def handle(self, *args, **options):
-        if not options['parallel']:
-            import_representatives_from_compotista(options['parallel'])
+        if options['celery']:
+            import_representatives_from_compotista.delay(delay=True)
         else:
-            import_representatives_from_compotista.delay(options['parallel'])
+            import_representatives_from_compotista(delay=False)

representatives/serializers.py

@@ -45,7 +45,7 @@ class WebsiteSerializer(serializers.ModelSerializer):
         '''
         Don’t validate url, because it could break import of not proper formed url
         '''
-        return value 
+        return value
 
 
 class PhoneSerializer(serializers.ModelSerializer):
@@ -73,7 +73,7 @@ class ContactField(serializers.Serializer):
     phones = PhoneSerializer(many=True)
     websites = WebsiteSerializer(many=True)
     address = AddressSerializer(many=True)
-    
+
     def get_attribute(self, obj):
         return {
             'emails': obj.email_set.all(),
@@ -85,13 +85,10 @@ class ContactField(serializers.Serializer):
 
 class MandateSerializer(serializers.ModelSerializer):
     name = serializers.CharField(source='group.name')
-    short_id = serializers.CharField(source='group.abbreviation', allow_null=True)
+    short_id = serializers.CharField(source='group.abbreviation', allow_blank=True)
     kind = serializers.CharField(source='group.kind')
     constituency = serializers.CharField(source='constituency.name')
 
-    # def validate_fingerprint(self, value):
-        # return value
-
     class Meta:
         model = models.Mandate
         fields = (
@@ -116,6 +113,12 @@ class MandateHyperLinkedSerializer(MandateSerializer):
 
 class RepresentativeMandateSerializer(MandateSerializer):
     class Meta(MandateSerializer.Meta):
+        extra_kwargs = {
+            'fingerprint': {
+                'validators': [],
+            },
+        }
+
         fields = [elem for elem in MandateSerializer.Meta.fields if elem != 'representative']
 
 
@@ -146,6 +149,7 @@ class RepresentativeHyperLinkedSerializer(RepresentativeSerializer):
 class RepresentativeDetailSerializer(RepresentativeSerializer):
     contacts = ContactField()
     mandates = RepresentativeMandateSerializer(many=True)
+
     class Meta(RepresentativeSerializer.Meta):
         fields = RepresentativeSerializer.Meta.fields + (
             'cv',
@@ -153,7 +157,6 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
             'mandates'
         )
 
-
     @transaction.atomic
     def create(self, validated_data):
         """
@@ -173,11 +176,12 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
         self._create_contacts(contacts_data, representative)
         return representative
 
+    @transaction.atomic
     def update(self, instance, validated_data):
         contacts_data = validated_data.pop('contacts')
         mandates_data = validated_data.pop('mandates')
-        
-        for attr, value in validated_data.iteritems(): 
+
+        for attr, value in validated_data.iteritems():
             setattr(instance, attr, value)
         instance.save()
 
@@ -187,17 +191,17 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
 
     def touch_model(self, model, **data):
         '''
-        This method create or look up a model with the given data 
-        it saves the given model if it exists, updating its 
+        This method create or look up a model with the given data
+        it saves the given model if it exists, updating its
         updated field
-        '''        
+        '''
         instance, created = model.objects.get_or_create(**data)
-        
+
         if not created:
             instance.save()
 
         return (instance, created)
-    
+
     def _create_contacts(self, contacts_data, representative):
         for contact_data in contacts_data['emails']:
             contact_data['representative'] = representative
@@ -222,7 +226,7 @@ class RepresentativeDetailSerializer(RepresentativeSerializer):
                 self.touch_model(model=models.Phone, **phone_data)
 
     def _create_mandates(self, mandates_data, representative):
-        for mandate_data in mandates_data:            
+        for mandate_data in mandates_data:
             # serializer = MandateSerializer(data=mandate_data)
             constituency, _ = self.touch_model(model=models.Constituency,
                 **mandate_data.pop('constituency')

representatives/tasks.py

@@ -18,19 +18,25 @@
 #
 # Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
 
-
 from __future__ import absolute_import
+import logging
 
 from django.conf import settings
 from django.utils import timezone
 
 import ijson
+import redis
 from celery import shared_task
 from urllib2 import urlopen
 
 from representatives.models import Representative, Group, Constituency, Mandate, Address, Phone, Email, WebSite
 from representatives.serializers import RepresentativeDetailSerializer
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+steam_handler = logging.StreamHandler()
+steam_handler.setLevel(logging.DEBUG)
+logger.addHandler(steam_handler)
 
 @shared_task
 def import_a_representative(data, verbose=False):
@@ -38,39 +44,44 @@ def import_a_representative(data, verbose=False):
     Import a representative from a serialized
     Python datatypes
     '''
-
-    try:
-        representative = Representative.objects.get(
-            fingerprint=data['fingerprint']
-        )
-        serializer = RepresentativeDetailSerializer(
-            instance=representative,
-            data=data
-        )
-    except:
-        serializer = RepresentativeDetailSerializer(data=data)
-
-    if serializer.is_valid():
-        return serializer.save()
-    else:
-        # print(data)
-        raise Exception(serializer.errors)
+    
+    # We use a lock to import only one representative at a time
+    # Avoid to deal with parallel aspect of importation
+    with redis.Redis().lock('import_a_representative'):
+        try:
+            representative = Representative.objects.get(
+                fingerprint=data['fingerprint']
+            )
+            serializer = RepresentativeDetailSerializer(
+                instance=representative,
+                data=data
+            )
+        except:
+            serializer = RepresentativeDetailSerializer(data=data)
+
+        if serializer.is_valid():
+            return serializer.save()
+        else:
+            raise Exception(serializer.errors)
 
 
 @shared_task
-def import_representatives_from_compotista(delay=False):    
+def import_representatives_from_compotista(delay=False):
     compotista_server = getattr(settings,
                                      'COMPOTISTA_SERVER',
                                      'http://compotista.mm.staz.be')
     import_start_datetime = timezone.now()
     url = compotista_server + '/export/latest/'
     res = urlopen(url)
-    for representative in ijson.items(res, 'item'):
+
+    logger.info(u'Import representative from {}'.format(url))
+    for i, representative in enumerate(ijson.items(res, 'item')):
+        logger.info(u'{}. Import representative {}'.format(i, representative['full_name']))
         if delay:
             representative = import_a_representative.delay(representative)
         else:
             representative = import_a_representative(representative)
-        
+
     for model in [Representative, Group, Constituency,
                   Mandate, Address, Phone, Email, WebSite]:
         model.objects.filter(updated__lt=import_start_datetime).delete()