moves utils to task, sync capabilities

representatives_votes/management/commands/import_dossier_from_toutatis.py

@@ -29,10 +29,11 @@ class Command(BaseCommand):
     
     def add_arguments(self, parser):
         parser.add_argument('--celery', action='store_true', default=False)
+        parser.add_argument('fingerprint')
         
     def handle(self, *args, **options):
-        reference = args[0]
+        fingerprint = options['fingerprint']
         if options['celery']:
-            import_a_dossier_from_toutatis.delay(reference, delay=True)
+            import_a_dossier_from_toutatis.delay(fingerprint, delay=True)
         else:
-            import_a_dossier_from_toutatis(reference, delay=False)
+            import_a_dossier_from_toutatis(fingerprint, delay=False)

representatives_votes/management/commands/import_proposal_from_toutatis.py

@@ -18,28 +18,22 @@
 #
 # Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
 # Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-import json
-from urllib2 import urlopen
 
 from django.core.management.base import BaseCommand
-from django.conf import settings
-
-from representatives_votes.utils import import_a_dossier
+from representatives_votes.tasks import import_a_proposal_from_toutatis
 
 class Command(BaseCommand):
-    def handle(self, *args, **options):
-        proposal_id = args[0]
-
-        toutatis_server = getattr(settings,
-                                  'TOUTATIS_SERVER',
-                                  'http://toutatis.mm.staz.be')
-        proposal_url = '{}/api/proposals/{}'.format(toutatis_server, proposal_id)
-        print('Import proposal from {}'.format(proposal_url))
-        proposal_data = json.load(urlopen(proposal_url))
+    """
+    Command to import a dossier from a toutatis server
+    """
+    
+    def add_arguments(self, parser):
+        parser.add_argument('--celery', action='store_true', default=False)
+        parser.add_argument('fingerprint')
         
-        dossier_url = proposal_data['dossier']
-        dossier_data = json.load(urlopen(dossier_url))
-        # Replace dossier proposals by the one proposal we want
-        dossier_data['proposals'] = [proposal_data]
-
-        import_a_dossier(dossier_data)
+    def handle(self, *args, **options):
+        fingerprint = options['fingerprint']
+        if options['celery']:
+            import_a_proposal_from_toutatis.delay(fingerprint, delay=True)
+        else:
+            import_a_proposal_from_toutatis(fingerprint, delay=False)

representatives_votes/migrations/0001_initial.py

@@ -14,44 +14,47 @@ class Migration(migrations.Migration):
             name='Dossier',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('fingerprint', models.CharField(unique=True, max_length=40)),
                 ('title', models.CharField(max_length=1000)),
                 ('reference', models.CharField(max_length=200)),
-                ('text', models.TextField(blank=True)),
+                ('text', models.TextField(default=b'', blank=True)),
                 ('link', models.URLField()),
             ],
             options={
+                'abstract': False,
             },
-            bases=(models.Model,),
         ),
         migrations.CreateModel(
             name='Proposal',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('fingerprint', models.CharField(unique=True, max_length=40)),
                 ('title', models.CharField(max_length=1000)),
-                ('description', models.TextField(blank=True)),
-                ('reference', models.CharField(max_length=200, blank=True)),
+                ('description', models.TextField(default=b'', blank=True)),
+                ('reference', models.CharField(max_length=200, null=True, blank=True)),
                 ('datetime', models.DateTimeField()),
-                ('kind', models.CharField(max_length=200, blank=True)),
+                ('kind', models.CharField(default=b'', max_length=200, blank=True)),
                 ('total_abstain', models.IntegerField()),
                 ('total_against', models.IntegerField()),
                 ('total_for', models.IntegerField()),
                 ('dossier', models.ForeignKey(related_name='proposals', to='representatives_votes.Dossier')),
             ],
             options={
+                'abstract': False,
             },
-            bases=(models.Model,),
         ),
         migrations.CreateModel(
             name='Vote',
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
-                ('representative_name', models.CharField(max_length=200, blank=True)),
-                ('representative_remote_id', models.CharField(max_length=200, blank=True)),
+                ('representative_name', models.CharField(max_length=200, null=True, blank=True)),
+                ('representative_remote_id', models.CharField(max_length=200, null=True, blank=True)),
                 ('position', models.CharField(max_length=10, choices=[(b'abstain', b'abstain'), (b'for', b'for'), (b'against', b'against')])),
                 ('proposal', models.ForeignKey(related_name='votes', to='representatives_votes.Proposal')),
             ],
-            options={
-            },
-            bases=(models.Model,),
         ),
     ]

representatives_votes/migrations/0002_auto_20150616_1249.py

-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('representatives_votes', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='dossier',
-            name='text',
-            field=models.TextField(default=b'', blank=True),
-            preserve_default=True,
-        ),
-        migrations.AlterField(
-            model_name='proposal',
-            name='description',
-            field=models.TextField(default=b'', blank=True),
-            preserve_default=True,
-        ),
-        migrations.AlterField(
-            model_name='proposal',
-            name='kind',
-            field=models.CharField(default=b'', max_length=200, blank=True),
-            preserve_default=True,
-        ),
-        migrations.AlterField(
-            model_name='proposal',
-            name='reference',
-            field=models.CharField(max_length=200, null=True, blank=True),
-            preserve_default=True,
-        ),
-        migrations.AlterField(
-            model_name='vote',
-            name='representative_name',
-            field=models.CharField(max_length=200, null=True, blank=True),
-            preserve_default=True,
-        ),
-        migrations.AlterField(
-            model_name='vote',
-            name='representative_remote_id',
-            field=models.CharField(max_length=200, null=True, blank=True),
-            preserve_default=True,
-        ),
-    ]

representatives_votes/models.py

@@ -16,11 +16,7 @@
 # License along with django-representatives.
 # If not, see <http://www.gnu.org/licenses/>.
 
-import hashlib
-
 from django.db import models
-from django.utils.functional import cached_property
-from django.utils.encoding import smart_str
 
 from representatives.models import TimeStampedModel, HashableModel
 
@@ -48,6 +44,7 @@ class Proposal(HashableModel, TimeStampedModel):
     total_for = models.IntegerField()
 
     hashable_fields = ['dossier', 'title', 'reference', 'kind']
+
     def __unicode__(self):
         return unicode(self.title)
 

representatives_votes/tasks.py

@@ -25,7 +25,7 @@ import json
 
 from django.conf import settings
 
-# import redis
+import redis
 from celery import shared_task
 from urllib2 import urlopen
 
@@ -33,36 +33,43 @@ from representatives_votes.models import Dossier
 from representatives_votes.serializers import DossierDetailSerializer
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-steam_handler = logging.StreamHandler()
-steam_handler.setLevel(logging.DEBUG)
-logger.addHandler(steam_handler)
 
 @shared_task
 def import_a_dossier(data):
     '''
     Import a dossier from serialized
     '''
-    serializer = DossierDetailSerializer(data=data)
-    if serializer.is_valid():
-        serializer.save()
-    else:
-        print(serializer.errors)
+    with redis.Redis().lock('import_a_dossier'):
+        try:
+            dossier = Dossier.objects.get(
+                fingerprint=data['fingerprint']
+            )
+            serializer = DossierDetailSerializer(
+                instance=dossier,
+                data=data
+            )
+        except:
+            serializer = DossierDetailSerializer(data=data)
+
+        if serializer.is_valid():
+            return serializer.save()
+        else:
+            raise Exception(serializer.errors)           
 
 @shared_task
 def import_a_dossier_from_toutatis(fingerprint, delay=False):
     '''
     Import a complete dossier from a toutatis server
     '''
-
-    toutatis_server = getattr(settings,
-                              'TOUTATIS_SERVER',
-                              'http://toutatis.mm.staz.be')
+    toutatis_server = settings.TOUTATIS_SERVER
     search_url = '{server}/api/dossiers/?fingerprint={fingerprint}'.format({
         'server': toutatis_server,
         'fingerprint': fingerprint
     })
-    logger.info('Import dossier from {}'.format(search_url))
+    logger.info('Import dossier with fingerprint {} from {}'.format(
+        fingerprint,
+        search_url
+    ))
     data = json.load(urlopen(search_url))
     if data['count'] != 1:
         raise Exception('Search should return one and only one result')
@@ -78,12 +85,28 @@ def import_a_proposal_from_toutatis(fingerprint, delay=False):
     '''
     Import a partial dossier from a toutatis server
     '''
-    pass    
-
-def import_dossiers(data):
-    return [import_a_dossier(d_data) for d_data in data]
+    toutatis_server = settings.TOUTATIS_SERVER
+    search_url = '{server}/api/proposals/?fingerprint={fingerprint}'.format({
+        'server': toutatis_server,
+        'fingerprint': fingerprint
+    })
+    logger.info('Import proposal with fingerprint {} from {}'.format(
+        fingerprint,
+        search_url
+    ))
+    data = json.load(urlopen(search_url))
+    if data['count'] != 1:
+        raise Exception('Search should return one and only one result')
+    detail_url = data['results'][0]['url']
+    proposal_data = json.load(urlopen(detail_url))
+    dossier_url = proposal_data['dossier']
+    dossier_data = json.load(urlopen(dossier_url))
+    dossier_data['proposals'] = [proposal_data]
+    if delay:
+        import_a_dossier.delay(dossier_data)
+    else:
+        import_a_dossier(dossier_data)
 
-# Export a dossier
 def export_a_dossier(dossier):
     serialized = DossierDetailSerializer(dossier)
     return serialized.data