From 1361a192d60c809026c19ec05f88bfc8203eeb8e Mon Sep 17 00:00:00 2001
From: Arnaud Fabre <arnaud.fabre@camobscura.fr>
Date: Wed, 1 Jul 2015 11:46:28 +0200
Subject: [PATCH] moves utils to task, sync capabilities
---
.../commands/import_dossier_from_toutatis.py | 7 ++-
.../commands/import_proposal_from_toutatis.py | 34 +++++-----
.../migrations/0001_initial.py | 25 ++++----
.../migrations/0002_auto_20150616_1249.py | 50 ---------------
representatives_votes/models.py | 5 +-
representatives_votes/tasks.py | 63 +++++++++++++------
6 files changed, 76 insertions(+), 108 deletions(-)
delete mode 100644 representatives_votes/migrations/0002_auto_20150616_1249.py
diff --git a/representatives_votes/management/commands/import_dossier_from_toutatis.py b/representatives_votes/management/commands/import_dossier_from_toutatis.py
index dcf3e0a..de8076a 100644
--- a/representatives_votes/management/commands/import_dossier_from_toutatis.py
+++ b/representatives_votes/management/commands/import_dossier_from_toutatis.py
@@ -29,10 +29,11 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('--celery', action='store_true', default=False)
+ parser.add_argument('fingerprint')
def handle(self, *args, **options):
- reference = args[0]
+ fingerprint = options['fingerprint']
if options['celery']:
- import_a_dossier_from_toutatis.delay(reference, delay=True)
+ import_a_dossier_from_toutatis.delay(fingerprint, delay=True)
else:
- import_a_dossier_from_toutatis(reference, delay=False)
+ import_a_dossier_from_toutatis(fingerprint, delay=False)
diff --git a/representatives_votes/management/commands/import_proposal_from_toutatis.py b/representatives_votes/management/commands/import_proposal_from_toutatis.py
index 78c6c18..7e2966b 100644
--- a/representatives_votes/management/commands/import_proposal_from_toutatis.py
+++ b/representatives_votes/management/commands/import_proposal_from_toutatis.py
@@ -18,28 +18,22 @@
#
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-import json
-from urllib2 import urlopen
from django.core.management.base import BaseCommand
-from django.conf import settings
-
-from representatives_votes.utils import import_a_dossier
+from representatives_votes.tasks import import_a_proposal_from_toutatis
class Command(BaseCommand):
- def handle(self, *args, **options):
- proposal_id = args[0]
-
- toutatis_server = getattr(settings,
- 'TOUTATIS_SERVER',
- 'http://toutatis.mm.staz.be')
- proposal_url = '{}/api/proposals/{}'.format(toutatis_server, proposal_id)
- print('Import proposal from {}'.format(proposal_url))
- proposal_data = json.load(urlopen(proposal_url))
+ """
+ Command to import a dossier from a toutatis server
+ """
+
+ def add_arguments(self, parser):
+ parser.add_argument('--celery', action='store_true', default=False)
+ parser.add_argument('fingerprint')
- dossier_url = proposal_data['dossier']
- dossier_data = json.load(urlopen(dossier_url))
- # Replace dossier proposals by the one proposal we want
- dossier_data['proposals'] = [proposal_data]
-
- import_a_dossier(dossier_data)
+ def handle(self, *args, **options):
+ fingerprint = options['fingerprint']
+ if options['celery']:
+ import_a_proposal_from_toutatis.delay(fingerprint, delay=True)
+ else:
+ import_a_proposal_from_toutatis(fingerprint, delay=False)
diff --git a/representatives_votes/migrations/0001_initial.py b/representatives_votes/migrations/0001_initial.py
index b14e686..22a3618 100644
--- a/representatives_votes/migrations/0001_initial.py
+++ b/representatives_votes/migrations/0001_initial.py
@@ -14,44 +14,47 @@ class Migration(migrations.Migration):
name='Dossier',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('fingerprint', models.CharField(unique=True, max_length=40)),
('title', models.CharField(max_length=1000)),
('reference', models.CharField(max_length=200)),
- ('text', models.TextField(blank=True)),
+ ('text', models.TextField(default=b'', blank=True)),
('link', models.URLField()),
],
options={
+ 'abstract': False,
},
- bases=(models.Model,),
),
migrations.CreateModel(
name='Proposal',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+ ('created', models.DateTimeField(auto_now_add=True)),
+ ('updated', models.DateTimeField(auto_now=True)),
+ ('fingerprint', models.CharField(unique=True, max_length=40)),
('title', models.CharField(max_length=1000)),
- ('description', models.TextField(blank=True)),
- ('reference', models.CharField(max_length=200, blank=True)),
+ ('description', models.TextField(default=b'', blank=True)),
+ ('reference', models.CharField(max_length=200, null=True, blank=True)),
('datetime', models.DateTimeField()),
- ('kind', models.CharField(max_length=200, blank=True)),
+ ('kind', models.CharField(default=b'', max_length=200, blank=True)),
('total_abstain', models.IntegerField()),
('total_against', models.IntegerField()),
('total_for', models.IntegerField()),
('dossier', models.ForeignKey(related_name='proposals', to='representatives_votes.Dossier')),
],
options={
+ 'abstract': False,
},
- bases=(models.Model,),
),
migrations.CreateModel(
name='Vote',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
- ('representative_name', models.CharField(max_length=200, blank=True)),
- ('representative_remote_id', models.CharField(max_length=200, blank=True)),
+ ('representative_name', models.CharField(max_length=200, null=True, blank=True)),
+ ('representative_remote_id', models.CharField(max_length=200, null=True, blank=True)),
('position', models.CharField(max_length=10, choices=[(b'abstain', b'abstain'), (b'for', b'for'), (b'against', b'against')])),
('proposal', models.ForeignKey(related_name='votes', to='representatives_votes.Proposal')),
],
- options={
- },
- bases=(models.Model,),
),
]
diff --git a/representatives_votes/migrations/0002_auto_20150616_1249.py b/representatives_votes/migrations/0002_auto_20150616_1249.py
deleted file mode 100644
index af0e50f..0000000
--- a/representatives_votes/migrations/0002_auto_20150616_1249.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ('representatives_votes', '0001_initial'),
- ]
-
- operations = [
- migrations.AlterField(
- model_name='dossier',
- name='text',
- field=models.TextField(default=b'', blank=True),
- preserve_default=True,
- ),
- migrations.AlterField(
- model_name='proposal',
- name='description',
- field=models.TextField(default=b'', blank=True),
- preserve_default=True,
- ),
- migrations.AlterField(
- model_name='proposal',
- name='kind',
- field=models.CharField(default=b'', max_length=200, blank=True),
- preserve_default=True,
- ),
- migrations.AlterField(
- model_name='proposal',
- name='reference',
- field=models.CharField(max_length=200, null=True, blank=True),
- preserve_default=True,
- ),
- migrations.AlterField(
- model_name='vote',
- name='representative_name',
- field=models.CharField(max_length=200, null=True, blank=True),
- preserve_default=True,
- ),
- migrations.AlterField(
- model_name='vote',
- name='representative_remote_id',
- field=models.CharField(max_length=200, null=True, blank=True),
- preserve_default=True,
- ),
- ]
diff --git a/representatives_votes/models.py b/representatives_votes/models.py
index f3b64b2..0bfe1ce 100644
--- a/representatives_votes/models.py
+++ b/representatives_votes/models.py
@@ -16,11 +16,7 @@
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
-import hashlib
-
from django.db import models
-from django.utils.functional import cached_property
-from django.utils.encoding import smart_str
from representatives.models import TimeStampedModel, HashableModel
@@ -48,6 +44,7 @@ class Proposal(HashableModel, TimeStampedModel):
total_for = models.IntegerField()
hashable_fields = ['dossier', 'title', 'reference', 'kind']
+
def __unicode__(self):
return unicode(self.title)
diff --git a/representatives_votes/tasks.py b/representatives_votes/tasks.py
index a0aff2b..b62b1fa 100644
--- a/representatives_votes/tasks.py
+++ b/representatives_votes/tasks.py
@@ -25,7 +25,7 @@ import json
from django.conf import settings
-# import redis
+import redis
from celery import shared_task
from urllib2 import urlopen
@@ -33,36 +33,43 @@ from representatives_votes.models import Dossier
from representatives_votes.serializers import DossierDetailSerializer
logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-steam_handler = logging.StreamHandler()
-steam_handler.setLevel(logging.DEBUG)
-logger.addHandler(steam_handler)
@shared_task
def import_a_dossier(data):
'''
Import a dossier from serialized
'''
- serializer = DossierDetailSerializer(data=data)
- if serializer.is_valid():
- serializer.save()
- else:
- print(serializer.errors)
+ with redis.Redis().lock('import_a_dossier'):
+ try:
+ dossier = Dossier.objects.get(
+ fingerprint=data['fingerprint']
+ )
+ serializer = DossierDetailSerializer(
+ instance=dossier,
+ data=data
+ )
+ except:
+ serializer = DossierDetailSerializer(data=data)
+
+ if serializer.is_valid():
+ return serializer.save()
+ else:
+ raise Exception(serializer.errors)
@shared_task
def import_a_dossier_from_toutatis(fingerprint, delay=False):
'''
Import a complete dossier from a toutatis server
'''
-
- toutatis_server = getattr(settings,
- 'TOUTATIS_SERVER',
- 'http://toutatis.mm.staz.be')
+ toutatis_server = settings.TOUTATIS_SERVER
search_url = '{server}/api/dossiers/?fingerprint={fingerprint}'.format({
'server': toutatis_server,
'fingerprint': fingerprint
})
- logger.info('Import dossier from {}'.format(search_url))
+ logger.info('Import dossier with fingerprint {} from {}'.format(
+ fingerprint,
+ search_url
+ ))
data = json.load(urlopen(search_url))
if data['count'] != 1:
raise Exception('Search should return one and only one result')
@@ -78,12 +85,28 @@ def import_a_proposal_from_toutatis(fingerprint, delay=False):
'''
Import a partial dossier from a toutatis server
'''
- pass
-
-def import_dossiers(data):
- return [import_a_dossier(d_data) for d_data in data]
+ toutatis_server = settings.TOUTATIS_SERVER
+ search_url = '{server}/api/proposals/?fingerprint={fingerprint}'.format({
+ 'server': toutatis_server,
+ 'fingerprint': fingerprint
+ })
+ logger.info('Import proposal with fingerprint {} from {}'.format(
+ fingerprint,
+ search_url
+ ))
+ data = json.load(urlopen(search_url))
+ if data['count'] != 1:
+ raise Exception('Search should return one and only one result')
+ detail_url = data['results'][0]['url']
+ proposal_data = json.load(urlopen(detail_url))
+ dossier_url = proposal_data['dossier']
+ dossier_data = json.load(urlopen(dossier_url))
+ dossier_data['proposals'] = [proposal_data]
+ if delay:
+ import_a_dossier.delay(dossier_data)
+ else:
+ import_a_dossier(dossier_data)
-# Export a dossier
def export_a_dossier(dossier):
serialized = DossierDetailSerializer(dossier)
return serialized.data
--
GitLab