Commit d913e263 authored by Jamesie Pic's avatar Jamesie Pic

Added parltrack_import_representatives command

It includes latest research by Arnaud Fabre but I ended up squashing all
commits and adding my own fixups in this commit.

Also:

- the import command is tested,
- code was PEP8'ed
- untested code was removed,
- duplicate country Israel was removed and migration was added,
- requirements were deleted in favor of proper setup.py.
parent c40f57aa
[run]
omit = representatives_votes/tests/*
omit = representatives_votes/migrations/*
sudo: false
language: python
env:
- DJANGO="django>1.8,<1.9" DJANGO_SETTINGS_MODULE=representatives.tests.settings
python:
- "2.7"
- "2.7"
before_install:
- pip install codecov
install:
- pip install django
- pip install -e .
- pip install $DJANGO pep8 flake8 pytest-django pytest-cov codecov
- pip install -e .
script:
- test_project/manage.py migrate
- pep8 representatives/ --exclude migrations --ignore E128
- flake8 representatives/ --exclude migrations --ignore E128
- django-admin migrate
- cat representatives/contrib/parltrack/tests/representatives_fixture.json | parltrack_import_representatives
- py.test
after_success:
- codecov
include *.rst *.txt README LICENSE AUTHORS CHANGELOG
recursive-include representatives *.html *.css *.js *.py *.po *.mo *.json *.png *.gif
[pytest]
DJANGO_SETTINGS_MODULE=representatives.tests.settings
addopts = --cov=representatives --create-db
......@@ -20,7 +20,9 @@
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.contrib import admin
from .models import Representative, Country, Mandate, Group, Constituency, Email, WebSite, Phone, Address
from .models import (Address, Constituency, Country, Email, Group, Mandate,
Phone, Representative, WebSite)
class EmailInline(admin.TabularInline):
......@@ -60,12 +62,21 @@ class RepresentativeAdmin(admin.ModelAdmin):
MandateInline
]
class GroupAdmin(admin.ModelAdmin):
list_display = ('id', 'name', 'abbreviation', 'kind')
list_filter = ('kind',)
class MandateAdmin(admin.ModelAdmin):
list_display = ('id', 'representative', 'group', 'role', 'constituency', 'begin_date', 'end_date')
list_display = (
'id',
'representative',
'group',
'role',
'constituency',
'begin_date',
'end_date')
search_fields = ('representative', 'group', 'constituency')
......
This diff is collapsed.
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.contrib.parltrack import import_representatives
@pytest.mark.django_db
def test_parltrack_import_representatives():
fixture = os.path.join(os.path.dirname(__file__),
'representatives_fixture.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated')
with open(fixture, 'r') as f:
import_representatives.main(f)
missing = []
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for field in exclude:
if field in compare:
compare.pop(field)
try:
type(obj.object).objects.get(**compare)
except:
missing.append(compare)
assert len(missing) is 0
assert Representative.objects.count() == 2
[{
"pk": 12,
"model": "representatives.country",
"fields": {
"code": "IL",
"name": "Israel"
}
}, {
"pk": 1031,
"model": "representatives.country",
"fields": {
......
# coding: utf-8
# This file is part of compotista.
#
# compotista is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# compotista is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with compotista
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2013 Laurent Peuch <cortex@worlddomination.be>
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.core.management.base import BaseCommand
from representatives.tasks import sync_from_compotista
class Command(BaseCommand):
"""
Command to import representative from a compotista server
"""
def add_arguments(self, parser):
parser.add_argument('--celery', action='store_true', default=False)
def handle(self, *args, **options):
if options['celery']:
sync_from_compotista.delay()
else:
sync_from_compotista()
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.db import migrations, models
class Migration(migrations.Migration):
......@@ -13,9 +13,18 @@ class Migration(migrations.Migration):
migrations.CreateModel(
name='Country',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('name', models.CharField(max_length=255)),
('code', models.CharField(max_length=2)),
('id',
models.AutoField(
verbose_name='ID',
serialize=False,
auto_created=True,
primary_key=True)),
('name',
models.CharField(
max_length=255)),
('code',
models.CharField(
max_length=2)),
],
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import os
from django.core import serializers
from django.db import migrations
fixture_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../fixtures'))
fixture_dir = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../fixtures'))
fixture_filename = 'country_initial_data.json'
def load_fixture(apps, schema_editor):
fixture_file = os.path.join(fixture_dir, fixture_filename)
......@@ -19,12 +22,14 @@ def load_fixture(apps, schema_editor):
obj.save()
fixture.close()
def unload_fixture(apps, schema_editor):
"Brutally deleting all entries for this model..."
MyModel = apps.get_model("representatives", "Country")
MyModel.objects.all().delete()
class Migration(migrations.Migration):
dependencies = [
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.db import migrations, models
class Migration(migrations.Migration):
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.db import migrations, models
class Migration(migrations.Migration):
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('representatives', '0004_auto_20150709_1601'),
]
operations = [
migrations.AlterField(
model_name='mandate',
name='role',
field=models.CharField(default=b'', help_text=b'Eg.: president of a political group', max_length=25, blank=True),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
def remove_duplicate(apps, schema_editor):
Country = apps.get_model('representatives', 'country')
Address = apps.get_model('representatives', 'address')
if Country.objects.filter(pk=12, code='IL').count():
Address.objects.filter(country_id=12).update(country_id=1121)
Country.objects.get(pk=12).delete()
class Migration(migrations.Migration):
dependencies = [
('representatives', '0005_auto_20151212_2251'),
]
operations = [
migrations.RunPython(remove_duplicate)
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('representatives', '0006_auto_20151213_0139'),
]
operations = [
migrations.AlterField(
model_name='country',
name='code',
field=models.CharField(unique=True, max_length=2),
),
]
......@@ -23,8 +23,8 @@ import hashlib
from datetime import datetime
from django.db import models
from django.utils.functional import cached_property
from django.utils.encoding import smart_str, smart_unicode
from django.utils.functional import cached_property
class TimeStampedModel(models.Model):
......@@ -32,10 +32,10 @@ class TimeStampedModel(models.Model):
An abstract base class model that provides self-updating
``created`` and ``modified`` fields.
"""
created = models.DateTimeField(auto_now_add=True)
updated = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
......@@ -45,7 +45,7 @@ class HashableModel(models.Model):
An abstract base class model that provides a fingerprint
field
"""
fingerprint = models.CharField(
max_length=40,
unique=True,
......@@ -57,7 +57,7 @@ class HashableModel(models.Model):
def calculate_hash(self):
fingerprint = hashlib.sha1()
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
field = self._meta.get_field(field_name)
if field.is_relation:
fingerprint.update(
getattr(self, field_name).fingerprint
......@@ -72,7 +72,7 @@ class HashableModel(models.Model):
def get_hash_str(self):
string = ''
for field_name in self.hashable_fields:
field = self._meta.get_field(field_name)
field = self._meta.get_field(field_name)
if field.is_relation:
string += getattr(self, field_name).fingerprint
else:
......@@ -86,7 +86,7 @@ class HashableModel(models.Model):
class Country(models.Model):
name = models.CharField(max_length=255)
code = models.CharField(max_length=2)
code = models.CharField(max_length=2, unique=True)
@property
def fingerprint(self):
......@@ -119,8 +119,8 @@ class Representative(HashableModel, TimeStampedModel):
birth_date = models.DateField(blank=True, null=True)
cv = models.TextField(blank=True, default='')
photo = models.CharField(max_length=512, null=True)
active = models.BooleanField(default=False)
active = models.BooleanField(default=False)
hashable_fields = ['remote_id']
def __unicode__(self):
......@@ -132,8 +132,10 @@ class Representative(HashableModel, TimeStampedModel):
class Meta:
ordering = ['last_name', 'first_name']
# Contact related models
class Contact(TimeStampedModel):
representative = models.ForeignKey(Representative)
......@@ -161,14 +163,14 @@ class Address(Contact):
office_number = models.CharField(max_length=255, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
name = models.CharField(max_length=255, blank=True, default='')
location = models.CharField(max_length=255, blank=True, default='')
location = models.CharField(max_length=255, blank=True, default='')
class Phone(Contact):
number = models.CharField(max_length=255, blank=True, default='')
kind = models.CharField(max_length=255, blank=True, default='')
address = models.ForeignKey(Address, null=True, related_name='phones')
class Group(HashableModel, TimeStampedModel):
"""
......@@ -205,21 +207,28 @@ class Constituency(HashableModel, TimeStampedModel):
class MandateManager(models.Manager):
def get_queryset(self):
return super(MandateManager, self).get_queryset().select_related('group', 'constituency')
return super(
MandateManager,
self).get_queryset().select_related(
'group',
'constituency')
class Mandate(HashableModel, TimeStampedModel):
objects = MandateManager()
group = models.ForeignKey(Group, null=True, related_name='mandates')
constituency = models.ForeignKey(Constituency, null=True, related_name='mandates')
constituency = models.ForeignKey(
Constituency, null=True, related_name='mandates')
representative = models.ForeignKey(Representative, related_name='mandates')
role = models.CharField(
max_length=25,
blank=True,
default='',
help_text="Eg.: president of a political group at the European Parliament"
help_text="Eg.: president of a political group"
)
begin_date = models.DateField(blank=True, null=True)
end_date = models.DateField(blank=True, null=True)
......@@ -233,9 +242,11 @@ class Mandate(HashableModel, TimeStampedModel):
return self.end_date >= datetime.now().date()
def __unicode__(self):
return u'Mandate : {representative},{role} {group} for {constituency}'.format(
t = u'Mandate : {representative},{role} {group} for {constituency}'
return t.format(
representative=self.representative,
role=(u' {} of'.format(self.role) if self.role else u''),
role=(
u' {} of'.format(
self.role) if self.role else u''),
constituency=self.constituency,
group=self.group
)
group=self.group)
......@@ -19,34 +19,39 @@
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.db import transaction
from rest_framework import serializers
import representatives.models as models
from rest_framework import serializers
class CountrySerializer(serializers.ModelSerializer):
class Meta:
model = models.Country
fields = ('name', 'code')
class EmailSerializer(serializers.ModelSerializer):
class Meta:
model = models.Email
fields = ('email', 'kind')
class WebsiteSerializer(serializers.ModelSerializer):
class Meta:
model = models.WebSite
fields = ('url', 'kind')
def validate_url(self, value):
# Don’t validate url, because it could break import of not proper formed url
# Don’t validate url, because it could break import of not proper
# formed url
return value
class PhoneSerializer(serializers.ModelSerializer):
class Meta:
model = models.Phone
fields = ('number', 'kind')
......@@ -58,12 +63,13 @@ class PhoneSerializer(serializers.ModelSerializer):
class AddressSerializer(serializers.ModelSerializer):
country = CountrySerializer()
phones = PhoneSerializer(many=True)
class Meta:
model = models.Address
fields = ('country', 'city', 'street',
'number', 'postcode', 'floor',
'office_number', 'kind', 'phones',
)
)
class ContactField(serializers.Serializer):
......@@ -82,21 +88,24 @@ class ContactField(serializers.Serializer):
class ConstituencySerializer(serializers.ModelSerializer):
class Meta:
model = models.Constituency
fields = ('id', 'name', 'fingerprint')
class GroupSerializer(serializers.ModelSerializer):
class Meta:
model = models.Group
fields = ('id', 'name', 'abbreviation', 'kind', 'fingerprint')
class MandateSerializer(serializers.ModelSerializer):
# name = serializers.CharField(source='group.name')
# short_id = serializers.CharField(source='group.abbreviation', allow_blank=True)
# short_id = serializers.CharField(
# source='group.abbreviation', allow_blank=True)
# kind = serializers.CharField(source='group.kind')
# constituency = serializers.CharField(source='constituency.name')
......@@ -104,7 +113,7 @@ class MandateSerializer(serializers.ModelSerializer):
source='group.fingerprint',
)
constituency = serializers.CharField(
source='constituency.fingerprint'
source='constituency.fingerprint'
)
representative = serializers.CharField(
source='representative.fingerprint'
......@@ -156,7 +165,7 @@ class MandateDetailSerializer(MandateSerializer):
class RepresentativeSerializer(serializers.ModelSerializer):
contacts = ContactField()
class Meta:
model = models.Representative
fields = (
......@@ -177,7 +186,6 @@ class RepresentativeSerializer(serializers.ModelSerializer):
'url',
)
@transaction.atomic
def create(self, validated_data):
contacts_data = validated_data.pop('contacts')
......@@ -234,7 +242,7 @@ class RepresentativeSerializer(serializers.ModelSerializer):
class RepresentativeDetailSerializer(RepresentativeSerializer):
mandates = MandateDetailSerializer(many=True)
class Meta(RepresentativeSerializer.Meta):
......
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from __future__ import absolute_import
import logging
import json
from django.conf import settings
from django.utils import timezone
from celery import shared_task
from urllib2 import urlopen
from representatives.models import Representative, Group, Constituency, Mandate, Address, Phone, Email, WebSite
from representatives.serializers import GroupSerializer, ConstituencySerializer, RepresentativeSerializer, MandateSerializer, RepresentativeDetailSerializer
logger = logging.getLogger(__name__)
def import_a_model(data, model, serializer, skip_old=False):
logging.info('Importing data: %s', data)
try:
instance = model.objects.get(
fingerprint=data['fingerprint']
)
if skip_old:
# Update 'updated' field
return instance.save()
serializer_instance = serializer(
instance=instance,
data=data
)
except model.DoesNotExist:
serializer_instance = serializer(
data=data
)
if serializer_instance.is_valid():
return serializer_instance.save()
else:
raise Exception(serializer_instance.errors)
@shared_task
def sync_from_compotista():
limit = 100
compotista_server = settings.COMPOTISTA_SERVER
import_start_datetime = timezone.now()
models_to_import = [{
'url': compotista_server + '/api/groups',
'model': Group,
'serializer': GroupSerializer
}, {
'url': compotista_server + '/api/constituencies',
'model': Constituency,
'serializer': ConstituencySerializer
}, {
'url': compotista_server + '/api/representatives',
'model': Representative,
'serializer': RepresentativeSerializer
}, {
'url': compotista_server + '/api/mandates',
'model': Mandate,
'serializer': MandateSerializer
}]
# url = compotista_server + '/'
# res = urlopen(url)