Commit 66600852 authored by okhin's avatar okhin

WIP #5: Representative importer done, along with the new category field inthe model

parent e77669eb
Pipeline #102 failed with stage
in 50 seconds
...@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _ ...@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _
from picampaign.importer.models import Importer from picampaign.importer.models import Importer
class ImporterAdmin(admin.ModelAdmin): class ImporterAdmin(admin.ModelAdmin):
list_display = ['name', 'kind', 'data_format', 'file', 'url'] list_display = ['name', 'kind', 'category']
actions = ['import_action'] actions = ['import_action']
def import_action(self, request, queryset): def import_action(self, request, queryset):
......
# -*- coding: utf-8 -*-
# Generated by Django 1.9.6 on 2016-08-09 12:42
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('importer', '0001_initial'),
]
operations = [
migrations.RemoveField(
model_name='importer',
name='data_format',
),
migrations.AddField(
model_name='importer',
name='category',
field=models.CharField(choices=[('update', 'CSV Updater'), ('repr', 'Representative format')], default='repr', max_length=6),
preserve_default=False,
),
]
...@@ -14,7 +14,8 @@ class Importer(models.Model): ...@@ -14,7 +14,8 @@ class Importer(models.Model):
"""Importer model. Used to populate campaign with contacts""" """Importer model. Used to populate campaign with contacts"""
id = models.AutoField(primary_key=True) id = models.AutoField(primary_key=True)
name = models.CharField(max_length=64) name = models.CharField(max_length=64)
data_format = models.CharField(max_length=4, choices=[('JSON', 'json'), ('CSV', 'csv')]) category = models.CharField(max_length=6, choices=(('update', _('CSV Updater'))
, ('repr', _("Representative format"))))
file = models.FileField(upload_to='imports/', blank=True, null=True) file = models.FileField(upload_to='imports/', blank=True, null=True)
url = models.URLField(blank=True, null=True) url = models.URLField(blank=True, null=True)
campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True) campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
...@@ -23,7 +24,7 @@ class Importer(models.Model): ...@@ -23,7 +24,7 @@ class Importer(models.Model):
last_imported = models.IntegerField(null=True, blank=True) last_imported = models.IntegerField(null=True, blank=True)
def __str__(self): def __str__(self):
return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format, return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.category,
'type': self.kind(), 'type': self.kind(),
'name': self.name} 'name': self.name}
...@@ -35,56 +36,24 @@ class Importer(models.Model): ...@@ -35,56 +36,24 @@ class Importer(models.Model):
else: else:
return 'url' return 'url'
def handle(self): def representative(self, imported_data):
""" """
This function is used to import the data described by the Importer. This handle all the representatives data import. The imported_data have all the contacts in
the representative format (and in a list).
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
""" """
import_data = []
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if self.kind() == 'file':
with open(self.file.path, u'rU') as f:
if self.data_format == 'CSV':
# If we're a csv format, let's use DictReader on a file object
reader = DictReader(f)
re_mandate = re.compile(r'^mandate:(\w+)')
for item in reader:
# We need to create the groups according to the header
for key in item.keys():
if re.match(re_mandate, key):
# We have a mandate
for mandate in item[key].split(':'):
item['mandates'].append({'group': {'kind': key.split(':')[0], 'name': mandate}})
item.delete(key)
import_data.append(item)
else:
# We're in json, let's load from a file
import_data = json.load(f)
# We now have all the data ready to be imported. # We now have all the data ready to be imported.
# We can count the data available # We can count the data available
self.last_count = len(import_data) self.last_count = len(import_data)
# Lets' go through the meps, for fun and profit
inserted = 0 inserted = 0
for import_mep in import_data: for import_contact in import_data:
# We want to know if all mandates exists # We want to know if all mandates exists
if not 'birth_date' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep: if not 'birth_date' in import_contact or not 'first_name' in import_contact or not 'last_name' in import_contact:
# We do not have enough field to discriminate our meps # We do not have enough field to discriminate our contacts
continue continue
updated_mep = {} updated_contact = {}
groups = [] groups = []
for mandate in import_mep['mandates']: for mandate in import_contact['mandates']:
# Parltrack gives us a date starting with 9999 if ends has not happens # Parltrack gives us a date starting with 9999 if ends has not happens
# FranceData gives us an empty one # FranceData gives us an empty one
end_date = mandate.get('end_date', None) end_date = mandate.get('end_date', None)
...@@ -108,52 +77,83 @@ class Importer(models.Model): ...@@ -108,52 +77,83 @@ class Importer(models.Model):
type = groupType) type = groupType)
groups.append(group) groups.append(group)
if groups == []: if groups == []:
# The mep have no groups active, he is not to be bothered. # The contact have no groups active, he is not to be bothered.
continue continue
# All groups are done # All groups are done
# If we have a contact item, we can go through it. Otherwise we probably have # If we have a contact item, we can go through it. Otherwise we probably have
# phone, twitter, etc fields. # phone, twitter, etc fields.
phones = [] phones = []
if 'contacts' in import_mep: # Let's get the phones
# Let's get the first phone if import_contact['contacts']['phones'] != []:
if import_mep['contacts']['phones'] != []: for phone in import_contact['contacts']['phones']:
for phone in import_mep['contacts']['phones']: if phone['kind'] == "office phone":
if phone['kind'] == "office phone": phones.append(phone['number'])
phones.append(phone['number'])
if import_mep['contacts']['emails'] != []: # These are the email adresses used
updated_mep['mail'] = import_mep['contacts']['emails'][0]['email'] if import_contact['contacts']['emails'] != []:
if import_mep['contacts']['websites'] != []: updated_contact['mail'] = import_contact['contacts']['emails'][0]['email']
for website in import_mep['contacts']['websites']:
if website['kind'] == 'twitter': # We're trying to find the twitter accounts
updated_mep['twitter'] = ''.join(['@', website['url'].split('/')[-1]]) if import_contact['contacts']['websites'] != []:
break for website in import_contact['contacts']['websites']:
else: if website['kind'] == 'twitter':
# We have phone, mail, twitter fields updated_contact['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
if 'phone' in import_mep: break
phones.append(import_mep['phone'])
if 'mail' in import_mep: # Let's update_or_create the contact
updated_mep['mail'] = import_mep['phone'] contact, updated = Contact.objects.update_or_create(first_name=import_contact['first_name'],
if 'twitter' in import_mep: last_name=import_contact['last_name'],
updated_mep['twitter'] = import_mep['twitter'] birthdate=import_contact['birth_date'],
defaults=updated_contact
# Let's update_or_create the mep
mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
last_name=import_mep['last_name'],
birthdate=import_mep['birth_date'],
defaults=updated_mep
) )
# We want to create the phones # We want to create the phones
for phone in phones: for phone in phones:
mep.phones.get_or_create(phone=phone) contact.phones.get_or_create(phone=phone)
# Let's link the MEP to the group # Let's link the MEP to the group
mep.groups = groups contact.groups = groups
mep.save() contact.save()
# We want to link the mep to our current campaign # We want to link the contact to our current campaign
CampaignContact(campaign=self.campaign, contact=mep).save() CampaignContact(campaign=self.campaign, contact=contact).save()
inserted += 1 inserted += 1
# Updating the last imported item
self.last_imported = inserted self.last_imported = inserted
self.save() self.save()
def handle(self):
"""
This function is used to import the data described by the Importer.
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
"""
import_data = []
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if self.kind() == 'file':
with open(self.file.path, u'rU') as f:
if self.data_format == 'CSV':
# If we're a csv format, let's use DictReader on a file object
reader = DictReader(f)
for item in reader:
# We need to create the groups according to the header
import_data.append(item)
else:
# We're in json, let's load from a file
import_data = json.load(f)
# Now, let's get the correct call
if self.category == 'repr':
self.representative(import_data)
return
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment