Commit ff7abd46 authored by okhin's avatar okhin
Browse files

FIrst run of the importer

parent 80367ef2
from django.contrib import admin from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from picampaign.importer.models import Importer from picampaign.importer.models import Importer
class ImporterAdmin(admin.ModelAdmin): class ImporterAdmin(admin.ModelAdmin):
list_display = ['name', 'kind', 'data_format', 'file', 'url'] list_display = ['name', 'kind', 'data_format', 'file', 'url']
actions = ['import_action'], ImporterAdmin) def import_action(self, request, queryset):
We're going to launch the import of data
for obj in queryset:
self.message_user(request, _("Imported %(last_imported)s of %(last_imported)s contacts") % {
'last_imported': obj.last_imported,
'last_count': obj.last_count})
import_action.short_description = _("Run the selected importers"), ImporterAdmin)
from csv import DictReader
import json
import requests
import re
from django.db import models from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from picampaign.campaign.models import Campaign from picampaign.campaign.models import Campaign
from picampaign.organization.models import Group, GroupType, Organization
class Importer(models.Model): class Importer(models.Model):
"""Importer model. Used to populate campaign with contacts""" """Importer model. Used to populate campaign with contacts"""
...@@ -11,11 +17,14 @@ class Importer(models.Model): ...@@ -11,11 +17,14 @@ class Importer(models.Model):
file = models.FileField(upload_to='imports/', blank=True, null=True) file = models.FileField(upload_to='imports/', blank=True, null=True)
url = models.URLField(blank=True, null=True) url = models.URLField(blank=True, null=True)
campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True) campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
last_count = models.IntegerField(null=True, blank=True)
last_imported = models.IntegerField(null=True, blank=True)
def __str__(self): def __str__(self):
return _('%{name}: %{format} %{type} importer' % {'format': self.data_format, return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format,
'type': self.kind(), 'type': self.kind(),
'name':}) 'name':}
def kind(self): def kind(self):
if self.url is None and self.file is None: if self.url is None and self.file is None:
...@@ -24,3 +33,95 @@ class Importer(models.Model): ...@@ -24,3 +33,95 @@ class Importer(models.Model):
return 'file' return 'file'
else: else:
return 'url' return 'url'
def handle(self):
This function is used to import the data described by the Importer.
We need to parse the provided file, using an appropriate deserilizer/parser,
and then to create or update contacts.
The operation should end providing the number of items imported and the number of items submitted.
Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
holding a list of groups (mandates) of a specific kind, column separated
For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
by columns.
import_data = []
# First let's check what kind of importer are we running.
# If it's a file, let's parse it according to our format.
if self.kind == 'file':
with open(self.file, u'rU') as f:
if self.data_format == 'CSV':
# If we're a csv format, let's use DictReader on a file object
reader = DictReader(f)
re_mandate = re.compile(r'^mandate:(\w+)')
for item in reader:
# We need to create the groups according to the header
for key in item.keys():
if re.match(re_mandate, key):
# We have a mandate
for mandate in item[key].split(':'):
item['mandates'].append({'kind': key.split(':')[0], 'name': mandate})
# We're in json, let's load from a file
import_data = json.load(f)
# We now have all the data ready to be imported.
# We can count the data available
self.last_count = len(import_data)
# Lets' go through the meps, for fun and profit
inserted = 0
for import_mep in import_data:
# We want to know if all mandates exists
if not 'birthdate' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
# We do not have enough field to discriminate our meps
updated_mep = {}
updated_mep['groups'] = []
for mandate in import_mep['mandates']:
if mandate['end_date'] != '':
# This mandate has ended
groupType, added = GroupType.objects.get_or_create(organization = self.organization,
name = mandate['kind'])
group, added = Group.objects.get_or_create(name = mandate['name'],
type = groupType)
# If we have a contact item, we can go through it. Otherwise we probably have
# phone, twitter, etc fields.
if 'contacts' in import_mep:
# Let's get the first phone
if import_mep['contacts']['phones'] != []:
updated_emp['phone'] = mep['contacts']['phones'][0]['phone']
if import_mep['contacts']['emails'] != []:
updated_mep['mail'] = mep['contacts']['phones'][0]['phone']
if import_mep['contacts']['websites'] != []:
for website in import_mep['contacts']['websites']:
if website['kind'] == 'twitter':
updated_mep['twitter'] = ''.join(['@', website['website'].split('/')[-1]])
# We have phone, mail, twitter fields
if 'phone' in import_mep:
updated_mep['phone'] = import_mep['phone']
if 'mail' in import_mep:
updated_mep['mail'] = import_mep['phone']
if 'twitter' in import_mep:
updated_mep['twitter'] = import_mep['twitter']
# All groups are done
# Let's update_or_create the mep
mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
inserted += 1
self.last_imported = inserted
...@@ -9,6 +9,7 @@ requires = [ ...@@ -9,6 +9,7 @@ requires = [
'django', 'django',
'djangorestframework', 'djangorestframework',
'drf-nested-routers', 'drf-nested-routers',
'pillow', 'pillow',
] ]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment