Skip to content

GitLab

C
campaign
Commit 66600852 authored by okhin's avatar okhin
Browse files
Options

WIP #5: Representative importer done, along with the new category field inthe model

parent e77669eb
Pipeline #102 failed with stage
    in 50 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 101 additions and 76 deletions
    picampaign/importer/admin.py
    View file @ 66600852
    ......@@ -4,7 +4,7 @@ from django.utils.translation import ugettext_lazy as _
    from picampaign.importer.models import Importer
    class ImporterAdmin(admin.ModelAdmin):
    list_display = ['name', 'kind', 'data_format', 'file', 'url']
    list_display = ['name', 'kind', 'category']
    actions = ['import_action']
    def import_action(self, request, queryset):
    ......
    picampaign/importer/migrations/0002_auto_20160809_1442.py 0 → 100644
    View file @ 66600852
    # -*- coding: utf-8 -*-
    # Generated by Django 1.9.6 on 2016-08-09 12:42
    from __future__ import unicode_literals
    from django.db import migrations, models
    class Migration(migrations.Migration):
    dependencies = [
    ('importer', '0001_initial'),
    ]
    operations = [
    migrations.RemoveField(
    model_name='importer',
    name='data_format',
    ),
    migrations.AddField(
    model_name='importer',
    name='category',
    field=models.CharField(choices=[('update', 'CSV Updater'), ('repr', 'Representative format')], default='repr', max_length=6),
    preserve_default=False,
    ),
    ]
    picampaign/importer/models.py
    View file @ 66600852
    ......@@ -14,7 +14,8 @@ class Importer(models.Model):
    """Importer model. Used to populate campaign with contacts"""
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=64)
    data_format = models.CharField(max_length=4, choices=[('JSON', 'json'), ('CSV', 'csv')])
    category = models.CharField(max_length=6, choices=(('update', _('CSV Updater'))
    , ('repr', _("Representative format"))))
    file = models.FileField(upload_to='imports/', blank=True, null=True)
    url = models.URLField(blank=True, null=True)
    campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
    ......@@ -23,7 +24,7 @@ class Importer(models.Model):
    last_imported = models.IntegerField(null=True, blank=True)
    def __str__(self):
    return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format,
    return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.category,
    'type': self.kind(),
    'name': self.name}
    ......@@ -35,56 +36,24 @@ class Importer(models.Model):
    else:
    return 'url'
    def handle(self):
    def representative(self, imported_data):
    """
    This function is used to import the data described by the Importer.
    We need to parse the provided file, using an appropriate deserilizer/parser,
    and then to create or update contacts.
    The operation should end providing the number of items imported and the number of items submitted.
    Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
    holding a list of groups (mandates) of a specific kind, column separated
    For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
    by columns.
    This handle all the representatives data import. The imported_data have all the contacts in
    the representative format (and in a list).
    """
    import_data = []
    # First let's check what kind of importer are we running.
    # If it's a file, let's parse it according to our format.
    if self.kind() == 'file':
    with open(self.file.path, u'rU') as f:
    if self.data_format == 'CSV':
    # If we're a csv format, let's use DictReader on a file object
    reader = DictReader(f)
    re_mandate = re.compile(r'^mandate:(\w+)')
    for item in reader:
    # We need to create the groups according to the header
    for key in item.keys():
    if re.match(re_mandate, key):
    # We have a mandate
    for mandate in item[key].split(':'):
    item['mandates'].append({'group': {'kind': key.split(':')[0], 'name': mandate}})
    item.delete(key)
    import_data.append(item)
    else:
    # We're in json, let's load from a file
    import_data = json.load(f)
    # We now have all the data ready to be imported.
    # We can count the data available
    self.last_count = len(import_data)
    # Lets' go through the meps, for fun and profit
    inserted = 0
    for import_mep in import_data:
    for import_contact in import_data:
    # We want to know if all mandates exists
    if not 'birth_date' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
    # We do not have enough field to discriminate our meps
    if not 'birth_date' in import_contact or not 'first_name' in import_contact or not 'last_name' in import_contact:
    # We do not have enough field to discriminate our contacts
    continue
    updated_mep = {}
    updated_contact = {}
    groups = []
    for mandate in import_mep['mandates']:
    for mandate in import_contact['mandates']:
    # Parltrack gives us a date starting with 9999 if ends has not happens
    # FranceData gives us an empty one
    end_date = mandate.get('end_date', None)
    ......@@ -108,52 +77,83 @@ class Importer(models.Model):
    type = groupType)
    groups.append(group)
    if groups == []:
    # The mep have no groups active, he is not to be bothered.
    # The contact have no groups active, he is not to be bothered.
    continue
    # All groups are done
    # If we have a contact item, we can go through it. Otherwise we probably have
    # phone, twitter, etc fields.
    phones = []
    if 'contacts' in import_mep:
    # Let's get the first phone
    if import_mep['contacts']['phones'] != []:
    for phone in import_mep['contacts']['phones']:
    if phone['kind'] == "office phone":
    phones.append(phone['number'])
    if import_mep['contacts']['emails'] != []:
    updated_mep['mail'] = import_mep['contacts']['emails'][0]['email']
    if import_mep['contacts']['websites'] != []:
    for website in import_mep['contacts']['websites']:
    if website['kind'] == 'twitter':
    updated_mep['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
    break
    else:
    # We have phone, mail, twitter fields
    if 'phone' in import_mep:
    phones.append(import_mep['phone'])
    if 'mail' in import_mep:
    updated_mep['mail'] = import_mep['phone']
    if 'twitter' in import_mep:
    updated_mep['twitter'] = import_mep['twitter']
    # Let's update_or_create the mep
    mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
    last_name=import_mep['last_name'],
    birthdate=import_mep['birth_date'],
    defaults=updated_mep
    # Let's get the phones
    if import_contact['contacts']['phones'] != []:
    for phone in import_contact['contacts']['phones']:
    if phone['kind'] == "office phone":
    phones.append(phone['number'])
    # These are the email adresses used
    if import_contact['contacts']['emails'] != []:
    updated_contact['mail'] = import_contact['contacts']['emails'][0]['email']
    # We're trying to find the twitter accounts
    if import_contact['contacts']['websites'] != []:
    for website in import_contact['contacts']['websites']:
    if website['kind'] == 'twitter':
    updated_contact['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
    break
    # Let's update_or_create the contact
    contact, updated = Contact.objects.update_or_create(first_name=import_contact['first_name'],
    last_name=import_contact['last_name'],
    birthdate=import_contact['birth_date'],
    defaults=updated_contact
    )
    # We want to create the phones
    for phone in phones:
    mep.phones.get_or_create(phone=phone)
    contact.phones.get_or_create(phone=phone)
    # Let's link the MEP to the group
    mep.groups = groups
    mep.save()
    # We want to link the mep to our current campaign
    CampaignContact(campaign=self.campaign, contact=mep).save()
    contact.groups = groups
    contact.save()
    # We want to link the contact to our current campaign
    CampaignContact(campaign=self.campaign, contact=contact).save()
    inserted += 1
    # Updating the last imported item
    self.last_imported = inserted
    self.save()
    def handle(self):
    """
    This function is used to import the data described by the Importer.
    We need to parse the provided file, using an appropriate deserilizer/parser,
    and then to create or update contacts.
    The operation should end providing the number of items imported and the number of items submitted.
    Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
    holding a list of groups (mandates) of a specific kind, column separated
    For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
    by columns.
    """
    import_data = []
    # First let's check what kind of importer are we running.
    # If it's a file, let's parse it according to our format.
    if self.kind() == 'file':
    with open(self.file.path, u'rU') as f:
    if self.data_format == 'CSV':
    # If we're a csv format, let's use DictReader on a file object
    reader = DictReader(f)
    for item in reader:
    # We need to create the groups according to the header
    import_data.append(item)
    else:
    # We're in json, let's load from a file
    import_data = json.load(f)
    # Now, let's get the correct call
    if self.category == 'repr':
    self.representative(import_data)
    return
    Markdown is supported
    0% or .
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment