models.py 6.21 KB
Newer Older
okhin's avatar
okhin committed
1 2 3 4 5
from csv import DictReader
import json
import requests
import re

okhin's avatar
okhin committed
6 7 8
from django.db import models
from django.utils.translation import ugettext_lazy as _

okhin's avatar
okhin committed
9
from picampaign.contact.models import Contact
okhin's avatar
okhin committed
10
from picampaign.campaign.models import Campaign
okhin's avatar
okhin committed
11
from picampaign.organization.models import Group, GroupType, Organization
okhin's avatar
okhin committed
12

okhin's avatar
okhin committed
13 14 15 16 17 18 19
class Importer(models.Model):
    """Importer model. Used to populate campaign with contacts"""
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=64)
    data_format = models.CharField(max_length=4, choices=[('JSON', 'json'), ('CSV', 'csv')])
    file = models.FileField(upload_to='imports/', blank=True, null=True)
    url = models.URLField(blank=True, null=True)
okhin's avatar
okhin committed
20
    campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
okhin's avatar
okhin committed
21 22 23
    organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
    last_count = models.IntegerField(null=True, blank=True)
    last_imported = models.IntegerField(null=True, blank=True)
okhin's avatar
okhin committed
24 25

    def __str__(self):
okhin's avatar
okhin committed
26
        return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format,
27
                                                'type': self.kind(),
okhin's avatar
okhin committed
28
                                                'name': self.name}
29 30

    def kind(self):
okhin's avatar
okhin committed
31
        if self.url is '' and self.file is None:
32
            return None
okhin's avatar
okhin committed
33
        if self.url is '':
34 35 36
            return 'file'
        else:
            return 'url'
okhin's avatar
okhin committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55

    def handle(self):
        """
        This function is used to import the data described by the Importer.

        We need to parse the provided file, using an appropriate deserilizer/parser,
        and then to create or update contacts.

        The operation should end providing the number of items imported and the number of items submitted.

        Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
        holding a list of groups (mandates) of a specific kind, column separated

        For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
        by columns.
        """
        import_data = []
        # First let's check what kind of importer are we running.
        # If it's a file, let's parse it according to our format.
okhin's avatar
okhin committed
56 57
        if self.kind() == 'file':
            with open(self.file.path, u'rU') as f:
okhin's avatar
okhin committed
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
                if self.data_format == 'CSV':
                    # If we're a csv format, let's use DictReader on a file object
                    reader = DictReader(f)
                    re_mandate = re.compile(r'^mandate:(\w+)')
                    for item in reader:
                        # We need to create the groups according to the header
                        for key in item.keys():
                            if re.match(re_mandate, key):
                                # We have a mandate
                                for mandate in item[key].split(':'):
                                    item['mandates'].append({'kind': key.split(':')[0], 'name': mandate})
                                item.delete(key)
                    import_data.append(item)
                else:
                    # We're in json, let's load from a file
                    import_data = json.load(f)
        # We now have all the data ready to be imported.
        # We can count the data available
        self.last_count = len(import_data)

        # Lets' go through the meps, for fun and profit
        inserted = 0
        for import_mep in import_data:
            # We want to know if all mandates exists
okhin's avatar
okhin committed
82
            if not 'birth_date' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
okhin's avatar
okhin committed
83 84 85 86 87
                # We do not have enough field to discriminate our meps
                continue
            updated_mep = {}
            updated_mep['groups'] = []
            for mandate in import_mep['mandates']:
okhin's avatar
okhin committed
88 89 90
                # Parltrack gives us a date starting with 9999 if ends has not happens
                # FranceData gives us an empty one
                if 'end_date' in mandate and (mandate['end_date'] != '' or mandate['end_date'].startswith('9999')):
okhin's avatar
okhin committed
91 92 93 94 95 96 97
                    # This mandate has ended
                    continue
                groupType, added = GroupType.objects.get_or_create(organization = self.organization,
                        name = mandate['kind'])
                group, added = Group.objects.get_or_create(name = mandate['name'],
                        type = groupType)
                updated_mep['groups'].append(group)
okhin's avatar
okhin committed
98 99
            if updated_mep['groups'] == []:
                del updated_mep['groups']
okhin's avatar
okhin committed
100

okhin's avatar
okhin committed
101
            # All groups are done
okhin's avatar
okhin committed
102 103 104 105 106
            # If we have a contact item, we can go through it. Otherwise we probably have
            # phone, twitter, etc fields.
            if 'contacts' in import_mep:
                # Let's get the first phone
                if import_mep['contacts']['phones'] != []:
okhin's avatar
okhin committed
107
                    updated_mep['phone'] = import_mep['contacts']['phones'][0]['number']
okhin's avatar
okhin committed
108
                if import_mep['contacts']['emails'] != []:
okhin's avatar
okhin committed
109
                    updated_mep['mail'] = import_mep['contacts']['emails'][0]['email']
okhin's avatar
okhin committed
110 111 112
                if import_mep['contacts']['websites'] != []:
                    for website in import_mep['contacts']['websites']:
                        if website['kind'] == 'twitter':
okhin's avatar
okhin committed
113
                            updated_mep['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
okhin's avatar
okhin committed
114 115 116 117 118 119 120 121 122 123
                            break
            else:
                # We have phone, mail, twitter fields
                if 'phone' in import_mep:
                    updated_mep['phone'] = import_mep['phone']
                if 'mail' in import_mep:
                    updated_mep['mail'] = import_mep['phone']
                if 'twitter' in import_mep:
                    updated_mep['twitter'] = import_mep['twitter']

okhin's avatar
okhin committed
124
            #import ipdb; ipdb.set_trace()
okhin's avatar
okhin committed
125 126 127
            # Let's update_or_create the mep
            mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
                        last_name=import_mep['last_name'],
okhin's avatar
okhin committed
128
                        birthdate=import_mep['birth_date'],
okhin's avatar
okhin committed
129 130 131 132 133
                        defaults=updated_mep
                    )
            inserted += 1
        self.last_imported = inserted
        self.save()