models.py 5.82 KB
Newer Older
okhin's avatar
okhin committed
1
2
3
4
5
from csv import DictReader
import json
import requests
import re

okhin's avatar
okhin committed
6
7
8
from django.db import models
from django.utils.translation import ugettext_lazy as _

okhin's avatar
okhin committed
9
from picampaign.campaign.models import Campaign
okhin's avatar
okhin committed
10
from picampaign.organization.models import Group, GroupType, Organization
okhin's avatar
okhin committed
11

okhin's avatar
okhin committed
12
13
14
15
16
17
18
class Importer(models.Model):
    """Importer model. Used to populate campaign with contacts"""
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=64)
    data_format = models.CharField(max_length=4, choices=[('JSON', 'json'), ('CSV', 'csv')])
    file = models.FileField(upload_to='imports/', blank=True, null=True)
    url = models.URLField(blank=True, null=True)
okhin's avatar
okhin committed
19
    campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
okhin's avatar
okhin committed
20
21
22
    organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
    last_count = models.IntegerField(null=True, blank=True)
    last_imported = models.IntegerField(null=True, blank=True)
okhin's avatar
okhin committed
23
24

    def __str__(self):
okhin's avatar
okhin committed
25
        return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format,
26
                                                'type': self.kind(),
okhin's avatar
okhin committed
27
                                                'name': self.name}
28
29
30
31
32
33
34
35

    def kind(self):
        if self.url is None and self.file is None:
            return None
        if self.url is None:
            return 'file'
        else:
            return 'url'
okhin's avatar
okhin committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

    def handle(self):
        """
        This function is used to import the data described by the Importer.

        We need to parse the provided file, using an appropriate deserilizer/parser,
        and then to create or update contacts.

        The operation should end providing the number of items imported and the number of items submitted.

        Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
        holding a list of groups (mandates) of a specific kind, column separated

        For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
        by columns.
        """
        import_data = []
        # First let's check what kind of importer are we running.
        # If it's a file, let's parse it according to our format.
        if self.kind == 'file':
            with open(self.file, u'rU') as f:
                if self.data_format == 'CSV':
                    # If we're a csv format, let's use DictReader on a file object
                    reader = DictReader(f)
                    re_mandate = re.compile(r'^mandate:(\w+)')
                    for item in reader:
                        # We need to create the groups according to the header
                        for key in item.keys():
                            if re.match(re_mandate, key):
                                # We have a mandate
                                for mandate in item[key].split(':'):
                                    item['mandates'].append({'kind': key.split(':')[0], 'name': mandate})
                                item.delete(key)
                    import_data.append(item)
                else:
                    # We're in json, let's load from a file
                    import_data = json.load(f)
        # We now have all the data ready to be imported.
        # We can count the data available
        self.last_count = len(import_data)

        # Lets' go through the meps, for fun and profit
        inserted = 0
        for import_mep in import_data:
            # We want to know if all mandates exists
            if not 'birthdate' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
                # We do not have enough field to discriminate our meps
                continue
            updated_mep = {}
            updated_mep['groups'] = []
            for mandate in import_mep['mandates']:
                if mandate['end_date'] != '':
                    # This mandate has ended
                    continue
                groupType, added = GroupType.objects.get_or_create(organization = self.organization,
                        name = mandate['kind'])
                group, added = Group.objects.get_or_create(name = mandate['name'],
                        type = groupType)
                updated_mep['groups'].append(group)

            # If we have a contact item, we can go through it. Otherwise we probably have
            # phone, twitter, etc fields.
            if 'contacts' in import_mep:
                # Let's get the first phone
                if import_mep['contacts']['phones'] != []:
                    updated_emp['phone'] = mep['contacts']['phones'][0]['phone']
                if import_mep['contacts']['emails'] != []:
                    updated_mep['mail'] = mep['contacts']['phones'][0]['phone']
                if import_mep['contacts']['websites'] != []:
                    for website in import_mep['contacts']['websites']:
                        if website['kind'] == 'twitter':
                            updated_mep['twitter'] = ''.join(['@', website['website'].split('/')[-1]])
                            break
            else:
                # We have phone, mail, twitter fields
                if 'phone' in import_mep:
                    updated_mep['phone'] = import_mep['phone']
                if 'mail' in import_mep:
                    updated_mep['mail'] = import_mep['phone']
                if 'twitter' in import_mep:
                    updated_mep['twitter'] = import_mep['twitter']

            # All groups are done
            # Let's update_or_create the mep
            mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
                        last_name=import_mep['last_name'],
                        birthdate=import_mep['birthdate'],
                        defaults=updated_mep
                    )
            inserted += 1
        self.last_imported = inserted
        self.save()