models.py 8.83 KB
Newer Older
okhin's avatar
okhin committed
1 2
from csv import DictReader
import json
okhin's avatar
okhin committed
3

okhin's avatar
okhin committed
4

okhin's avatar
okhin committed
5 6 7
from django.db import models
from django.utils.translation import ugettext_lazy as _

okhin's avatar
okhin committed
8 9

from picampaign.contact.models import Contact
10
from picampaign.campaign.models import Campaign, CampaignContact
11
from picampaign.organization.models import Group, GroupType
okhin's avatar
okhin committed
12

okhin's avatar
okhin committed
13

okhin's avatar
okhin committed
14 15 16 17
class Importer(models.Model):
    """Importer model. Used to populate campaign with contacts"""
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=64)
okhin's avatar
okhin committed
18 19
    category = models.CharField(max_length=6, choices=(('update', _('CSV Updater')),
                                                       ('repr', _("Representative format"))))
okhin's avatar
okhin committed
20 21
    file = models.FileField(upload_to='imports/', blank=True, null=True)
    url = models.URLField(blank=True, null=True)
okhin's avatar
okhin committed
22
    campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
okhin's avatar
okhin committed
23 24
    last_count = models.IntegerField(null=True, blank=True)
    last_imported = models.IntegerField(null=True, blank=True)
okhin's avatar
okhin committed
25 26

    def __str__(self):
27
        return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.category,
28
                                                'type': self.kind(),
okhin's avatar
okhin committed
29
                                                'name': self.name}
30 31

    def kind(self):
okhin's avatar
okhin committed
32
        if self.url is None and self.file.name is None:
33
            return None
okhin's avatar
okhin committed
34
        if self.url is not '':
35
            return 'url'
okhin's avatar
okhin committed
36 37
        else:
            return 'file'
okhin's avatar
okhin committed
38

39
    def representative(self, import_data):
okhin's avatar
okhin committed
40
        """
41 42
        This handle all the representatives data import. The imported_data have all the contacts in
        the representative format (and in a list).
okhin's avatar
okhin committed
43 44 45 46 47 48
        """
        # We now have all the data ready to be imported.
        # We can count the data available
        self.last_count = len(import_data)

        inserted = 0
49
        for import_contact in import_data:
okhin's avatar
okhin committed
50
            # We want to know if all mandates exists
okhin's avatar
okhin committed
51
            if 'birth_date' not in import_contact or 'first_name' not in import_contact or 'last_name' not in import_contact:
52
                # We do not have enough field to discriminate our contacts
okhin's avatar
okhin committed
53
                continue
54
            updated_contact = {}
55
            groups = []
56
            for mandate in import_contact['mandates']:
okhin's avatar
okhin committed
57 58
                # Parltrack gives us a date starting with 9999 if ends has not happens
                # FranceData gives us an empty one
59 60
                end_date = mandate.get('end_date', None)
                if end_date is None:
okhin's avatar
okhin committed
61
                    continue
62 63 64 65
                else:
                    if not (end_date == '' or end_date.startswith('9999')):
                        # This mandate has ended
                        continue
okhin's avatar
okhin committed
66
                # We want to only use the groups own by the organisation
okhin's avatar
okhin committed
67 68
                for groupType in GroupType.objects.filter(organization=self.campaign.organization,
                        name=mandate['group']['kind']):
okhin's avatar
okhin committed
69
                    if 'abbreviation' in mandate['group']:
okhin's avatar
okhin committed
70 71
                        group, added = Group.objects.get_or_create(name=mandate['group']['abbreviation'],
                                type=groupType)
okhin's avatar
okhin committed
72
                    else:
okhin's avatar
okhin committed
73 74
                        group, added=Group.objects.get_or_create(name=mandate['group']['name'],
                                type=groupType)
okhin's avatar
okhin committed
75
                    groups.append(group)
76
            if groups == []:
77
                # The contact have no groups active, he is not to be bothered.
78
                continue
okhin's avatar
okhin committed
79

okhin's avatar
okhin committed
80
            # All groups are done
okhin's avatar
okhin committed
81 82
            # If we have a contact item, we can go through it. Otherwise we probably have
            # phone, twitter, etc fields.
83
            phones = []
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
            # Let's get the phones
            if import_contact['contacts']['phones'] != []:
                for phone in import_contact['contacts']['phones']:
                    if phone['kind'] == "office phone":
                        phones.append(phone['number'])

            # These are the email adresses used
            if import_contact['contacts']['emails'] != []:
                updated_contact['mail'] = import_contact['contacts']['emails'][0]['email']

            # We're trying to find the twitter accounts
            if import_contact['contacts']['websites'] != []:
                for website in import_contact['contacts']['websites']:
                    if website['kind'] == 'twitter':
                        updated_contact['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
                        break

101 102 103 104
            # Let's retrieve the photo URL
            if 'photo' in import_contact:
                updated_contact['photo'] = import_contact['photo']

105 106
            # Let's update_or_create the contact
            contact, updated = Contact.objects.update_or_create(first_name=import_contact['first_name'],
okhin's avatar
okhin committed
107 108 109
                                                                last_name=import_contact['last_name'],
                                                                birthdate=import_contact['birth_date'],
                                                                defaults=updated_contact)
110 111 112

            # We want to create the phones
            for phone in phones:
113
                contact.phones.get_or_create(phone=phone)
114

115
            # Let's link the MEP to the group
116 117 118
            contact.groups = groups
            contact.save()
            # We want to link the contact to our current campaign
okhin's avatar
okhin committed
119
            campaigncontact, created = CampaignContact.objects.update_or_create(campaign=self.campaign, contact=contact)
okhin's avatar
okhin committed
120
            inserted += 1
121

122
        # Updating the last imported item
okhin's avatar
okhin committed
123 124
        self.last_imported = inserted
        self.save()
okhin's avatar
okhin committed
125

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    def handle(self):
        """
        This function is used to import the data described by the Importer.

        We need to parse the provided file, using an appropriate deserilizer/parser,
        and then to create or update contacts.

        The operation should end providing the number of items imported and the number of items submitted.

        Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
        holding a list of groups (mandates) of a specific kind, column separated

        For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
        by columns.
        """
        import_data = []
        # First let's check what kind of importer are we running.
        # If it's a file, let's parse it according to our format.
        if self.kind() == 'file':
            with open(self.file.path, u'rU') as f:
146
                if self.category == 'updater':
147 148 149
                    # If we're a csv format, let's use DictReader on a file object
                    reader = DictReader(f)
                    for item in reader:
okhin's avatar
okhin committed
150 151 152 153 154 155 156 157 158
                        # FIXME: We need to create the groups according to the header
                        # Which means for each mandate:kind columsn we must add a item['mandates']
                        # entry with an end-date of ''
                        # One can belong to different groups
                        mandates = []
                        parsed_item = {}
                        for key in [k for k in item if k.startswith('mandates:')]:
                            for group in item[key].split(','):
                                mandates.append({'group':
okhin's avatar
okhin committed
159 160 161
                                        {'kind': key.split(':')[1],
                                        'name': group},
                                    'end_date': ''})
okhin's avatar
okhin committed
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
                        # Same for phones and websites
                        contacts = {}
                        contacts['phones'] = []
                        for key in [k for k in item if k.startswith('phones:')]:
                            contacts['phones'].append({'kind': 'office phone',
                                'number': item[key]})
                        contacts['emails'] = []
                        for key in [k for k in item if k.startswith('emails:')]:
                            contacts['emails'].append({'kind': key.split(':')[1],
                                'email': item[key]})
                        contacts['websites'] = []
                        for key in [k for k in item if k == 'twitter']:
                            contacts['websites'].append({'url': item[key],
                                'kind': 'twitter'})

                        parsed_item['first_name'] = item['first_name']
                        parsed_item['last_name'] = item['last_name']
                        parsed_item['birth_date'] = item['birth_date']
                        parsed_item['mandates'] = mandates
                        parsed_item['contacts'] = contacts
                        import_data.append(parsed_item)
183 184 185 186
                else:
                    # We're in json, let's load from a file
                    import_data = json.load(f)
        # Now, let's get the correct call
okhin's avatar
okhin committed
187
        self.representative(import_data)
188
        return