models.py 8.76 KB
Newer Older
okhin's avatar
okhin committed
1 2 3 4 5
from csv import DictReader
import json
import requests
import re

okhin's avatar
okhin committed
6 7 8
from django.db import models
from django.utils.translation import ugettext_lazy as _

9 10
from picampaign.contact.models import Contact, Phone
from picampaign.campaign.models import Campaign, CampaignContact
11
from picampaign.organization.models import Group, GroupType
okhin's avatar
okhin committed
12

okhin's avatar
okhin committed
13 14 15 16
class Importer(models.Model):
    """Importer model. Used to populate campaign with contacts"""
    id = models.AutoField(primary_key=True)
    name = models.CharField(max_length=64)
17 18
    category = models.CharField(max_length=6, choices=(('update', _('CSV Updater'))
        , ('repr', _("Representative format"))))
okhin's avatar
okhin committed
19 20
    file = models.FileField(upload_to='imports/', blank=True, null=True)
    url = models.URLField(blank=True, null=True)
okhin's avatar
okhin committed
21
    campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
okhin's avatar
okhin committed
22 23
    last_count = models.IntegerField(null=True, blank=True)
    last_imported = models.IntegerField(null=True, blank=True)
okhin's avatar
okhin committed
24 25

    def __str__(self):
26
        return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.category,
27
                                                'type': self.kind(),
okhin's avatar
okhin committed
28
                                                'name': self.name}
29 30

    def kind(self):
okhin's avatar
okhin committed
31
        if self.url is None and self.file.name is None:
32
            return None
okhin's avatar
okhin committed
33
        if self.url is None:
34 35 36
            return 'file'
        else:
            return 'url'
okhin's avatar
okhin committed
37

38
    def representative(self, import_data):
okhin's avatar
okhin committed
39
        """
40 41
        This handle all the representatives data import. The imported_data have all the contacts in
        the representative format (and in a list).
okhin's avatar
okhin committed
42 43 44 45 46 47
        """
        # We now have all the data ready to be imported.
        # We can count the data available
        self.last_count = len(import_data)

        inserted = 0
48
        for import_contact in import_data:
okhin's avatar
okhin committed
49
            # We want to know if all mandates exists
50 51
            if not 'birth_date' in import_contact or not 'first_name' in import_contact or not 'last_name' in import_contact:
                # We do not have enough field to discriminate our contacts
okhin's avatar
okhin committed
52
                continue
53
            updated_contact = {}
54
            groups = []
55
            for mandate in import_contact['mandates']:
okhin's avatar
okhin committed
56 57
                # Parltrack gives us a date starting with 9999 if ends has not happens
                # FranceData gives us an empty one
58 59
                end_date = mandate.get('end_date', None)
                if end_date is None:
okhin's avatar
okhin committed
60
                    continue
61 62 63 64
                else:
                    if not (end_date == '' or end_date.startswith('9999')):
                        # This mandate has ended
                        continue
okhin's avatar
okhin committed
65
                # We want to only use the groups own by the organisation
66
                for groupType in GroupType.objects.filter(organization = self.campaign.organization,
okhin's avatar
okhin committed
67 68 69 70 71 72 73 74
                        name = mandate['group']['kind']):
                    if 'abbreviation' in mandate['group']:
                        group, added = Group.objects.get_or_create(name = mandate['group']['abbreviation'],
                                type = groupType)
                    else:
                        group, added = Group.objects.get_or_create(name = mandate['group']['name'],
                                type = groupType)
                    groups.append(group)
75
            if groups == []:
76
                # The contact have no groups active, he is not to be bothered.
77
                continue
okhin's avatar
okhin committed
78

okhin's avatar
okhin committed
79
            # All groups are done
okhin's avatar
okhin committed
80 81
            # If we have a contact item, we can go through it. Otherwise we probably have
            # phone, twitter, etc fields.
82
            phones = []
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
            # Let's get the phones
            if import_contact['contacts']['phones'] != []:
                for phone in import_contact['contacts']['phones']:
                    if phone['kind'] == "office phone":
                        phones.append(phone['number'])

            # These are the email adresses used
            if import_contact['contacts']['emails'] != []:
                updated_contact['mail'] = import_contact['contacts']['emails'][0]['email']

            # We're trying to find the twitter accounts
            if import_contact['contacts']['websites'] != []:
                for website in import_contact['contacts']['websites']:
                    if website['kind'] == 'twitter':
                        updated_contact['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
                        break

okhin's avatar
okhin committed
100 101 102 103
            # Let's retrieve the photo URL
            if 'photo' in import_contact:
                updated_contact['photo'] = import_contact['photo']

104 105 106 107 108
            # Let's update_or_create the contact
            contact, updated = Contact.objects.update_or_create(first_name=import_contact['first_name'],
                        last_name=import_contact['last_name'],
                        birthdate=import_contact['birth_date'],
                        defaults=updated_contact
okhin's avatar
okhin committed
109
                    )
110 111 112

            # We want to create the phones
            for phone in phones:
113
                contact.phones.get_or_create(phone=phone)
114

115
            # Let's link the MEP to the group
116 117 118 119
            contact.groups = groups
            contact.save()
            # We want to link the contact to our current campaign
            CampaignContact(campaign=self.campaign, contact=contact).save()
okhin's avatar
okhin committed
120
            inserted += 1
121

122
        # Updating the last imported item
okhin's avatar
okhin committed
123 124
        self.last_imported = inserted
        self.save()
okhin's avatar
okhin committed
125

126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    def handle(self):
        """
        This function is used to import the data described by the Importer.

        We need to parse the provided file, using an appropriate deserilizer/parser,
        and then to create or update contacts.

        The operation should end providing the number of items imported and the number of items submitted.

        Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
        holding a list of groups (mandates) of a specific kind, column separated

        For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
        by columns.
        """
        import_data = []
        # First let's check what kind of importer are we running.
        # If it's a file, let's parse it according to our format.
        if self.kind() == 'file':
            with open(self.file.path, u'rU') as f:
146
                if self.category == 'updater':
147 148 149
                    # If we're a csv format, let's use DictReader on a file object
                    reader = DictReader(f)
                    for item in reader:
okhin's avatar
okhin committed
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
                        # FIXME: We need to create the groups according to the header
                        # Which means for each mandate:kind columsn we must add a item['mandates']
                        # entry with an end-date of ''
                        # One can belong to different groups
                        mandates = []
                        parsed_item = {}
                        for key in [k for k in item if k.startswith('mandates:')]:
                            for group in item[key].split(','):
                                mandates.append({'group':
                                        {'kind':key.split(':')[1],
                                        'name': group
                                        },
                                    'end_date': ''
                                    })
                        # Same for phones and websites
                        contacts = {}
                        contacts['phones'] = []
                        for key in [k for k in item if k.startswith('phones:')]:
                            contacts['phones'].append({'kind': 'office phone',
                                'number': item[key]})
                        contacts['emails'] = []
                        for key in [k for k in item if k.startswith('emails:')]:
                            contacts['emails'].append({'kind': key.split(':')[1],
                                'email': item[key]})
                        contacts['websites'] = []
                        for key in [k for k in item if k == 'twitter']:
                            contacts['websites'].append({'url': item[key],
                                'kind': 'twitter'})

                        parsed_item['first_name'] = item['first_name']
                        parsed_item['last_name'] = item['last_name']
                        parsed_item['birth_date'] = item['birth_date']
                        parsed_item['mandates'] = mandates
                        parsed_item['contacts'] = contacts
                        import_data.append(parsed_item)
185 186 187 188
                else:
                    # We're in json, let's load from a file
                    import_data = json.load(f)
        # Now, let's get the correct call
okhin's avatar
okhin committed
189
        self.representative(import_data)
190
        return