FIrst run of the importer

picampaign/importer/admin.py

 from django.contrib import admin
+from django.utils.translation import ugettext_lazy as _
 
 from picampaign.importer.models import Importer
 
 class ImporterAdmin(admin.ModelAdmin):
     list_display = ['name', 'kind', 'data_format', 'file', 'url']
+    actions = ['import_action']
 
-admin.site.register(Importer, ImporterAdmin)
+    def import_action(self, request, queryset):
+        """
+        We're going to launch the import of data
+        """
+        for obj in queryset:
+            obj.handle()
+            self.message_user(request, _("Imported %(last_imported)s of %(last_imported)s contacts") % {
+                'last_imported': obj.last_imported,
+                'last_count': obj.last_count})
+    import_action.short_description = _("Run the selected importers")
 
+admin.site.register(Importer, ImporterAdmin)

picampaign/importer/models.py

+from csv import DictReader
+import json
+import requests
+import re
+
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
 from picampaign.campaign.models import Campaign
+from picampaign.organization.models import Group, GroupType, Organization
 
 class Importer(models.Model):
     """Importer model. Used to populate campaign with contacts"""
@@ -11,11 +17,14 @@ class Importer(models.Model):
     file = models.FileField(upload_to='imports/', blank=True, null=True)
     url = models.URLField(blank=True, null=True)
     campaign = models.ForeignKey(Campaign, on_delete=models.SET_NULL, null=True)
+    organization = models.ForeignKey(Organization, on_delete=models.CASCADE)
+    last_count = models.IntegerField(null=True, blank=True)
+    last_imported = models.IntegerField(null=True, blank=True)
 
     def __str__(self):
-        return _('%{name}: %{format} %{type} importer' % {'format': self.data_format,
+        return _(u'%(name)s: %(format)s %(type)s importer') % {'format': self.data_format,
                                                 'type': self.kind(),
-                                                'name': self.name})
+                                                'name': self.name}
 
     def kind(self):
         if self.url is None and self.file is None:
@@ -24,3 +33,95 @@ class Importer(models.Model):
             return 'file'
         else:
             return 'url'
+
+    def handle(self):
+        """
+        This function is used to import the data described by the Importer.
+
+        We need to parse the provided file, using an appropriate deserilizer/parser,
+        and then to create or update contacts.
+
+        The operation should end providing the number of items imported and the number of items submitted.
+
+        Importing mandates from CSV requires to have header with "mandate:kind" in it, each one
+        holding a list of groups (mandates) of a specific kind, column separated
+
+        For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
+        by columns.
+        """
+        import_data = []
+        # First let's check what kind of importer are we running.
+        # If it's a file, let's parse it according to our format.
+        if self.kind == 'file':
+            with open(self.file, u'rU') as f:
+                if self.data_format == 'CSV':
+                    # If we're a csv format, let's use DictReader on a file object
+                    reader = DictReader(f)
+                    re_mandate = re.compile(r'^mandate:(\w+)')
+                    for item in reader:
+                        # We need to create the groups according to the header
+                        for key in item.keys():
+                            if re.match(re_mandate, key):
+                                # We have a mandate
+                                for mandate in item[key].split(':'):
+                                    item['mandates'].append({'kind': key.split(':')[0], 'name': mandate})
+                                item.delete(key)
+                    import_data.append(item)
+                else:
+                    # We're in json, let's load from a file
+                    import_data = json.load(f)
+        # We now have all the data ready to be imported.
+        # We can count the data available
+        self.last_count = len(import_data)
+
+        # Lets' go through the meps, for fun and profit
+        inserted = 0
+        for import_mep in import_data:
+            # We want to know if all mandates exists
+            if not 'birthdate' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
+                # We do not have enough field to discriminate our meps
+                continue
+            updated_mep = {}
+            updated_mep['groups'] = []
+            for mandate in import_mep['mandates']:
+                if mandate['end_date'] != '':
+                    # This mandate has ended
+                    continue
+                groupType, added = GroupType.objects.get_or_create(organization = self.organization,
+                        name = mandate['kind'])
+                group, added = Group.objects.get_or_create(name = mandate['name'],
+                        type = groupType)
+                updated_mep['groups'].append(group)
+
+            # If we have a contact item, we can go through it. Otherwise we probably have
+            # phone, twitter, etc fields.
+            if 'contacts' in import_mep:
+                # Let's get the first phone
+                if import_mep['contacts']['phones'] != []:
+                    updated_emp['phone'] = mep['contacts']['phones'][0]['phone']
+                if import_mep['contacts']['emails'] != []:
+                    updated_mep['mail'] = mep['contacts']['phones'][0]['phone']
+                if import_mep['contacts']['websites'] != []:
+                    for website in import_mep['contacts']['websites']:
+                        if website['kind'] == 'twitter':
+                            updated_mep['twitter'] = ''.join(['@', website['website'].split('/')[-1]])
+                            break
+            else:
+                # We have phone, mail, twitter fields
+                if 'phone' in import_mep:
+                    updated_mep['phone'] = import_mep['phone']
+                if 'mail' in import_mep:
+                    updated_mep['mail'] = import_mep['phone']
+                if 'twitter' in import_mep:
+                    updated_mep['twitter'] = import_mep['twitter']
+
+            # All groups are done
+            # Let's update_or_create the mep
+            mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
+                        last_name=import_mep['last_name'],
+                        birthdate=import_mep['birthdate'],
+                        defaults=updated_mep
+                    )
+            inserted += 1
+        self.last_imported = inserted
+        self.save()

setup.py

@@ -9,6 +9,7 @@ requires = [
     'django',
     'djangorestframework',
     'drf-nested-routers',
+    'requests',
     'pillow',
 ]