Importer works

picampaign/contact/migrations/0005_auto_20160807_2052.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2016-08-07 18:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contact', '0004_auto_20160807_1834'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='contact',
+            name='birthdate',
+            field=models.DateField(blank=True),
+        ),
+    ]

picampaign/contact/migrations/0006_auto_20160807_2057.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2016-08-07 18:57
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contact', '0005_auto_20160807_2052'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='contact',
+            name='birthdate',
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

picampaign/contact/models.py

@@ -7,7 +7,7 @@ class Contact(models.Model):
     id = models.AutoField(primary_key=True)
     first_name = models.CharField(max_length=64)
     last_name = models.CharField(max_length=64)
-    birthdate = models.DateField()
+    birthdate = models.DateField(blank=True, null=True)
     phone = models.CharField(max_length=32, blank=True)
     twitter = models.CharField(max_length=64, blank=True)
     mail = models.CharField(max_length=255, blank=True)

picampaign/importer/models.py

@@ -6,6 +6,7 @@ import re
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
 
+from picampaign.contact.models import Contact
 from picampaign.campaign.models import Campaign
 from picampaign.organization.models import Group, GroupType, Organization
 
@@ -49,12 +50,11 @@ class Importer(models.Model):
         For instance, the comittee of a MEP would be listed in a column named 'group:comittee', separated
         by columns.
         """
-        import ipdb; ipdb.set_trace()
         import_data = []
         # First let's check what kind of importer are we running.
         # If it's a file, let's parse it according to our format.
-        if self.kind == 'file':
-            with open(self.file, u'rU') as f:
+        if self.kind() == 'file':
+            with open(self.file.path, u'rU') as f:
                 if self.data_format == 'CSV':
                     # If we're a csv format, let's use DictReader on a file object
                     reader = DictReader(f)
@@ -79,13 +79,15 @@ class Importer(models.Model):
         inserted = 0
         for import_mep in import_data:
             # We want to know if all mandates exists
-            if not 'birthdate' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
+            if not 'birth_date' in import_mep or not 'first_name' in import_mep or not 'last_name' in import_mep:
                 # We do not have enough field to discriminate our meps
                 continue
             updated_mep = {}
             updated_mep['groups'] = []
             for mandate in import_mep['mandates']:
-                if mandate['end_date'] != '':
+                # Parltrack gives us a date starting with 9999 if ends has not happens
+                # FranceData gives us an empty one
+                if 'end_date' in mandate and (mandate['end_date'] != '' or mandate['end_date'].startswith('9999')):
                     # This mandate has ended
                     continue
                 groupType, added = GroupType.objects.get_or_create(organization = self.organization,
@@ -93,19 +95,22 @@ class Importer(models.Model):
                 group, added = Group.objects.get_or_create(name = mandate['name'],
                         type = groupType)
                 updated_mep['groups'].append(group)
+            if updated_mep['groups'] == []:
+                del updated_mep['groups']
 
+            # All groups are done
             # If we have a contact item, we can go through it. Otherwise we probably have
             # phone, twitter, etc fields.
             if 'contacts' in import_mep:
                 # Let's get the first phone
                 if import_mep['contacts']['phones'] != []:
-                    updated_emp['phone'] = mep['contacts']['phones'][0]['phone']
+                    updated_mep['phone'] = import_mep['contacts']['phones'][0]['number']
                 if import_mep['contacts']['emails'] != []:
-                    updated_mep['mail'] = mep['contacts']['phones'][0]['phone']
+                    updated_mep['mail'] = import_mep['contacts']['emails'][0]['email']
                 if import_mep['contacts']['websites'] != []:
                     for website in import_mep['contacts']['websites']:
                         if website['kind'] == 'twitter':
-                            updated_mep['twitter'] = ''.join(['@', website['website'].split('/')[-1]])
+                            updated_mep['twitter'] = ''.join(['@', website['url'].split('/')[-1]])
                             break
             else:
                 # We have phone, mail, twitter fields
@@ -116,11 +121,11 @@ class Importer(models.Model):
                 if 'twitter' in import_mep:
                     updated_mep['twitter'] = import_mep['twitter']
 
-            # All groups are done
+            #import ipdb; ipdb.set_trace()
             # Let's update_or_create the mep
             mep, updated = Contact.objects.update_or_create(first_name=import_mep['first_name'],
                         last_name=import_mep['last_name'],
-                        birthdate=import_mep['birthdate'],
+                        birthdate=import_mep['birth_date'],
                         defaults=updated_mep
                     )
             inserted += 1