Merge pull request #38 from political-memory/remove-remote-id

Remove remote ID on representatives

.coveragerc

 [run]
-omit = representatives_votes/tests/*
-omit = representatives_votes/migrations/*
+omit =
+	representatives/tests/*
+	representatives/migrations/*
+	representatives/contrib/francedata/tests/*
+	representatives/contrib/parltrack/tests/*

representatives/api.py

@@ -55,7 +55,6 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
         'active': ['exact'],
         'slug': ['exact', 'icontains'],
         'id': ['exact'],
-        'remote_id': ['exact'],
         'first_name': ['exact', 'icontains'],
         'last_name': ['exact', 'icontains'],
         'full_name': ['exact', 'icontains'],

representatives/contrib/francedata/import_representatives.py

@@ -11,6 +11,7 @@ import django
 from django.apps import apps
 from django.db import transaction
 from django.utils import timezone
+from django.utils.text import slugify
 
 from representatives.models import (Country, Mandate, Email, Address, WebSite,
                                     Representative, Constituency, Phone, Group,
@@ -122,16 +123,6 @@ class FranceDataImporter(GenericImporter):
         Import a rep as a representative from the json dict fetched from
         FranceData (which comes from nosdeputes.fr)
         '''
-        remote_id = rep_json[self.variant['remote_id_field']]
-
-        if rep_json['num_circo'] == 'non disponible':
-            rep_json['num_circo'] = 'nd'
-
-        if not remote_id:
-            logger.warning('Skipping MEP without UID %s %s',
-                           rep_json['nom'],
-                           rep_json[self.variant['remote_id_field']])
-            return
 
         # Some versions of memopol will connect to this and skip inactive reps.
         responses = representative_pre_import.send(sender=self,
@@ -143,15 +134,23 @@ class FranceDataImporter(GenericImporter):
                     'Skipping MEP %s', rep_json['nom'])
                 return
 
+        changed = False
+        slug = slugify(
+            rep_json['nom'] if 'nom' in rep_json
+            else rep_json['prenom'] + " " + rep_json['nom_de_famille']
+        )
+
         try:
-            representative = Representative.objects.get(remote_id=remote_id)
+            representative = Representative.objects.get(slug=slug)
         except Representative.DoesNotExist:
-            representative = Representative(remote_id=remote_id)
+            representative = Representative(slug=slug)
+            changed = True
 
-        # Save representative attributes
-        self.import_representative_details(representative, rep_json)
+        if rep_json['num_circo'] == 'non disponible':
+            rep_json['num_circo'] = 'nd'
 
-        representative.save()
+        # Save representative attributes
+        self.import_representative_details(representative, rep_json, changed)
 
         self.add_mandates(representative, rep_json)
 
@@ -161,34 +160,59 @@ class FranceDataImporter(GenericImporter):
 
         return representative
 
-    def import_representative_details(self, representative, rep_json):
-        representative.active = True
+    def import_representative_details(self, representative, rep_json, changed):
+        active = True
         if rep_json.get("ancien_depute", 0) == 1:
-            representative.active = False
+            active = False
         if rep_json.get("ancien_senateur", 0) == 1:
-            representative.active = False
+            active = False
+        if representative.active != active:
+            representative.active = active
+            changed = True
 
         if rep_json.get("date_naissance"):
-            representative.birth_date = _parse_date(rep_json["date_naissance"])
-        if rep_json.get("lieu_naissance"):
-            representative.birth_place = rep_json["lieu_naissance"]
+            birth_date = _parse_date(rep_json["date_naissance"])
+            if representative.birth_date != birth_date:
+                representative.birth_date = birth_date
+                changed = True
 
-        representative.photo = rep_json['photo_url']
-        representative.first_name = rep_json['prenom']
-        representative.last_name = rep_json['nom_de_famille']
-        representative.full_name = rep_json["nom"]
-
-        gender_convertion_dict = {
-            u"F": 1,
-            u"H": 2
-        }
+        if rep_json.get("lieu_naissance"):
+            birth_place = rep_json["lieu_naissance"]
+            if representative.birth_place != birth_place:
+                representative.birth_place = birth_place
+                changed = True
+
+        photo = rep_json['photo_url']
+        if representative.photo != photo:
+            representative.photo = photo
+            changed = True
+
+        first_name = rep_json['prenom']
+        if representative.first_name != first_name:
+            representative.first_name = first_name
+            changed = True
+
+        last_name = rep_json['nom_de_famille']
+        if representative.last_name != last_name:
+            representative.last_name = last_name
+            changed = True
+
+        full_name = rep_json["nom"]
+        if representative.full_name != full_name:
+            representative.full_name = full_name
+            changed = True
+
+        gender_convertion_dict = {u"F": 1, u"H": 2}
         if 'sexe' in rep_json:
-            representative.gender = gender_convertion_dict.get(rep_json[
-                                                               'sexe'], 0)
+            gender = gender_convertion_dict.get(rep_json['sexe'], 0)
         else:
-            representative.gender = 0
+            gender = 0
+        if representative.gender != gender:
+            representative.gender = gender
+            changed = True
 
-        representative.slug = rep_json['slug']
+        if changed:
+            representative.save()
 
     def add_mandates(self, representative, rep_json):
         '''
@@ -245,6 +269,23 @@ class FranceDataImporter(GenericImporter):
                     mdef['kind'], role, name, abbr, start, end))
 
     def add_contacts(self, representative, rep_json):
+        # Chamber page
+        changed = False
+        try:
+            site = WebSite.objects.get(kind=self.variant['abbreviation'],
+                                       representative=representative)
+        except WebSite.DoesNotExist:
+            site = WebSite(kind=self.variant['abbreviation'],
+                           representative=representative)
+            changed = True
+
+        if site.url != rep_json[self.variant['chamber_url_field']]:
+            site.url = rep_json[self.variant['chamber_url_field']]
+            changed = True
+
+        if changed:
+            site.save()
+
         # Websites
         websites = rep_json.get('sites_web', [])
         for site in websites:

representatives/contrib/francedata/tests/representatives_expected.json

@@ -3,7 +3,6 @@
     "fields": {
         "last_name": "Roman",
         "gender": 2,
-        "remote_id": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611",
         "first_name": "Bernard",
         "cv": "",
         "active": true,
@@ -20,7 +19,6 @@
     "fields": {
         "last_name": "Assouline",
         "gender": 2,
-        "remote_id": "http://www.senat.fr/senateur/assouline_david04059m.html",
         "first_name": "David",
         "cv": "",
         "active": true,
@@ -60,6 +58,15 @@
     "model": "representatives.email",
     "pk": 3
 },
+{
+    "fields": {
+        "url": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611",
+        "representative": 1,
+        "kind": "AN"
+    },
+    "model": "representatives.website",
+    "pk": 1
+},
 {
     "fields": {
         "url": "http://www.bernard-roman.net",
@@ -67,7 +74,7 @@
         "kind": ""
     },
     "model": "representatives.website",
-    "pk": 1
+    "pk": 2
 },
 {
     "fields": {
@@ -76,7 +83,16 @@
         "kind": "twitter"
     },
     "model": "representatives.website",
-    "pk": 2
+    "pk": 3
+},
+{
+    "fields": {
+        "url": "http://www.senat.fr/senateur/assouline_david04059m.html",
+        "representative": 2,
+        "kind": "SEN"
+    },
+    "model": "representatives.website",
+    "pk": 4
 },
 {
     "fields": {
@@ -85,7 +101,7 @@
         "kind": ""
     },
     "model": "representatives.website",
-    "pk": 3
+    "pk": 5
 },
 {
     "fields": {
@@ -94,7 +110,7 @@
         "kind": "facebook"
     },
     "model": "representatives.website",
-    "pk": 4
+    "pk": 6
 },
 {
     "fields": {
@@ -103,7 +119,7 @@
         "kind": ""
     },
     "model": "representatives.website",
-    "pk": 5
+    "pk": 7
 },
 {
     "fields": {
@@ -112,7 +128,7 @@
         "kind": "twitter"
     },
     "model": "representatives.website",
-    "pk": 6
+    "pk": 8
 },
 {
     "fields": {

representatives/contrib/francedata/variants.py

@@ -131,7 +131,7 @@ FranceDataVariants = {
     "AN": {
         "chamber": u"Assemblée nationale",
         "abbreviation": "AN",
-        "remote_id_field": "url_an",
+        "chamber_url_field": "url_an",
         "mail_domain": "@assemblee-nationale.fr",
         "off_city": "Paris",
         "off_street": u"Rue de l'Université",
@@ -197,7 +197,7 @@ FranceDataVariants = {
     "SEN": {
         "chamber": u"Sénat",
         "abbreviation": "SEN",
-        "remote_id_field": "url_institution",
+        "chamber_url_field": "url_institution",
         "mail_domain": "@senat.fr",
         "off_city": "Paris",
         "off_street": u"Rue de Vaugirard",

representatives/contrib/parltrack/import_representatives.py

@@ -9,8 +9,8 @@ import ijson
 import django
 from django.apps import apps
 from django.db import transaction
-from django.template.defaultfilters import slugify
 from django.utils import timezone
+from django.utils.text import slugify
 
 from representatives.models import (Address, Constituency, Country, Email,
                                     Group, Mandate, Phone, Representative,
@@ -79,12 +79,6 @@ class ParltrackImporter(GenericImporter):
         Import a mep as a representative from the json dict fetched from
         parltrack
         '''
-        remote_id = mep_json['UserID']
-
-        if not remote_id:
-            logger.warning('Skipping MEP without UID %s %s',
-                           mep_json['Name']['full'], mep_json['UserID'])
-            return
 
         # Some versions of memopol will connect to this and skip inactive meps.
         responses = representative_pre_import.send(sender=self,
@@ -96,15 +90,19 @@ class ParltrackImporter(GenericImporter):
                     'Skipping MEP %s', mep_json['Name']['full'])
                 return
 
+        changed = False
+        slug = slugify(
+            mep_json["Name"]["full"] if 'full' in mep_json["Name"]
+            else mep_json["Name"]["sur"] + " " + mep_json["Name"]["family"]
+        )
         try:
-            representative = Representative.objects.get(remote_id=remote_id)
+            representative = Representative.objects.get(slug=slug)
         except Representative.DoesNotExist:
-            representative = Representative(remote_id=remote_id)
+            representative = Representative(slug=slug)
+            changed = True
 
         # Save representative attributes
-        self.import_representative_details(representative, mep_json)
-
-        representative.save()
+        self.import_representative_details(representative, mep_json, changed)
 
         self.add_mandates(representative, mep_json)
 
@@ -114,19 +112,35 @@ class ParltrackImporter(GenericImporter):
 
         return representative
 
-    def import_representative_details(self, representative, mep_json):
-        representative.active = mep_json['active']
+    def import_representative_details(self, representative, mep_json, changed):
+        if representative.active != mep_json['active']:
+            representative.active = mep_json['active']
+            changed = True
 
         if mep_json.get("Birth"):
-            representative.birth_date = _parse_date(mep_json["Birth"]["date"])
+            birth_date = _parse_date(mep_json["Birth"]["date"])
+            if representative.birth_date != birth_date:
+                representative.birth_date = birth_date
+                changed = True
             if "place" in mep_json["Birth"]:
-                representative.birth_place = mep_json["Birth"]["place"]
+                birth_place = mep_json["Birth"]["place"]
+                if representative.birth_place != birth_place:
+                    representative.birth_place = birth_place
+                    changed = True
+
+        if representative.first_name != mep_json["Name"]["sur"]:
+            representative.first_name = mep_json["Name"]["sur"]
+            changed = True
 
-        representative.first_name = mep_json["Name"]["sur"]
-        representative.last_name = mep_json["Name"]["family"]
-        representative.full_name = mep_json["Name"]["full"]
+        last_name = mep_json["Name"]["family"]
 
-        representative.photo = mep_json["Photo"]
+        if representative.full_name != mep_json["Name"]["full"]:
+            representative.full_name = mep_json["Name"]["full"]
+            changed = True
+
+        if representative.photo != mep_json["Photo"]:
+            representative.photo = mep_json["Photo"]
+            changed = True
 
         fix_last_name_with_prefix = {
             "Esther de LANGE": "de LANGE",
@@ -161,28 +175,30 @@ class ParltrackImporter(GenericImporter):
         }
 
         if fix_last_name_with_prefix.get(representative.full_name):
-            representative.last_name = fix_last_name_with_prefix[
-                representative.full_name]
-        elif representative.last_name == "J.A.J. STASSEN":
-            representative.last_name_with_prefix = "STASSEN"
-
-        gender_convertion_dict = {
-            u"F": 1,
-            u"M": 2
-        }
+            last_name = fix_last_name_with_prefix[representative.full_name]
+        elif last_name == "J.A.J. STASSEN":
+            last_name = "STASSEN"
+
+        if representative.last_name != last_name:
+            representative.last_name = last_name
+            changed = True
+
+        gender_convertion_dict = {u"F": 1, u"M": 2}
         if 'Gender' in mep_json:
-            representative.gender = gender_convertion_dict.get(mep_json[
-                                                               'Gender'], 0)
+            gender = gender_convertion_dict.get(mep_json['Gender'], 0)
         else:
-            representative.gender = 0
+            gender = 0
+        if representative.gender != gender:
+            representative.gender = gender
+            changed = True
 
-        representative.cv = "\n".join(
-            [cv_title for cv_title in mep_json.get("CV", [])])
+        cv = "\n".join([cv_title for cv_title in mep_json.get("CV", [])])
+        if representative.cv != cv:
+            representative.cv = cv
+            changed = True
 
-        representative.slug = slugify(
-            representative.full_name if representative.full_name
-            else representative.first_name + " " + representative.last_name
-        )
+        if changed:
+            representative.save()
 
     def add_mandates(self, representative, mep_json):
         def create_mandate(mandate_data, representative, group, constituency):
@@ -353,6 +369,25 @@ class ParltrackImporter(GenericImporter):
                     kind=('official' if '@europarl.europa.eu' in mail
                         else 'other'),
                     email=mail)
+
+        # EP page
+        changed = False
+        try:
+            site = WebSite.objects.get(kind='EP',
+                                       representative=representative)
+        except WebSite.DoesNotExist:
+            site = WebSite(kind='EP', representative=representative)
+            changed = True
+
+        uid = mep_json['UserID']
+        url = 'http://www.europarl.europa.eu/meps/en/%s/_home.html' % uid
+        if site.url != url:
+            site.url = url
+            changed = True
+
+        if changed:
+            site.save()
+
         # WebSite
         websites = mep_json.get('Homepage', [])
         for url in websites:

representatives/contrib/parltrack/tests/representatives_expected.json

@@ -4,7 +4,6 @@
         "last_name": "PIRKER",
         "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
         "gender": 2,
-        "remote_id": "2307",
         "first_name": "Hubert",
         "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
         "active": false,
@@ -21,7 +20,6 @@
         "last_name": "LUDVIGSSON",
         "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
         "gender": 2,
-        "remote_id": "96673",
         "first_name": "Olle",
         "cv": "",
         "active": true,
@@ -42,6 +40,24 @@
     "model": "representatives.email",
     "pk": 1
 },
+{
+    "fields": {
+        "url": "http://www.europarl.europa.eu/meps/en/2307/_home.html",
+        "representative": 1,
+        "kind": "EP"
+    },
+    "model": "representatives.website",
+    "pk": 1
+},
+{
+    "fields": {
+        "url": "http://www.europarl.europa.eu/meps/en/96673/_home.html",
+        "representative": 2,
+        "kind": "EP"
+    },
+    "model": "representatives.website",
+    "pk": 2
+},
 {
     "fields": {
         "url": "http://www.sap.se/olle",
@@ -49,7 +65,7 @@
         "kind": ""
     },
     "model": "representatives.website",
-    "pk": 1
+    "pk": 3
 },
 {
     "fields": {
@@ -58,7 +74,7 @@
         "kind": "twitter"
     },
     "model": "representatives.website",
-    "pk": 2
+    "pk": 4
 },
 {
     "fields": {
@@ -67,7 +83,7 @@
         "kind": "facebook"
     },
     "model": "representatives.website",
-    "pk": 3
+    "pk": 5
 },
 {
     "fields": {

representatives/fixtures/representatives_test.json

@@ -78,7 +78,6 @@
         "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
         "created": "2015-12-13T02:07:23.995",
         "gender": 2,
-        "remote_id": "2307",
         "first_name": "Hubert",
         "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
         "active": false,
@@ -97,7 +96,6 @@
         "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
         "created": "2015-12-13T02:07:24.361",
         "gender": 2,
-        "remote_id": "96673",
         "first_name": "Olle",
         "cv": "",
         "active": true,
@@ -153,6 +151,28 @@
     "model": "representatives.website",
     "pk": 3
 },
+{
+    "fields": {
+        "url": "http://www.europarl.europa.eu/meps/en/2307/_home.html",
+        "updated": "2015-12-13T02:07:24.408",
+        "representative": 1,
+        "kind": "EP",
+        "created": "2015-12-13T02:07:24.408"
+    },
+    "model": "representatives.website",
+    "pk": 4
+},
+{
+    "fields": {
+        "url": "http://www.europarl.europa.eu/meps/en/96673/_home.html",
+        "updated": "2015-12-13T02:07:24.408",
+        "representative": 2,
+        "kind": "EP",
+        "created": "2015-12-13T02:07:24.408"
+    },
+    "model": "representatives.website",
+    "pk": 5
+},
 {
     "fields": {
         "city": "Brussels",

representatives/migrations/0020_rep_unique_slug_remove_remoteid.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import datetime
+
+from django.db import connection, migrations, models
+
+
+def create_parl_websites(apps, schema_editor):
+    """
+    Prepare for remote_id removal by creating WebSite entities from it.
+    """
+
+    # Get model managers
+    Representative = apps.get_model("representatives", "Representative")
+    WebSite = apps.get_model("representatives", "WebSite")
+
+    today = datetime.date.today()
+
+    # EP
+    ep_url = 'http://www.europarl.europa.eu/meps/en/%s/_home.html'
+    qs = Representative.objects.filter(
+        models.Q(mandates__end_date__gte=today) |
+        models.Q(mandates__end_date__isnull=True),
+        mandates__group__chamber__abbreviation='EP'
+    )
+    for rep in qs: