Commit f43ee25c authored by Nicolas Joyard's avatar Nicolas Joyard

Remove remote ID on representatives

Enables having representatives belong to multiple chambers.
Import scripts now rely on the slug to identify representatives.
The remote_id field was moved to a website entity (linking to the
representative page on the chamber website).
parent 22e8fcbc
......@@ -55,7 +55,6 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
'active': ['exact'],
'slug': ['exact', 'icontains'],
'id': ['exact'],
'remote_id': ['exact'],
'first_name': ['exact', 'icontains'],
'last_name': ['exact', 'icontains'],
'full_name': ['exact', 'icontains'],
......
......@@ -11,6 +11,7 @@ import django
from django.apps import apps
from django.db import transaction
from django.utils import timezone
from django.utils.text import slugify
from representatives.models import (Country, Mandate, Email, Address, WebSite,
Representative, Constituency, Phone, Group,
......@@ -122,16 +123,6 @@ class FranceDataImporter(GenericImporter):
Import a rep as a representative from the json dict fetched from
FranceData (which comes from nosdeputes.fr)
'''
remote_id = rep_json[self.variant['remote_id_field']]
if rep_json['num_circo'] == 'non disponible':
rep_json['num_circo'] = 'nd'
if not remote_id:
logger.warning('Skipping MEP without UID %s %s',
rep_json['nom'],
rep_json[self.variant['remote_id_field']])
return
# Some versions of memopol will connect to this and skip inactive reps.
responses = representative_pre_import.send(sender=self,
......@@ -143,15 +134,23 @@ class FranceDataImporter(GenericImporter):
'Skipping MEP %s', rep_json['nom'])
return
changed = False
slug = slugify(
rep_json['nom'] if 'nom' in rep_json
else rep_json['prenom'] + " " + rep_json['nom_de_famille']
)
try:
representative = Representative.objects.get(remote_id=remote_id)
representative = Representative.objects.get(slug=slug)
except Representative.DoesNotExist:
representative = Representative(remote_id=remote_id)
representative = Representative(slug=slug)
changed = True
# Save representative attributes
self.import_representative_details(representative, rep_json)
if rep_json['num_circo'] == 'non disponible':
rep_json['num_circo'] = 'nd'
representative.save()
# Save representative attributes
self.import_representative_details(representative, rep_json, changed)
self.add_mandates(representative, rep_json)
......@@ -161,34 +160,59 @@ class FranceDataImporter(GenericImporter):
return representative
def import_representative_details(self, representative, rep_json):
representative.active = True
def import_representative_details(self, representative, rep_json, changed):
active = True
if rep_json.get("ancien_depute", 0) == 1:
representative.active = False
active = False
if rep_json.get("ancien_senateur", 0) == 1:
representative.active = False
active = False
if representative.active != active:
representative.active = active
changed = True
if rep_json.get("date_naissance"):
representative.birth_date = _parse_date(rep_json["date_naissance"])
if rep_json.get("lieu_naissance"):
representative.birth_place = rep_json["lieu_naissance"]
birth_date = _parse_date(rep_json["date_naissance"])
if representative.birth_date != birth_date:
representative.birth_date = birth_date
changed = True
representative.photo = rep_json['photo_url']
representative.first_name = rep_json['prenom']
representative.last_name = rep_json['nom_de_famille']
representative.full_name = rep_json["nom"]
gender_convertion_dict = {
u"F": 1,
u"H": 2
}
if rep_json.get("lieu_naissance"):
birth_place = rep_json["lieu_naissance"]
if representative.birth_place != birth_place:
representative.birth_place = birth_place
changed = True
photo = rep_json['photo_url']
if representative.photo != photo:
representative.photo = photo
changed = True
first_name = rep_json['prenom']
if representative.first_name != first_name:
representative.first_name = first_name
changed = True
last_name = rep_json['nom_de_famille']
if representative.last_name != last_name:
representative.last_name = last_name
changed = True
full_name = rep_json["nom"]
if representative.full_name != full_name:
representative.full_name = full_name
changed = True
gender_convertion_dict = {u"F": 1, u"H": 2}
if 'sexe' in rep_json:
representative.gender = gender_convertion_dict.get(rep_json[
'sexe'], 0)
gender = gender_convertion_dict.get(rep_json['sexe'], 0)
else:
representative.gender = 0
gender = 0
if representative.gender != gender:
representative.gender = gender
changed = True
representative.slug = rep_json['slug']
if changed:
representative.save()
def add_mandates(self, representative, rep_json):
'''
......@@ -245,6 +269,23 @@ class FranceDataImporter(GenericImporter):
mdef['kind'], role, name, abbr, start, end))
def add_contacts(self, representative, rep_json):
# Chamber page
changed = False
try:
site = WebSite.objects.get(kind=self.variant['abbreviation'],
representative=representative)
except WebSite.DoesNotExist:
site = WebSite(kind=self.variant['abbreviation'],
representative=representative)
changed = True
if site.url != rep_json[self.variant['chamber_url_field']]:
site.url = rep_json[self.variant['chamber_url_field']]
changed = True
if changed:
site.save()
# Websites
websites = rep_json.get('sites_web', [])
for site in websites:
......
......@@ -3,7 +3,6 @@
"fields": {
"last_name": "Roman",
"gender": 2,
"remote_id": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611",
"first_name": "Bernard",
"cv": "",
"active": true,
......@@ -20,7 +19,6 @@
"fields": {
"last_name": "Assouline",
"gender": 2,
"remote_id": "http://www.senat.fr/senateur/assouline_david04059m.html",
"first_name": "David",
"cv": "",
"active": true,
......@@ -60,6 +58,15 @@
"model": "representatives.email",
"pk": 3
},
{
"fields": {
"url": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611",
"representative": 1,
"kind": "AN"
},
"model": "representatives.website",
"pk": 1
},
{
"fields": {
"url": "http://www.bernard-roman.net",
......@@ -67,7 +74,7 @@
"kind": ""
},
"model": "representatives.website",
"pk": 1
"pk": 2
},
{
"fields": {
......@@ -76,7 +83,16 @@
"kind": "twitter"
},
"model": "representatives.website",
"pk": 2
"pk": 3
},
{
"fields": {
"url": "http://www.senat.fr/senateur/assouline_david04059m.html",
"representative": 2,
"kind": "SEN"
},
"model": "representatives.website",
"pk": 4
},
{
"fields": {
......@@ -85,7 +101,7 @@
"kind": ""
},
"model": "representatives.website",
"pk": 3
"pk": 5
},
{
"fields": {
......@@ -94,7 +110,7 @@
"kind": "facebook"
},
"model": "representatives.website",
"pk": 4
"pk": 6
},
{
"fields": {
......@@ -103,7 +119,7 @@
"kind": ""
},
"model": "representatives.website",
"pk": 5
"pk": 7
},
{
"fields": {
......@@ -112,7 +128,7 @@
"kind": "twitter"
},
"model": "representatives.website",
"pk": 6
"pk": 8
},
{
"fields": {
......
......@@ -131,7 +131,7 @@ FranceDataVariants = {
"AN": {
"chamber": u"Assemblée nationale",
"abbreviation": "AN",
"remote_id_field": "url_an",
"chamber_url_field": "url_an",
"mail_domain": "@assemblee-nationale.fr",
"off_city": "Paris",
"off_street": u"Rue de l'Université",
......@@ -197,7 +197,7 @@ FranceDataVariants = {
"SEN": {
"chamber": u"Sénat",
"abbreviation": "SEN",
"remote_id_field": "url_institution",
"chamber_url_field": "url_institution",
"mail_domain": "@senat.fr",
"off_city": "Paris",
"off_street": u"Rue de Vaugirard",
......
......@@ -9,8 +9,8 @@ import ijson
import django
from django.apps import apps
from django.db import transaction
from django.template.defaultfilters import slugify
from django.utils import timezone
from django.utils.text import slugify
from representatives.models import (Address, Constituency, Country, Email,
Group, Mandate, Phone, Representative,
......@@ -79,12 +79,6 @@ class ParltrackImporter(GenericImporter):
Import a mep as a representative from the json dict fetched from
parltrack
'''
remote_id = mep_json['UserID']
if not remote_id:
logger.warning('Skipping MEP without UID %s %s',
mep_json['Name']['full'], mep_json['UserID'])
return
# Some versions of memopol will connect to this and skip inactive meps.
responses = representative_pre_import.send(sender=self,
......@@ -96,15 +90,19 @@ class ParltrackImporter(GenericImporter):
'Skipping MEP %s', mep_json['Name']['full'])
return
changed = False
slug = slugify(
mep_json["Name"]["full"] if 'full' in mep_json["Name"]
else mep_json["Name"]["sur"] + " " + mep_json["Name"]["family"]
)
try:
representative = Representative.objects.get(remote_id=remote_id)
representative = Representative.objects.get(slug=slug)
except Representative.DoesNotExist:
representative = Representative(remote_id=remote_id)
representative = Representative(slug=slug)
changed = True
# Save representative attributes
self.import_representative_details(representative, mep_json)
representative.save()
self.import_representative_details(representative, mep_json, changed)
self.add_mandates(representative, mep_json)
......@@ -114,19 +112,35 @@ class ParltrackImporter(GenericImporter):
return representative
def import_representative_details(self, representative, mep_json):
representative.active = mep_json['active']
def import_representative_details(self, representative, mep_json, changed):
if representative.active != mep_json['active']:
representative.active = mep_json['active']
changed = True
if mep_json.get("Birth"):
representative.birth_date = _parse_date(mep_json["Birth"]["date"])
birth_date = _parse_date(mep_json["Birth"]["date"])
if representative.birth_date != birth_date:
representative.birth_date = birth_date
changed = True
if "place" in mep_json["Birth"]:
representative.birth_place = mep_json["Birth"]["place"]
birth_place = mep_json["Birth"]["place"]
if representative.birth_place != birth_place:
representative.birth_place = birth_place
changed = True
if representative.first_name != mep_json["Name"]["sur"]:
representative.first_name = mep_json["Name"]["sur"]
changed = True
representative.first_name = mep_json["Name"]["sur"]
representative.last_name = mep_json["Name"]["family"]
representative.full_name = mep_json["Name"]["full"]
last_name = mep_json["Name"]["family"]
representative.photo = mep_json["Photo"]
if representative.full_name != mep_json["Name"]["full"]:
representative.full_name = mep_json["Name"]["full"]
changed = True
if representative.photo != mep_json["Photo"]:
representative.photo = mep_json["Photo"]
changed = True
fix_last_name_with_prefix = {
"Esther de LANGE": "de LANGE",
......@@ -161,28 +175,30 @@ class ParltrackImporter(GenericImporter):
}
if fix_last_name_with_prefix.get(representative.full_name):
representative.last_name = fix_last_name_with_prefix[
representative.full_name]
elif representative.last_name == "J.A.J. STASSEN":
representative.last_name_with_prefix = "STASSEN"
gender_convertion_dict = {
u"F": 1,
u"M": 2
}
last_name = fix_last_name_with_prefix[representative.full_name]
elif last_name == "J.A.J. STASSEN":
last_name = "STASSEN"
if representative.last_name != last_name:
representative.last_name = last_name
changed = True
gender_convertion_dict = {u"F": 1, u"M": 2}
if 'Gender' in mep_json:
representative.gender = gender_convertion_dict.get(mep_json[
'Gender'], 0)
gender = gender_convertion_dict.get(mep_json['Gender'], 0)
else:
representative.gender = 0
gender = 0
if representative.gender != gender:
representative.gender = gender
changed = True
representative.cv = "\n".join(
[cv_title for cv_title in mep_json.get("CV", [])])
cv = "\n".join([cv_title for cv_title in mep_json.get("CV", [])])
if representative.cv != cv:
representative.cv = cv
changed = True
representative.slug = slugify(
representative.full_name if representative.full_name
else representative.first_name + " " + representative.last_name
)
if changed:
representative.save()
def add_mandates(self, representative, mep_json):
def create_mandate(mandate_data, representative, group, constituency):
......@@ -353,6 +369,25 @@ class ParltrackImporter(GenericImporter):
kind=('official' if '@europarl.europa.eu' in mail
else 'other'),
email=mail)
# EP page
changed = False
try:
site = WebSite.objects.get(kind='EP',
representative=representative)
except WebSite.DoesNotExist:
site = WebSite(kind='EP', representative=representative)
changed = True
uid = mep_json['UserID']
url = 'http://www.europarl.europa.eu/meps/en/%s/_home.html' % uid
if site.url != url:
site.url = url
changed = True
if changed:
site.save()
# WebSite
websites = mep_json.get('Homepage', [])
for url in websites:
......
......@@ -4,7 +4,6 @@
"last_name": "PIRKER",
"photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
"gender": 2,
"remote_id": "2307",
"first_name": "Hubert",
"cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
"active": false,
......@@ -21,7 +20,6 @@
"last_name": "LUDVIGSSON",
"photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
"gender": 2,
"remote_id": "96673",
"first_name": "Olle",
"cv": "",
"active": true,
......@@ -42,6 +40,24 @@
"model": "representatives.email",
"pk": 1
},
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/2307/_home.html",
"representative": 1,
"kind": "EP"
},
"model": "representatives.website",
"pk": 1
},
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/96673/_home.html",
"representative": 2,
"kind": "EP"
},
"model": "representatives.website",
"pk": 2
},
{
"fields": {
"url": "http://www.sap.se/olle",
......@@ -49,7 +65,7 @@
"kind": ""
},
"model": "representatives.website",
"pk": 1
"pk": 3
},
{
"fields": {
......@@ -58,7 +74,7 @@
"kind": "twitter"
},
"model": "representatives.website",
"pk": 2
"pk": 4
},
{
"fields": {
......@@ -67,7 +83,7 @@
"kind": "facebook"
},
"model": "representatives.website",
"pk": 3
"pk": 5
},
{
"fields": {
......
......@@ -78,7 +78,6 @@
"photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
"created": "2015-12-13T02:07:23.995",
"gender": 2,
"remote_id": "2307",
"first_name": "Hubert",
"cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
"active": false,
......@@ -97,7 +96,6 @@
"photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
"created": "2015-12-13T02:07:24.361",
"gender": 2,
"remote_id": "96673",
"first_name": "Olle",
"cv": "",
"active": true,
......@@ -153,6 +151,28 @@
"model": "representatives.website",
"pk": 3
},
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/2307/_home.html",
"updated": "2015-12-13T02:07:24.408",
"representative": 1,
"kind": "EP",
"created": "2015-12-13T02:07:24.408"
},
"model": "representatives.website",
"pk": 4
},
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/96673/_home.html",
"updated": "2015-12-13T02:07:24.408",
"representative": 2,
"kind": "EP",
"created": "2015-12-13T02:07:24.408"
},
"model": "representatives.website",
"pk": 5
},
{
"fields": {
"city": "Brussels",
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import datetime
from django.db import connection, migrations, models
def create_parl_websites(apps, schema_editor):
"""
Prepare for remote_id removal by creating WebSite entities from it.
"""
# Get model managers
Representative = apps.get_model("representatives", "Representative")
WebSite = apps.get_model("representatives", "WebSite")
today = datetime.date.today()
# EP
ep_url = 'http://www.europarl.europa.eu/meps/en/%s/_home.html'
qs = Representative.objects.filter(
models.Q(mandates__end_date__gte=today) |
models.Q(mandates__end_date__isnull=True),
mandates__group__chamber__abbreviation='EP'
)
for rep in qs:
changed = False
url = ep_url % rep.remote_id
try:
site = WebSite.objects.get(representative=rep, kind='EP')
except WebSite.DoesNotExist:
site = WebSite(representative=rep, kind='EP', url=url)
changed = True
if site.url != url:
site.url = url
changed = True
if changed:
site.save()
# AN/SEN
for chamber in ['AN', 'SEN']:
qs = Representative.objects.filter(
models.Q(mandates__end_date__gte=today) |
models.Q(mandates__end_date__isnull=True),
mandates__group__chamber__abbreviation=chamber
)
for rep in qs:
changed = False
url = rep.remote_id
try:
site = WebSite.objects.get(representative=rep, kind=chamber)
except WebSite.DoesNotExist:
site = WebSite(representative=rep, kind=chamber, url=url)
changed = True
if site.url != url: