Commit ae6bb35f authored by Nicolas Joyard's avatar Nicolas Joyard
Browse files

Update to unique slugs/removed remote_id

parent 24c86729
...@@ -39,7 +39,9 @@ class VotesImporter: ...@@ -39,7 +39,9 @@ class VotesImporter:
if self.deputes_rid is None: if self.deputes_rid is None:
self.deputes_rid = { self.deputes_rid = {
r[0]: r[1] for r in r[0]: r[1] for r in
Representative.objects.values_list('remote_id', 'pk') Representative.objects.prefetch_related('website_set')
.filter(website__kind__in=['AN', 'SEN'])
.values_list('website__url', 'pk')
} }
return self.deputes_rid.get(url, None) return self.deputes_rid.get(url, None)
......
...@@ -6,14 +6,13 @@ ...@@ -6,14 +6,13 @@
"photo": "http://www.nosdeputes.fr/depute/photo/bernard-roman", "photo": "http://www.nosdeputes.fr/depute/photo/bernard-roman",
"created": "2016-02-14T14:01:37.343Z", "created": "2016-02-14T14:01:37.343Z",
"gender": 2, "gender": 2,
"remote_id": "2611",
"first_name": "", "first_name": "",
"cv": "", "cv": "",
"active": true, "active": true,
"birth_place": "Lille (Nord)", "birth_place": "Lille (Nord)",
"full_name": "Bernard Roman", "full_name": "Bernard Roman",
"birth_date": "1952-07-15", "birth_date": "1952-07-15",
"slug": "bernard-roman" "slug": "bernard-roman-1952-07-15"
}, },
"model": "representatives.representative", "model": "representatives.representative",
"pk": 1 "pk": 1
......
# coding: utf-8 # coding: utf-8
import logging import logging
import re
import sys import sys
from os.path import join from os.path import join
...@@ -166,8 +167,13 @@ class Command(object): ...@@ -166,8 +167,13 @@ class Command(object):
return self.cache['dossiers'].get(reference, None) return self.cache['dossiers'].get(reference, None)
def index_representatives(self): def index_representatives(self):
self.cache['meps'] = {int(l[0]): l[1] for l in epre = r'/meps/en/(\d+)/_home.html'
Representative.objects.values_list('remote_id', 'pk')} self.cache['meps'] = {
int(re.search(epre, l[0]).group(1)): l[1] for l in
Representative.objects.prefetch_related('website_set')
.filter(website__kind='EP')
.values_list('website__url', 'pk')
}
def get_representative(self, vote_data): def get_representative(self, vote_data):
if vote_data.get('ep_id', None) is None: if vote_data.get('ep_id', None) is None:
......
...@@ -29,7 +29,6 @@ ...@@ -29,7 +29,6 @@
"photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg", "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
"created": "2015-12-12T22:56:14.401Z", "created": "2015-12-12T22:56:14.401Z",
"gender": 2, "gender": 2,
"remote_id": "2307",
"first_name": "Hubert", "first_name": "Hubert",
"cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);", "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
"active": false, "active": false,
...@@ -48,7 +47,6 @@ ...@@ -48,7 +47,6 @@
"photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg", "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
"created": "2015-12-12T22:56:14.757Z", "created": "2015-12-12T22:56:14.757Z",
"gender": 2, "gender": 2,
"remote_id": "96673",
"first_name": "Olle", "first_name": "Olle",
"cv": "", "cv": "",
"active": true, "active": true,
......
...@@ -282,14 +282,13 @@ ...@@ -282,14 +282,13 @@
"photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg", "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
"created": "2015-12-13T02:07:23.995Z", "created": "2015-12-13T02:07:23.995Z",
"gender": 2, "gender": 2,
"remote_id": "2307",
"first_name": "Hubert", "first_name": "Hubert",
"cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);", "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
"active": false, "active": false,
"birth_place": "Gries", "birth_place": "Gries",
"full_name": "Hubert PIRKER", "full_name": "Hubert PIRKER",
"birth_date": "1948-10-03", "birth_date": "1948-10-03",
"slug": "hubert-pirker" "slug": "hubert-pirker-1948-10-03"
}, },
"model": "representatives.representative", "model": "representatives.representative",
"pk": 1 "pk": 1
...@@ -301,14 +300,13 @@ ...@@ -301,14 +300,13 @@
"photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg", "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
"created": "2015-12-13T02:07:24.361Z", "created": "2015-12-13T02:07:24.361Z",
"gender": 2, "gender": 2,
"remote_id": "96673",
"first_name": "Olle", "first_name": "Olle",
"cv": "", "cv": "",
"active": true, "active": true,
"birth_place": "H\u00e4ls\u00f6", "birth_place": "H\u00e4ls\u00f6",
"full_name": "Olle LUDVIGSSON", "full_name": "Olle LUDVIGSSON",
"birth_date": "1948-10-28", "birth_date": "1948-10-28",
"slug": "olle-ludvigsson" "slug": "olle-ludvigsson-1948-10-03"
}, },
"model": "representatives.representative", "model": "representatives.representative",
"pk": 2 "pk": 2
...@@ -357,6 +355,28 @@ ...@@ -357,6 +355,28 @@
"model": "representatives.website", "model": "representatives.website",
"pk": 3 "pk": 3
}, },
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/2307/_home.html",
"updated": "2015-12-13T02:07:24.406Z",
"representative": 1,
"kind": "EP",
"created": "2015-12-13T02:07:24.406Z"
},
"model": "representatives.website",
"pk": 4
},
{
"fields": {
"url": "http://www.europarl.europa.eu/meps/en/96673/_home.html",
"updated": "2015-12-13T02:07:24.406Z",
"representative": 2,
"kind": "EP",
"created": "2015-12-13T02:07:24.406Z"
},
"model": "representatives.website",
"pk": 5
},
{ {
"fields": { "fields": {
"updated": "2015-12-13T02:07:23.997Z", "updated": "2015-12-13T02:07:23.997Z",
......
...@@ -13,7 +13,7 @@ setup( ...@@ -13,7 +13,7 @@ setup(
keywords='django government parliament votes', keywords='django government parliament votes',
install_requires=[ install_requires=[
'django>1.8,<1.9', 'django>1.8,<1.9',
'django-representatives>=0.0.29', 'django-representatives>=0.0.32',
'py-dateutil>=2,<3', 'py-dateutil>=2,<3',
'ijson>=2,<3', 'ijson>=2,<3',
'pytz', # Always use up-to-date TZ data 'pytz', # Always use up-to-date TZ data
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment