Commit c622f421 authored by Nicolas Joyard's avatar Nicolas Joyard

Add chamber import to .contrib.parltrack

parent c294c6ca
......@@ -14,7 +14,7 @@ from django.utils import timezone
from representatives.models import (Address, Constituency, Country, Email,
Group, Mandate, Phone, Representative,
WebSite)
WebSite, Chamber)
logger = logging.getLogger(__name__)
......@@ -65,8 +65,13 @@ class ParltrackImporter(GenericImporter):
self.cache = {
'countries': {c.name: c.pk for c in Country.objects.all()},
}
self.european_constituency, _ = Constituency.objects.get_or_create(
self.ep_chamber, _ = Chamber.objects.get_or_create(
name='European Parliament')
self.ep_constituency, _ = Constituency.objects.get_or_create(
name='European Parliament')
self.ep_group, _ = Group.objects.get_or_create(
name='European Parliament', kind='chamber', abbreviation='EP',
chamber=self.ep_chamber)
@transaction.atomic
def manage_mep(self, mep_json):
......@@ -208,23 +213,25 @@ class ParltrackImporter(GenericImporter):
if mandate_data.get("committee_id"):
group, _ = self.touch_model(model=Group,
abbreviation=mandate_data['committee_id'],
kind='committee', name=mandate_data['Organization'])
kind='committee', name=mandate_data['Organization'],
chamber=self.ep_chamber)
self.mep_cache['committees'].append(
get_or_create_mandate(mandate_data, representative,
group, self.european_constituency)
group, self.ep_constituency)
)
# Delegations
for mandate_data in mep_json.get('Delegations', []):
group, _ = self.touch_model(model=Group,
kind='delegation',
name=mandate_data['Organization']
name=mandate_data['Organization'],
chamber=self.ep_chamber
)
self.mep_cache['delegations'].append(
get_or_create_mandate(mandate_data, representative, group,
self.european_constituency)
self.ep_constituency)
)
# Group
......@@ -247,12 +254,13 @@ class ParltrackImporter(GenericImporter):
group, _ = self.touch_model(model=Group,
abbreviation=abbreviation,
kind='group',
name=mandate_data['Organization']
name=mandate_data['Organization'],
chamber=self.ep_chamber
)
self.mep_cache['groups'].append(
get_or_create_mandate(mandate_data, representative, group,
self.european_constituency)
self.ep_constituency)
)
# Countries
......@@ -293,6 +301,11 @@ class ParltrackImporter(GenericImporter):
constituency)
)
self.mep_cache['chambers'].append(
get_or_create_mandate(mandate_data, representative,
self.ep_group, self.ep_constituency)
)
# Organisations
for mandate_data in mep_json.get('Staff', []):
......@@ -304,7 +317,7 @@ class ParltrackImporter(GenericImporter):
self.mep_cache['staff'].append(
get_or_create_mandate(mandate_data, representative, group,
self.european_constituency)
self.ep_constituency)
)
def add_contacts(self, representative, mep_json):
......@@ -387,7 +400,7 @@ def main(stream=None):
GenericImporter.pre_import(importer)
for data in ijson.items(stream or sys.stdin, 'item'):
importer.mep_cache = dict(staff=[], constituencies=[],
importer.mep_cache = dict(staff=[], constituencies=[], chambers=[],
committees=[], groups=[], delegations=[])
importer.manage_mep(data)
# Commenting for now, it's a bit dangerous, if a json file was corrupt it
......
[
{
"fields": {
"last_name": "PIRKER",
"photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg",
"gender": 2,
"remote_id": "2307",
"first_name": "Hubert",
"cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);",
"active": false,
"birth_place": "Gries",
"full_name": "Hubert PIRKER",
"birth_date": "1948-10-03",
"slug": "hubert-pirker"
},
"model": "representatives.representative",
"pk": 1
},
{
"fields": {
"last_name": "LUDVIGSSON",
"photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg",
"gender": 2,
"remote_id": "96673",
"first_name": "Olle",
"cv": "",
"active": true,
"birth_place": "H\u00e4ls\u00f6",
"full_name": "Olle LUDVIGSSON",
"birth_date": "1948-10-28",
"slug": "olle-ludvigsson"
},
"model": "representatives.representative",
"pk": 2
},
{
"fields": {
"email": "olle.ludvigsson@europarl.europa.eu",
"representative": 2,
"kind": "official"
},
"model": "representatives.email",
"pk": 1
},
{
"fields": {
"url": "http://www.sap.se/olle",
"representative": 2,
"kind": ""
},
"model": "representatives.website",
"pk": 1
},
{
"fields": {
"url": "http://twitter.com/olleludvigsson",
"representative": 2,
"kind": "twitter"
},
"model": "representatives.website",
"pk": 2
},
{
"fields": {
"url": "https://www.facebook.com/olle.ludvigsson",
"representative": 2,
"kind": "facebook"
},
"model": "representatives.website",
"pk": 3
},
{
"fields": {
"city": "Brussels",
"name": "Brussels European Parliament",
"country": 1050,
"floor": "14G",
"number": "60",
"street": "rue Wiertz / Wiertzstraat",
"office_number": "257",
"representative": 2,
"location": "",
"kind": "official",
"postcode": "1047"
},
"model": "representatives.address",
"pk": 1
},
{
"fields": {
"city": "Strasbourg",
"name": "Strasbourg European Parliament",
"country": 1095,
"floor": "T07",
"number": "1",
"street": "Av. du Pr\u00e9sident Robert Schuman - CS 91024",
"office_number": "070",
"representative": 2,
"location": "",
"kind": "official",
"postcode": "67070"
},
"model": "representatives.address",
"pk": 2
},
{
"fields": {
"number": "+322 28 45442",
"representative": 2,
"address": 1,
"kind": "office phone"
},
"model": "representatives.phone",
"pk": 1
},
{
"fields": {
"number": "+333 88 1 75442",
"representative": 2,
"address": 2,
"kind": "office phone"
},
"model": "representatives.phone",
"pk": 2
},
{
"fields": {
"country": null,
"name": "European Parliament"
},
"model": "representatives.chamber",
"pk": 1
},
{
"fields": {
"name": "European Parliament",
"kind": "chamber",
"abbreviation": "EP",
"chamber": 1
},
"model": "representatives.group",
"pk": 1
},
{
"fields": {
"name": "Committee on Employment and Social Affairs",
"kind": "committee",
"abbreviation": "EMPL",
"chamber": 1
},
"model": "representatives.group",
"pk": 2
},
{
"fields": {
"name": "Delegation for relations with the countries of Southeast Asia and the Association of Southeast Asian Nations (ASEAN)",
"kind": "delegation",
"abbreviation": "",
"chamber": 1
},
"model": "representatives.group",
"pk": 3
},
{
"fields": {
"name": "Delegation for relations with the Member States of ASEAN, South-east Asia and the Republic of Korea",
"kind": "delegation",
"abbreviation": "",
"chamber": 1
},
"model": "representatives.group",
"pk": 4
},
{
"fields": {
"name": "Group of the European People's Party (Christian Democrats) and European Democrats",
"kind": "group",
"abbreviation": "PPE-DE",
"chamber": 1
},
"model": "representatives.group",
"pk": 5
},
{
"fields": {
"name": "Group of the European People's Party (Christian-Democratic Group)",
"kind": "group",
"abbreviation": "EPP",
"chamber": 1
},
"model": "representatives.group",
"pk": 6
},
{
"fields": {
"name": "Austria",
"kind": "country",
"abbreviation": "AT",
"chamber": null
},
"model": "representatives.group",
"pk": 7
},
{
"fields": {
"name": "Conference of Delegation Chairs",
"kind": "organization",
"abbreviation": "",
"chamber": null
},
"model": "representatives.group",
"pk": 8
},
{
"fields": {
"name": "Committee on Economic and Monetary Affairs",
"kind": "committee",
"abbreviation": "ECON",
"chamber": 1
},
"model": "representatives.group",
"pk": 9
},
{
"fields": {
"name": "Committee on Industry, Research and Energy",
"kind": "committee",
"abbreviation": "ITRE",
"chamber": 1
},
"model": "representatives.group",
"pk": 10
},
{
"fields": {
"name": "Delegation to the EU-Serbia Stabilisation and Association Parliamentary Committee",
"kind": "delegation",
"abbreviation": "",
"chamber": 1
},
"model": "representatives.group",
"pk": 11
},
{
"fields": {
"name": "Delegation for relations with Bosnia and Herzegovina, and Kosovo",
"kind": "delegation",
"abbreviation": "",
"chamber": 1
},
"model": "representatives.group",
"pk": 12
},
{
"fields": {
"name": "Delegation for relations with Australia and New Zealand",
"kind": "delegation",
"abbreviation": "",
"chamber": 1
},
"model": "representatives.group",
"pk": 13
},
{
"fields": {
"name": "Group of the Progressive Alliance of Socialists and Democrats in the European Parliament",
"kind": "group",
"abbreviation": "SD",
"chamber": 1
},
"model": "representatives.group",
"pk": 14
},
{
"fields": {
"name": "Sweden",
"kind": "country",
"abbreviation": "SE",
"chamber": null
},
"model": "representatives.group",
"pk": 15
},
{
"fields": {
"country": null,
"name": "European Parliament"
},
"model": "representatives.constituency",
"pk": 1
},
{
"fields": {
"country": 1043,
"name": "\u00d6sterreichische Volkspartei"
},
"model": "representatives.constituency",
"pk": 2
},
{
"fields": {
"country": 1202,
"name": "Arbetarepartiet- Socialdemokraterna"
},
"model": "representatives.constituency",
"pk": 3
},
{
"fields": {
"group": 2,
"end_date": "1999-07-19",
"role": "Substitute",
"representative": 1,
"link": "",
"begin_date": "1997-01-16",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 1
},
{
"fields": {
"group": 3,
"end_date": "2014-06-30",
"role": "Member",
"representative": 1,
"link": "",
"begin_date": "2013-10-09",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 2
},
{
"fields": {
"group": 4,
"end_date": "1997-01-15",
"role": "Member",
"representative": 1,
"link": "",
"begin_date": "1996-11-14",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 3
},
{
"fields": {
"group": 5,
"end_date": "2004-07-19",
"role": "Member",
"representative": 1,
"link": "",
"begin_date": "1999-07-20",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 4
},
{
"fields": {
"group": 6,
"end_date": "1999-07-19",
"role": "Member",
"representative": 1,
"link": "",
"begin_date": "1996-11-11",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 5
},
{
"fields": {
"group": 7,
"end_date": "2004-07-19",
"role": "",
"representative": 1,
"link": "",
"begin_date": "1999-07-20",
"constituency": 2
},
"model": "representatives.mandate",
"pk": 6
},
{
"fields": {
"group": 1,
"end_date": "2004-07-19",
"role": "",
"representative": 1,
"link": "",
"begin_date": "1999-07-20",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 7
},
{
"fields": {
"group": 7,
"end_date": "1999-07-19",
"role": "",
"representative": 1,
"link": "",
"begin_date": "1996-11-11",
"constituency": 2
},
"model": "representatives.mandate",
"pk": 8
},
{
"fields": {
"group": 1,
"end_date": "1999-07-19",
"role": "",
"representative": 1,
"link": "",
"begin_date": "1996-11-11",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 9
},
{
"fields": {
"group": 8,
"end_date": "2009-07-13",
"role": "Member",
"representative": 1,
"link": "",
"begin_date": "2006-03-21",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 10
},
{
"fields": {
"group": 9,
"end_date": "9999-12-31",
"role": "Member",
"representative": 2,
"link": "",
"begin_date": "2014-07-01",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 11
},
{
"fields": {
"group": 10,
"end_date": "9999-12-31",
"role": "Substitute",
"representative": 2,
"link": "",
"begin_date": "2014-07-08",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 12
},
{
"fields": {
"group": 9,
"end_date": "2012-01-18",
"role": "Substitute",
"representative": 2,
"link": "",
"begin_date": "2009-07-16",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 13
},
{
"fields": {
"group": 11,
"end_date": "9999-12-31",
"role": "Member",
"representative": 2,
"link": "",
"begin_date": "2014-07-14",
"constituency": 1
},
"model": "representatives.mandate",
"pk": 14
},
{
"fields": {