Skip to content
GitLab
Commit c294c6ca authored by Nicolas Joyard's avatar Nicolas Joyard
Browse files

Add chamber import to .contrib.francedata

parent bbb11560
Hide whitespace changes
Inline Side-by-side
representatives/contrib/francedata/import_representatives.py
View file @ c294c6ca
......@@ -12,7 +12,8 @@ from django.db import transaction
from django.utils import timezone
from representatives.models import (Country, Mandate, Email, Address, WebSite,
Representative, Constituency, Phone, Group)
Representative, Constituency, Phone, Group,
Chamber)
logger = logging.getLogger(__name__)
......@@ -52,6 +53,7 @@ def _get_rep_comittees(json):
#
# Mandates are defined as follows
# - 'kind' indicates the group kind, a constant string
# - 'chamber' tells whether the group belongs to the chamber
# - 'from', if present, must be a function that takes the rep json and returns
# an array of dicts; one group will be created from each item in the dict.
# When 'from' is not present, only one group wil be created using the rep
......@@ -66,8 +68,8 @@ def _get_rep_comittees(json):
# and returns the value
#
FranceDataVariants = {
"an": {
"constituency_name": u"Assemblée nationale",
"AN": {
"chamber": u"Assemblée nationale",
"remote_id_field": "id_an",
"mail_domain": "@assemblee-nationale.fr",
"off_city": "Paris",
......@@ -78,29 +80,28 @@ FranceDataVariants = {
"mandates": [
{
"kind": "group",
"chamber": True,
"abbr": "%(groupe_sigle)s",
"name_path": "groupe/organisme",
"start": "%(mandat_debut)s"
},
{
"kind": "party",
"abbr": "%(parti_ratt_financier)s",
"name": "%(parti_ratt_financier)s"
},
{
"kind": "department",
"chamber": True,
"abbr": "%(num_deptmt)s",
"name": "%(nom_circo)s",
"start": "%(mandat_debut)s"
},
{
"kind": "district",
"chamber": True,
"abbr": "%(num_deptmt)s-%(num_circo)d",
"name_fn": _get_rep_district_name,
"start": "%(mandat_debut)s"
},
{
"kind": "parl-group",
"chamber": True,
"from": _get_rep_parl_groups,
"abbr": "%(name)s",
"name": "%(name)s",
......@@ -109,6 +110,7 @@ FranceDataVariants = {
},
{
"kind": "comittee",
"chamber": True,
"from": _get_rep_comittees,
"abbr": "%(name)s",
"name": "%(name)s",
......@@ -137,8 +139,8 @@ def _parse_date(date):
return datetime.strptime(date, "%Y-%m-%d").date()
def _get_or_create_mandate(representative, group, constituency, role='',
begin_date=None, end_date=None):
def _create_mandate(representative, group, constituency, role='',
begin_date=None, end_date=None):
mandate, _ = Mandate.objects.get_or_create(
representative=representative,
group=group,
......@@ -151,8 +153,6 @@ def _get_or_create_mandate(representative, group, constituency, role='',
if _:
logger.debug('Created mandate %s', mandate.pk)
return mandate
def _get_path(dict_, path):
'''
......@@ -176,6 +176,7 @@ class GenericImporter(object):
it saves the given model if it exists, updating its
updated field
'''
instance, created = model.objects.get_or_create(**data)
if not created:
......@@ -194,9 +195,13 @@ class FranceDataImporter(GenericImporter):
def __init__(self, variant):
self.france = Country.objects.get(name="France")
self.variant = FranceDataVariants[variant]
self.variant_constituency, _ = Constituency.objects.get_or_create(
name=self.variant['constituency_name'],
country=self.france)
self.chamber, _ = Chamber.objects.get_or_create(
name=self.variant['chamber'], country=self.france)
self.ch_constituency, _ = Constituency.objects.get_or_create(
name=self.variant['chamber'], country=self.france)
self.ch_group, _ = Group.objects.get_or_create(
name=self.variant['chamber'], kind='chamber', abbreviation=variant,
chamber=self.chamber)
@transaction.atomic
def manage_rep(self, rep_json):
......@@ -268,7 +273,25 @@ class FranceDataImporter(GenericImporter):
Create mandates from rep data based on variant configuration
'''
# Mandate in country group for party constituency
if rep_json.get('parti_ratt_financier'):
constituency, _ = Constituency.objects.get_or_create(
name=rep_json.get('parti_ratt_financier'), country=self.france)
group, _ = self.touch_model(model=Group,
abbreviation=self.france.code,
kind='country',
name=self.france.name)
_create_mandate(representative, group, constituency, 'membre')
# Configurable mandates
for mdef in self.variant['mandates']:
if mdef.get('chamber', False):
chamber = self.chamber
else:
chamber = None
if 'from' in mdef:
elems = mdef['from'](rep_json)
else:
......@@ -281,6 +304,7 @@ class FranceDataImporter(GenericImporter):
group, _ = self.touch_model(model=Group,
abbreviation=abbr,
kind=mdef['kind'],
chamber=chamber,
name=name)
role = _get_mdef_item(mdef, 'role', elem, 'membre')
......@@ -291,11 +315,8 @@ class FranceDataImporter(GenericImporter):
if end is not None:
end = _parse_date(end)
self.rep_cache['groups'].append(
_get_or_create_mandate(representative, group,
self.variant_constituency, role,
start, end)
)
_create_mandate(representative, group, self.ch_constituency,
role, start, end)
logger.debug(
'%s => %s: %s of "%s" (%s) %s-%s' % (rep_json['slug'],
......@@ -379,11 +400,9 @@ def main(stream=None):
if not apps.ready:
django.setup()
importer = FranceDataImporter('an')
importer = FranceDataImporter('AN')
GenericImporter.pre_import(importer)
for data in ijson.items(stream or sys.stdin, ''):
for rep in data:
importer.rep_cache = dict(groups=[], parties=[], departments=[],
districts=[])
importer.manage_rep(rep)
representatives/contrib/francedata/tests/deputes_expected.json
View file @ c294c6ca
......@@ -191,81 +191,98 @@
},
{
"fields": {
"name": "Socialiste, r\u00e9publicain et citoyen",
"kind": "group",
"abbreviation": "SRC"
"country": 1095,
"name": "Assembl\u00e9e nationale"
},
"model": "representatives.chamber",
"pk": 1
},
{
"fields": {
"name": "Assembl\u00e9e nationale",
"kind": "chamber",
"abbreviation": "AN",
"chamber": 1
},
"model": "representatives.group",
"pk": 1
},
{
"fields": {
"name": "Parti socialiste",
"kind": "party",
"abbreviation": "Parti socialiste"
"name": "France",
"kind": "country",
"abbreviation": "FR",
"chamber": null
},
"model": "representatives.group",
"pk": 2
},
{
"fields": {
"name": "Nord",
"kind": "department",
"abbreviation": "59"
"name": "Socialiste, r\u00e9publicain et citoyen",
"kind": "group",
"abbreviation": "SRC",
"chamber": 1
},
"model": "representatives.group",
"pk": 3
},
{
"fields": {
"name": "Nord (1\u00e8re circonscription)",
"kind": "district",
"abbreviation": "59-1"
"name": "Nord",
"kind": "department",
"abbreviation": "59",
"chamber": 1
},
"model": "representatives.group",
"pk": 4
},
{
"fields": {
"name": "Groupe d'amiti\u00e9 france-mexique",
"kind": "parl-group",
"abbreviation": "Groupe d'amiti\u00e9 france-mexique"
"name": "Nord (1\u00e8re circonscription)",
"kind": "district",
"abbreviation": "59-1",
"chamber": 1
},
"model": "representatives.group",
"pk": 5
},
{
"fields": {
"name": "Groupe d'amiti\u00e9 france-argentine",
"name": "Groupe d'amiti\u00e9 france-mexique",
"kind": "parl-group",
"abbreviation": "Groupe d'amiti\u00e9 france-argentine"
"abbreviation": "Groupe d'amiti\u00e9 france-mexique",
"chamber": 1
},
"model": "representatives.group",
"pk": 6
},
{
"fields": {
"name": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique",
"kind": "comittee",
"abbreviation": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique"
"name": "Groupe d'amiti\u00e9 france-argentine",
"kind": "parl-group",
"abbreviation": "Groupe d'amiti\u00e9 france-argentine",
"chamber": 1
},
"model": "representatives.group",
"pk": 7
},
{
"fields": {
"name": "Les r\u00e9publicains",
"kind": "group",
"abbreviation": "LES-REP"
"name": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique",
"kind": "comittee",
"abbreviation": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique",
"chamber": 1
},
"model": "representatives.group",
"pk": 8
},
{
"fields": {
"name": "Les R\u00e9publicains",
"kind": "party",
"abbreviation": "Les R\u00e9publicains"
"name": "Les r\u00e9publicains",
"kind": "group",
"abbreviation": "LES-REP",
"chamber": 1
},
"model": "representatives.group",
"pk": 9
......@@ -274,7 +291,8 @@
"fields": {
"name": "Is\u00e8re",
"kind": "department",
"abbreviation": "38"
"abbreviation": "38",
"chamber": 1
},
"model": "representatives.group",
"pk": 10
......@@ -283,7 +301,8 @@
"fields": {
"name": "Is\u00e8re (7\u00e8me circonscription)",
"kind": "district",
"abbreviation": "38-7"
"abbreviation": "38-7",
"chamber": 1
},
"model": "representatives.group",
"pk": 11
......@@ -292,7 +311,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes \u00c9conomie verte et \u00e9conomie circulaire",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes \u00c9conomie verte et \u00e9conomie circulaire"
"abbreviation": "Groupe d'\u00e9tudes \u00c9conomie verte et \u00e9conomie circulaire",
"chamber": 1
},
"model": "representatives.group",
"pk": 12
......@@ -301,7 +321,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes am\u00e9nagement du territoire",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes am\u00e9nagement du territoire"
"abbreviation": "Groupe d'\u00e9tudes am\u00e9nagement du territoire",
"chamber": 1
},
"model": "representatives.group",
"pk": 13
......@@ -310,7 +331,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes politique de l'eau",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes politique de l'eau"
"abbreviation": "Groupe d'\u00e9tudes politique de l'eau",
"chamber": 1
},
"model": "representatives.group",
"pk": 14
......@@ -319,7 +341,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes coordination des droits europ\u00e9ens",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes coordination des droits europ\u00e9ens"
"abbreviation": "Groupe d'\u00e9tudes coordination des droits europ\u00e9ens",
"chamber": 1
},
"model": "representatives.group",
"pk": 15
......@@ -328,7 +351,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes automobile",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes automobile"
"abbreviation": "Groupe d'\u00e9tudes automobile",
"chamber": 1
},
"model": "representatives.group",
"pk": 16
......@@ -337,7 +361,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes pr\u00e9vention et lutte contre la toxicomanie",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes pr\u00e9vention et lutte contre la toxicomanie"
"abbreviation": "Groupe d'\u00e9tudes pr\u00e9vention et lutte contre la toxicomanie",
"chamber": 1
},
"model": "representatives.group",
"pk": 17
......@@ -346,7 +371,8 @@
"fields": {
"name": "Groupe d'amiti\u00e9 france-r\u00e9publique d'autriche",
"kind": "parl-group",
"abbreviation": "Groupe d'amiti\u00e9 france-r\u00e9publique d'autriche"
"abbreviation": "Groupe d'amiti\u00e9 france-r\u00e9publique d'autriche",
"chamber": 1
},
"model": "representatives.group",
"pk": 18
......@@ -355,7 +381,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes chr\u00e9tiens d'orient",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes chr\u00e9tiens d'orient"
"abbreviation": "Groupe d'\u00e9tudes chr\u00e9tiens d'orient",
"chamber": 1
},
"model": "representatives.group",
"pk": 19
......@@ -364,7 +391,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes granit, pierres naturelles, carri\u00e8res et mat\u00e9riaux de construction",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes granit, pierres naturelles, carri\u00e8res et mat\u00e9riaux de construction"
"abbreviation": "Groupe d'\u00e9tudes granit, pierres naturelles, carri\u00e8res et mat\u00e9riaux de construction",
"chamber": 1
},
"model": "representatives.group",
"pk": 20
......@@ -373,7 +401,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes \u00c9levage",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes \u00c9levage"
"abbreviation": "Groupe d'\u00e9tudes \u00c9levage",
"chamber": 1
},
"model": "representatives.group",
"pk": 21
......@@ -382,7 +411,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes sectes",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes sectes"
"abbreviation": "Groupe d'\u00e9tudes sectes",
"chamber": 1
},
"model": "representatives.group",
"pk": 22
......@@ -391,7 +421,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes famille et adoption",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes famille et adoption"
"abbreviation": "Groupe d'\u00e9tudes famille et adoption",
"chamber": 1
},
"model": "representatives.group",
"pk": 23
......@@ -400,7 +431,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes enseignement priv\u00e9 sous contrat et hors contrat",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes enseignement priv\u00e9 sous contrat et hors contrat"
"abbreviation": "Groupe d'\u00e9tudes enseignement priv\u00e9 sous contrat et hors contrat",
"chamber": 1
},
"model": "representatives.group",
"pk": 24
......@@ -409,7 +441,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes pme",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes pme"
"abbreviation": "Groupe d'\u00e9tudes pme",
"chamber": 1
},
"model": "representatives.group",
"pk": 25
......@@ -418,7 +451,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes r\u00e9publique et religions",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes r\u00e9publique et religions"
"abbreviation": "Groupe d'\u00e9tudes r\u00e9publique et religions",
"chamber": 1
},
"model": "representatives.group",
"pk": 26
......@@ -427,7 +461,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes industrie chimique",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes industrie chimique"
"abbreviation": "Groupe d'\u00e9tudes industrie chimique",
"chamber": 1
},
"model": "representatives.group",
"pk": 27
......@@ -436,7 +471,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes antis\u00e9mitisme",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes antis\u00e9mitisme"
"abbreviation": "Groupe d'\u00e9tudes antis\u00e9mitisme",
"chamber": 1
},
"model": "representatives.group",
"pk": 28
......@@ -445,7 +481,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes alimentation et sant\u00e9",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes alimentation et sant\u00e9"
"abbreviation": "Groupe d'\u00e9tudes alimentation et sant\u00e9",
"chamber": 1
},
"model": "representatives.group",
"pk": 29
......@@ -454,7 +491,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes politiques de la ruralit\u00e9",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes politiques de la ruralit\u00e9"
"abbreviation": "Groupe d'\u00e9tudes politiques de la ruralit\u00e9",
"chamber": 1
},
"model": "representatives.group",
"pk": 30
......@@ -463,7 +501,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes m\u00e9dicaments et produits de sant\u00e9",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes m\u00e9dicaments et produits de sant\u00e9"
"abbreviation": "Groupe d'\u00e9tudes m\u00e9dicaments et produits de sant\u00e9",
"chamber": 1
},
"model": "representatives.group",
"pk": 31
......@@ -472,7 +511,8 @@
"fields": {
"name": "Groupe d'\u00e9tudes d\u00e9veloppement de la m\u00e9thanisation",
"kind": "parl-group",
"abbreviation": "Groupe d'\u00e9tudes d\u00e9veloppement de la m\u00e9thanisation"
"abbreviation": "Groupe d'\u00e9tudes d\u00e9veloppement de la m\u00e9thanisation",
"chamber": 1
},
"model": "representatives.group",
"pk": 32
......@@ -481,7 +521,8 @@
"fields": {
"name": "Commission des affaires sociales",
"kind": "comittee",
"abbreviation": "Commission des affaires sociales"
"abbreviation": "Commission des affaires sociales",
"chamber": 1
},
"model": "representatives.group",
"pk": 33
......@@ -496,23 +537,39 @@
},
{
"fields": {
"group": 1,
"country": 1095,
"name": "Parti socialiste"
},
"model": "representatives.constituency",
"pk": 2
},
{
"fields": {
"country": 1095,
"name": "Les R\u00e9publicains"
},
"model": "representatives.constituency",
"pk": 3
},
{
"fields": {
"group": 2,
"end_date": null,
"role": "membre",
"representative": 1,
"begin_date": "2012-06-20",
"constituency": 1
"begin_date": null,
"constituency": 2
},
"model": "representatives.mandate",
"pk": 1
},
{
"fields": {
"group": 2,
"group": 3,
"end_date": null,
"role": "membre",
"representative": 1,
"begin_date": null,
"begin_date": "2012-06-20",
"constituency": 1
},
"model": "representatives.mandate",
......@@ -520,7 +577,7 @@
},
{
"fields": {
"group": 3,
"group": 4,
"end_date": null,
"role": "membre",
"representative": 1,
......@@ -532,7 +589,7 @@
},
{
"fields": {
"group": 4,
"group": 5,
"end_date": null,
"role": "membre",
"representative": 1,
......@@ -544,7 +601,7 @@
},
{
"fields": {
"group": 5,
"group": 6,
"end_date": null,
"role": "vice-pr\u00e9sident",
"representative": 1,
......@@ -556,7 +613,7 @@
},
{