Add import from FranceData

.travis.yml

@@ -13,6 +13,7 @@ script:
 - pep8 representatives/ --exclude migrations --ignore E128
 - flake8 representatives/ --exclude migrations --ignore E128
 - django-admin migrate
+- cat representatives/contrib/francedata/tests/deputes_input.json | francedata_import_representatives
 - cat representatives/contrib/parltrack/tests/representatives_fixture.json | parltrack_import_representatives
 - py.test
 after_success:

representatives/contrib/francedata/import_representatives.py

+# coding: utf-8
+
+import logging
+import sys
+from datetime import datetime
+
+import django.dispatch
+import ijson
+import django
+from django.apps import apps
+from django.db import transaction
+from django.utils import timezone
+
+from representatives.models import (Group, Mandate, Representative, Email,
+                                    Constituency, WebSite, Phone)
+
+logger = logging.getLogger(__name__)
+
+representative_pre_import = django.dispatch.Signal(
+    providing_args=['representative_data'])
+
+
+def _get_rep_district_name(json):
+    num = json.get('num_circo')
+    nom = json.get('nom_circo')
+    ordinal = u'ère' if num == 1 else u'ème'
+    return '%s (%d%s circonscription)' % (nom, num, ordinal)
+
+
+def _get_rep_parl_groups(json):
+    return [{
+        'name': g['responsabilite']['organisme'],
+        'role': g['responsabilite']['fonction'],
+        'start': json['mandat_debut']
+    } for g in json['groupes_parlementaires']]
+
+
+def _get_rep_comittees(json):
+    return [{
+        'name': g['responsabilite']['organisme'],
+        'role': g['responsabilite']['fonction'],
+        'start': json['mandat_debut']
+    } for g in json['responsabilites']
+        if g['responsabilite']['organisme'].startswith('Commission')]
+
+
+#
+# Variant configuration
+# - mail_domain is used to distinguish official vs personal emails
+# - mandates defines how mandates are created from the rep json
+#
+# Mandates are defined as follows
+# - 'kind' indicates the group kind, a constant string
+# - 'from', if present, must be a function that takes the rep json and returns
+#   an array of dicts; one group will be created from each item in the dict.
+#   When 'from' is not present, only one group wil be created using the rep
+#   json (IOW, 'from' defaults to lambda repjson: [repjson])
+# - 'name', 'abbr', 'role', 'start', 'end' are strings that are interpolated
+#   against the rep json or items returned by 'from'.
+# - 'name_path', 'abbr_path', etc. can replace 'name', 'abbr'... by specifying
+#   a slash-separated dictionnary path where the value is to be found in the
+#   rep json or item returned by 'from'
+# - 'name_fn', 'abbr_fn', etc. can also replace 'name', 'abbr'... by a
+#   function that takes the input item (rep json or item returned by 'from')
+#   and returns the value
+#
+FranceDataVariants = {
+    "an": {
+        "constituency_name": "Assemblée Nationale",
+        "remote_id_field": "id_an",
+        "mail_domain": "@assemblee-nationale.fr",
+        "mandates": [
+            {
+                "kind": "group",
+                "abbr": "%(groupe_sigle)s",
+                "name_path": "groupe/organisme",
+                "start": "%(mandat_debut)s"
+            },
+            {
+                "kind": "party",
+                "abbr": "%(parti_ratt_financier)s",
+                "name": "%(parti_ratt_financier)s"
+            },
+            {
+                "kind": "department",
+                "abbr": "%(num_deptmt)s",
+                "name": "%(nom_circo)s",
+                "start": "%(mandat_debut)s"
+            },
+            {
+                "kind": "district",
+                "abbr": "%(num_deptmt)s-%(num_circo)d",
+                "name_fn": _get_rep_district_name,
+                "start": "%(mandat_debut)s"
+            },
+            {
+                "kind": "parl-group",
+                "from": _get_rep_parl_groups,
+                "abbr": "%(name)s",
+                "name": "%(name)s",
+                "role": "%(role)s",
+                "start": "%(start)s"
+            },
+            {
+                "kind": "comittee",
+                "from": _get_rep_comittees,
+                "abbr": "%(name)s",
+                "name": "%(name)s",
+                "role": "%(role)s",
+                "start": "%(start)s"
+            }
+        ]
+    }
+}
+
+
+def _get_mdef_item(mdef, item, json, default=None):
+    if item in mdef:
+        return mdef[item] % json
+
+    if '%s_path' % item in mdef:
+        return _get_path(json, mdef['%s_path' % item])
+
+    if '%s_fn' % item in mdef:
+        return mdef['%s_fn' % item](json)
+
+    return default
+
+
+def _parse_date(date):
+    return datetime.strptime(date, "%Y-%m-%d").date()
+
+
+def _get_or_create_mandate(representative, group, constituency, role='',
+                          begin_date=None, end_date=None):
+    mandate, _ = Mandate.objects.get_or_create(
+        representative=representative,
+        group=group,
+        constituency=constituency,
+        role=role,
+        begin_date=begin_date,
+        end_date=end_date
+    )
+
+    if _:
+        logger.debug('Created mandate %s', mandate.pk)
+
+    return mandate
+
+
+def _get_path(dict_, path):
+    '''
+    Get value at specific path in dictionary. Path is specified by slash-
+    separated string, eg _get_path(foo, 'bar/baz') returns foo['bar']['baz']
+    '''
+    cur = dict_
+    for part in path.split('/'):
+        cur = cur[part]
+    return cur
+
+
+class GenericImporter(object):
+
+    def pre_import(self):
+        self.import_start_datetime = timezone.now()
+
+    def touch_model(self, model, **data):
+        '''
+        This method create or look up a model with the given data
+        it saves the given model if it exists, updating its
+        updated field
+        '''
+        instance, created = model.objects.get_or_create(**data)
+
+        if not created:
+            if instance.updated < self.import_start_datetime:
+                instance.save()     # Updates updated field
+
+        return (instance, created)
+
+
+class FranceDataImporter(GenericImporter):
+    url = 'http://francedata.future/data/deputes.json'
+
+    def parse_date(self, date):
+        return _parse_date(date)
+
+    def __init__(self, variant):
+        self.variant = FranceDataVariants[variant]
+        self.variant_constituency, _ = Constituency.objects.get_or_create(
+            name=self.variant['constituency_name'])
+
+    @transaction.atomic
+    def manage_rep(self, rep_json):
+        '''
+        Import a rep as a representative from the json dict fetched from
+        FranceData (which comes from nosdeputes.fr)
+        '''
+        remote_id = rep_json[self.variant['remote_id_field']]
+
+        if not remote_id:
+            logger.warning('Skipping MEP without UID %s %s',
+                           rep_json['nom'],
+                           rep_json[self.variant['remote_id_field']])
+            return
+
+        # Some versions of memopol will connect to this and skip inactive reps.
+        responses = representative_pre_import.send(sender=self,
+                representative_data=rep_json)
+
+        for receiver, response in responses:
+            if response is False:
+                logger.debug(
+                    'Skipping MEP %s', rep_json['nom'])
+                return
+
+        try:
+            representative = Representative.objects.get(remote_id=remote_id)
+        except Representative.DoesNotExist:
+            representative = Representative(remote_id=remote_id)
+
+        # Save representative attributes
+        self.import_representative_details(representative, rep_json)
+
+        representative.save()
+
+        self.add_mandates(representative, rep_json)
+
+        self.add_contacts(representative, rep_json)
+
+        logger.debug('Imported MEP %s', unicode(representative))
+
+        return representative
+
+    def import_representative_details(self, representative, rep_json):
+        representative.active = True
+
+        if rep_json.get("date_naissance"):
+            representative.birth_date = _parse_date(rep_json["date_naissance"])
+        if rep_json.get("lieu_naissance"):
+            representative.birth_place = rep_json["lieu_naissance"]
+
+        representative.photo = rep_json['photo_url']
+        representative.full_name = rep_json["nom"]
+
+        gender_convertion_dict = {
+            u"F": 1,
+            u"H": 2
+        }
+        if 'sexe' in rep_json:
+            representative.gender = gender_convertion_dict.get(rep_json[
+                                                               'sexe'], 0)
+        else:
+            representative.gender = 0
+
+        representative.slug = rep_json['slug']
+
+    def add_mandates(self, representative, rep_json):
+        '''
+        Create mandates from rep data based on variant configuration
+        '''
+
+        for mdef in self.variant['mandates']:
+            if 'from' in mdef:
+                elems = mdef['from'](rep_json)
+            else:
+                elems = [rep_json]
+
+            for elem in elems:
+                name = _get_mdef_item(mdef, 'name', elem, '')
+                abbr = _get_mdef_item(mdef, 'abbr', elem, '')
+
+                group, _ = self.touch_model(model=Group,
+                                            abbreviation=abbr,
+                                            kind=mdef['kind'],
+                                            name=name)
+
+                role = _get_mdef_item(mdef, 'role', elem, 'membre')
+                start = _get_mdef_item(mdef, 'start', elem, None)
+                if start is not None:
+                    start = _parse_date(start)
+                end = _get_mdef_item(mdef, 'end', elem, None)
+                if end is not None:
+                    end = _parse_date(end)
+
+                self.rep_cache['groups'].append(
+                    _get_or_create_mandate(representative, group,
+                                          self.variant_constituency, role,
+                                          start, end)
+                )
+
+                logger.debug(
+                    '%s => %s: %s of "%s" (%s) %s-%s' % (rep_json['slug'],
+                    mdef['kind'], role, name, abbr, start, end))
+
+    def add_contacts(self, representative, rep_json):
+        # Websites
+        websites = rep_json.get('sites_web', [])
+        for site in websites:
+            if not site['site'].startswith('http://twitter.com/'):
+                self.touch_model(model=WebSite,
+                                 url=site['site'],
+                                 representative=representative
+                                 )
+
+        # Twitter
+        if rep_json.get('twitter'):
+            tid = rep_json.get('twitter')
+            self.touch_model(model=WebSite,
+                             representative=representative,
+                             kind='twitter',
+                             url='http://twitter.com/%s' % tid
+                             )
+
+        # E-mails
+        emails = rep_json.get('emails', [])
+        for email in emails:
+            mail = email['email']
+            self.touch_model(
+                model=Email,
+                representative=representative,
+                kind=('official' if mail.endswith(self.variant['mail_domain'])
+                    else 'other'),
+                email=mail)
+
+        # Addresses & phone numbers
+        addresses = rep_json.get('adresses', [])
+        for ad in addresses:
+            if 'tel' in ad:
+                self.touch_model(model=Phone,
+                                 representative=representative,
+                                 kind='', number=ad['tel']
+                                 )
+
+
+def main(stream=None):
+    if not apps.ready:
+        django.setup()
+
+    importer = FranceDataImporter('an')
+    GenericImporter.pre_import(importer)
+
+    for data in ijson.items(stream or sys.stdin, ''):
+        for rep in data:
+            importer.rep_cache = dict(groups=[], parties=[], departments=[],
+                                      districts=[])
+            importer.manage_rep(rep)

representatives/contrib/francedata/tests/deputes_expected.json

+[
+{
+    "fields": {
+        "photo": "http://www.nosdeputes.fr/depute/photo/bernard-roman",
+        "gender": 2,
+        "remote_id": "2611",
+        "active": true,
+        "birth_place": "Lille (Nord)",
+        "full_name": "Bernard Roman",
+        "birth_date": "1952-07-15",
+        "slug": "bernard-roman"
+    },
+    "model": "representatives.representative",
+    "pk": 1
+},
+{
+    "fields": {
+        "photo": "http://www.nosdeputes.fr/depute/photo/jean-pierre-barbier",
+        "gender": 2,
+        "remote_id": "606888",
+        "active": true,
+        "birth_place": "Bron (Rh\u00f4ne)",
+        "full_name": "Jean-Pierre Barbier",
+        "birth_date": "1960-11-11",
+        "slug": "jean-pierre-barbier"
+    },
+    "model": "representatives.representative",
+    "pk": 2
+},
+{
+    "fields": {
+        "email": "contact@bernard-roman.org",
+        "representative": 1,
+        "kind": "other"
+    },
+    "model": "representatives.email",
+    "pk": 1
+},
+{
+    "fields": {
+        "email": "broman@assemblee-nationale.fr",
+        "representative": 1,
+        "kind": "official"
+    },
+    "model": "representatives.email",
+    "pk": 2
+},
+{
+    "fields": {
+        "email": "jpbarbier.depute@orange.fr",
+        "representative": 2,
+        "kind": "other"
+    },
+    "model": "representatives.email",
+    "pk": 3
+},
+{
+    "fields": {
+        "email": "jpbarbier@assemblee-nationale.fr",
+        "representative": 2,
+        "kind": "official"
+    },
+    "model": "representatives.email",
+    "pk": 4
+},
+{
+    "fields": {
+        "url": "http://www.bernard-roman.net",
+        "representative": 1,
+        "kind": ""
+    },
+    "model": "representatives.website",
+    "pk": 1
+},
+{
+    "fields": {
+        "url": "https://twitter.com/bernardroman59",
+        "representative": 1,
+        "kind": ""
+    },
+    "model": "representatives.website",
+    "pk": 2
+},
+{
+    "fields": {
+        "url": "http://twitter.com/bernardroman59",
+        "representative": 1,
+        "kind": "twitter"
+    },
+    "model": "representatives.website",
+    "pk": 3
+},
+{
+    "fields": {
+        "url": "http://www.jeanpierrebarbier.fr",
+        "representative": 2,
+        "kind": ""
+    },
+    "model": "representatives.website",
+    "pk": 4
+},
+{
+    "fields": {
+        "number": "03 20 52 09 20",
+        "representative": 1,
+        "address": null,
+        "kind": ""
+    },
+    "model": "representatives.phone",
+    "pk": 1
+},
+{
+    "fields": {
+        "number": "04 74 59 76 76",
+        "representative": 2,
+        "address": null,
+        "kind": ""
+    },
+    "model": "representatives.phone",
+    "pk": 2
+},
+{
+    "fields": {
+        "name": "Socialiste, r\u00e9publicain et citoyen",
+        "kind": "group",
+        "abbreviation": "SRC"
+    },
+    "model": "representatives.group",
+    "pk": 1
+},
+{
+    "fields": {
+        "name": "Parti socialiste",
+        "kind": "party",
+        "abbreviation": "Parti socialiste"
+    },
+    "model": "representatives.group",
+    "pk": 2
+},
+{
+    "fields": {
+        "name": "Nord",
+        "kind": "department",
+        "abbreviation": "59"
+    },
+    "model": "representatives.group",
+    "pk": 3
+},
+{
+    "fields": {
+        "name": "Nord (1\u00e8re circonscription)",
+        "kind": "district",
+        "abbreviation": "59-1"
+    },
+    "model": "representatives.group",
+    "pk": 4
+},
+{
+    "fields": {
+        "name": "Groupe d'amiti\u00e9 france-mexique",
+        "kind": "parl-group",
+        "abbreviation": "Groupe d'amiti\u00e9 france-mexique"
+    },
+    "model": "representatives.group",
+    "pk": 5
+},
+{
+    "fields": {
+        "name": "Groupe d'amiti\u00e9 france-argentine",
+        "kind": "parl-group",
+        "abbreviation": "Groupe d'amiti\u00e9 france-argentine"
+    },
+    "model": "representatives.group",
+    "pk": 6
+},
+{
+    "fields": {
+        "name": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique",
+        "kind": "comittee",
+        "abbreviation": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique"
+    },
+    "model": "representatives.group",
+    "pk": 7
+},
+{
+    "fields": {
+        "name": "Les r\u00e9publicains",
+        "kind": "group",
+        "abbreviation": "LES-REP"
+    },
+    "model": "representatives.group",
+    "pk": 8
+},
+{
+    "fields": {
+        "name": "Les R\u00e9publicains",
+        "kind": "party",
+        "abbreviation": "Les R\u00e9publicains"
+    },
+    "model": "representatives.group",
+    "pk": 9
+},
+{
+    "fields": {
+        "name": "Is\u00e8re",
+        "kind": "department",
+        "abbreviation": "38"
+    },
+    "model": "representatives.group",
+    "pk": 10
+},
+{
+    "fields": {
+        "name": "Is\u00e8re (7\u00e8me circonscription)",
+        "kind": "district",
+        "abbreviation": "38-7"
+    },
+    "model": "representatives.group",
+    "pk": 11
+},
+{
+    "fields": {
+        "name": "Groupe d'\u00e9tudes \u00c9conomie verte et \u00e9conomie circulaire",
+        "kind": "parl-group",
+        "abbreviation": "Groupe d'\u00e9tudes \u00c9conomie verte et \u00e9conomie circulaire"
+    },
+    "model": "representatives.group",
+    "pk": 12
+},
+{
+    "fields": {
+        "name": "Groupe d'\u00e9tudes am\u00e9nagement du territoire",