Commit d54e7b98 authored by Nicolas Joyard's avatar Nicolas Joyard

Update to new francedata schema

parent 6cac0520
......@@ -12,31 +12,64 @@ from representatives_votes.models import Dossier
logger = logging.getLogger(__name__)
def parse_dossier_data(data):
def find_dossier(data):
'''
Find dossier with reference matching either 'url_an' or 'url_sen',
create it if not found. Ensure its reference and source are 'url_an' if
both fields are present.
'''
changed = False
ref = data['uri']
dossier = None
reffield = None
try:
dossier = Dossier.objects.get(reference=ref)
except Dossier.DoesNotExist:
dossier = Dossier(reference=ref)
logger.debug('Created dossier %s' % ref)
for field in [k for k in ('url_an', 'url_sen') if k in data]:
try:
dossier = Dossier.objects.get(reference=data[field])
reffield = field
break
except Dossier.DoesNotExist:
pass
if dossier is None:
reffield = 'url_an' if 'url_an' in data else 'url_sen'
dossier = Dossier(reference=data[reffield])
logger.debug('Created dossier %s' % data[reffield])
changed = True
if 'url_an' in data and reffield != 'url_an':
logger.debug('Changed dossier reference to %s' % data['url_an'])
dossier.reference = data['url_an']
changed = True
return dossier, changed
def parse_dossier_data(data):
dossier, changed = find_dossier(data)
thisurl = data['url_an' if data['chambre'] == 'AN' else 'url_sen']
if dossier.reference != dossier.link:
logger.debug('Changed dossier link to %s' % dossier.reference)
dossier.link = dossier.reference
changed = True
title = data['titre']
if dossier.title != title:
if dossier.reference == thisurl and dossier.title != title:
logger.debug('Changed dossier title to %s' % title)
dossier.title = title
changed = True
source = data['url']
if dossier.link != source:
logger.debug('Changed dossier link to %s' % source)
dossier.link = source
changed = True
if 'url_an' in data and 'url_sen' in data:
ext_link = data['url_sen']
if dossier.ext_link != ext_link:
logger.debug('Changed dossier ext. link to %s' % ext_link)
dossier.ext_link = ext_link
changed = True
if changed:
logger.debug('Saved dossier %s' % ref)
logger.debug('Saved dossier %s' % dossier.reference)
dossier.save()
......
......@@ -48,28 +48,35 @@ def _get_unique_title(proposal_pk, candidate):
class ScrutinImporter:
dossiers = None
dossiers_ref = None
dossiers_ext = None
def get_dossier(self, ref):
if self.dossiers is None:
self.dossiers = {
def get_dossier(self, url):
if self.dossiers_ref is None:
self.dossiers_ref = {
d[0]: d[1] for d in Dossier.objects.values_list('reference',
'pk')
}
return self.dossiers.get(ref, None)
if self.dossiers_ext is None:
self.dossiers_ext = {
d[0]: d[1] for d in Dossier.objects.exclude(ext_link='')
.values_list('ext_link', 'pk')
}
return self.dossiers_ref.get(url, self.dossiers_ext.get(url, None))
def parse_scrutin_data(self, data):
ref = data['uri']
ref = data['url']
if 'dossier_uri' not in data:
if 'dossier_url' not in data:
logger.debug('Cannot create proposal without dossier')
return
dossier = self.get_dossier(data['dossier_uri'])
dossier = self.get_dossier(data['dossier_url'])
if dossier is None:
logger.debug('Cannot create proposal for unknown dossier %s'
% data['dossier_uri'])
% data['dossier_url'])
return
changed = False
......@@ -84,7 +91,7 @@ class ScrutinImporter:
values = dict(
title=_get_unique_title(proposal.pk, data["objet"]),
datetime=_parse_date(data["date"]),
dossier_id=self.get_dossier(data['dossier_uri']),
dossier_id=dossier,
kind='dossier'
)
......
......@@ -14,7 +14,8 @@ logger = logging.getLogger(__name__)
class VotesImporter:
deputes = None
deputes_slug = None
deputes_rid = None
scrutins = None
touched = []
......@@ -24,15 +25,24 @@ class VotesImporter:
abstention="abstain"
)
def get_depute(self, prenom, nom):
if self.deputes is None:
self.deputes = {
def get_depute_by_name(self, prenom, nom):
if self.deputes_slug is None:
self.deputes_slug = {
slugify(r[0]): r[1] for r in
Representative.objects.values_list('full_name', 'pk')
}
full = (u'%s %s' % (prenom, nom)).replace(u' ', ' ')
return self.deputes.get(slugify(full), None)
return self.deputes_slug.get(slugify(full), None)
def get_depute_by_url(self, url):
if self.deputes_rid is None:
self.deputes_rid = {
r[0]: r[1] for r in
Representative.objects.values_list('remote_id', 'pk')
}
return self.deputes_rid.get(url, None)
def get_scrutin(self, ref):
if self.scrutins is None:
......@@ -44,16 +54,21 @@ class VotesImporter:
return self.scrutins.get(ref, None)
def parse_vote_data(self, data):
scrutin = self.get_scrutin(data['scrutin_uri'])
scrutin = self.get_scrutin(data['scrutin_url'])
if scrutin is None:
logger.debug('Cannot import vote for unknown scrutin %s'
% data['scrutin_uri'])
% data['scrutin_url'])
return
depute = self.get_depute(data['prenom'], data['nom'])
if 'parl_url' in data:
repdesc = data['parl_url']
depute = self.get_depute_by_url(data['parl_url'])
else:
repdesc = '%s %s' % (data['prenom'], data['nom'])
depute = self.get_depute_by_name(data['prenom'], data['nom'])
if depute is None:
logger.debug('Cannot import vote by unknown rep %s %s'
% (data['prenom'], data['nom']))
logger.debug('Cannot import vote by unknown rep %s' % repdesc)
return
if not data['division'].lower() in self.positions:
......
......@@ -2,11 +2,12 @@
{
"fields": {
"updated": "2016-02-14T13:16:31.417Z",
"reference": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"reference": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"text": "",
"created": "2016-02-14T13:16:31.417Z",
"link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"ext_link": "",
"fingerprint": "5d1707e6663bb28d0308cdb36e9e91c5f235f8a1"
},
"model": "representatives_votes.dossier",
......@@ -15,14 +16,29 @@
{
"fields": {
"updated": "2016-02-14T13:16:31.428Z",
"reference": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"reference": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"text": "",
"created": "2016-02-14T13:16:31.428Z",
"link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"ext_link": "http://www.senat.fr/dossier-legislatif/pjl12-495.html",
"fingerprint": "c03f5e32f66e5f03ebe0a5d100f2f4ade941accc"
},
"model": "representatives_votes.dossier",
"pk": 2
},
{
"fields": {
"updated": "2016-02-21T14:34:35.721Z",
"reference": "http://www.senat.fr/dossier-legislatif/ppl13-799.html",
"title": "Protection de l'enfant",
"text": "",
"created": "2016-02-21T14:34:35.721Z",
"link": "http://www.senat.fr/dossier-legislatif/ppl13-799.html",
"ext_link": "",
"fingerprint": "5c58ebd00b9614e0965f173c8acb2bb77eb28f00"
},
"model": "representatives_votes.dossier",
"pk": 3
}
]
[
{
"url": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"titre": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp"
"chambre": "AN",
"url_an": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"titre": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9"
},
{
"url": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"titre": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"uri": "/14/dossiers/action_publique_territoriale_metropoles.asp"
"chambre": "SEN",
"url_sen": "http://www.senat.fr/dossier-legislatif/pjl12-495.html",
"titre": "Modernisation de l'action publique territoriale et affirmation des m\u00e9tropoles"
},
{
"chambre": "AN",
"url_an": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"url_sen": "http://www.senat.fr/dossier-legislatif/pjl12-495.html",
"titre": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles"
},
{
"chambre": "SEN",
"url_sen": "http://www.senat.fr/dossier-legislatif/ppl13-799.html",
"titre": "Protection de l'enfant"
}
]
\ No newline at end of file
......@@ -4,7 +4,7 @@
"updated": "2016-02-14T13:44:37.550Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:44:37.550Z",
......@@ -22,7 +22,7 @@
"updated": "2016-02-14T13:44:37.578Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740-2",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:44:37.578Z",
......@@ -40,7 +40,7 @@
"updated": "2016-02-14T13:44:37.587Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/748",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.587Z",
......@@ -58,7 +58,7 @@
"updated": "2016-02-14T13:44:37.596Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/747",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"title": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.596Z",
......
[
{
"chambre": "AN",
"objet": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"uri": "/scrutins/detail/(legislature)/14/(num)/740",
"url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"numero": "740",
"dossier_uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"date": "2013-12-05"
},
{
"chambre": "AN",
"objet": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"uri": "/scrutins/detail/(legislature)/14/(num)/740-2",
"url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"numero": "740-2",
"dossier_uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"date": "2013-12-06"
},
{
"chambre": "AN",
"objet": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"uri": "/scrutins/detail/(legislature)/14/(num)/748",
"url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"numero": "748",
"dossier_uri": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"date": "2013-12-11"
},
{
"chambre": "AN",
"objet": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"uri": "/scrutins/detail/(legislature)/14/(num)/747",
"url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"numero": "747",
"dossier_uri": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"date": "2013-12-11"
},
{
"chambre": "AN",
"objet": "Dossier inexistant.",
"url": "http://www2.assemblee-nationale.fr/404",
"uri": "/scrutins/404",
"url": "http://www.assemblee-nationale.fr/404",
"numero": "000",
"dossier_uri": "/14/dossiers/inexistant",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/inexistant",
"dossier_url": "http://www.assemblee-nationale.fr/14/dossiers/inexistant",
"date": "2099-12-11"
}
]
\ No newline at end of file
[
{
"fields": {
"updated": "2016-02-14T13:58:46.991Z",
"updated": "2016-02-14T13:44:37.550Z",
"total_for": 1,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:58:00.536Z",
"created": "2016-02-14T13:44:37.550Z",
"kind": "dossier",
"datetime": "2013-12-04T23:00:00Z",
"total_against": 0,
"fingerprint": "73ca11e690cc6db1d806f927538d05b49eb6fd9d",
"fingerprint": "40bb927c36b00bb688c1d7e7f4be5b9a1aae4af3",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
......@@ -19,17 +19,17 @@
},
{
"fields": {
"updated": "2016-02-14T13:58:47.003Z",
"updated": "2016-02-14T13:44:37.578Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740-2",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:58:00.551Z",
"created": "2016-02-14T13:44:37.578Z",
"kind": "dossier",
"datetime": "2013-12-05T23:00:00Z",
"total_against": 1,
"fingerprint": "92ece2396838f1612ada0d896cbdce8fe5deaf13",
"fingerprint": "a8709fb12e8e6e4a5f46931d855bf70453dd7fd2",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
......@@ -37,17 +37,17 @@
},
{
"fields": {
"updated": "2016-02-14T13:58:47.016Z",
"updated": "2016-02-14T13:44:37.587Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/748",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:58:00.561Z",
"created": "2016-02-14T13:44:37.587Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"fingerprint": "4379d073c5ffd05ecbf9a0d7e69c4e519050a90a",
"fingerprint": "abf1dbdff878fa750f6ffb33fb362cb734e553e3",
"total_abstain": 1
},
"model": "representatives_votes.proposal",
......@@ -55,13 +55,13 @@
},
{
"fields": {
"updated": "2016-02-14T13:58:00.571Z",
"updated": "2016-02-14T13:44:37.596Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/747",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"title": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:58:00.571Z",
"created": "2016-02-14T13:44:37.596Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
......
[
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Pour",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/740",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"prenom": "Bernard",
"nom": "Roman"
},
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Contre",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/740-2",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"prenom": "Bernard",
"nom": "Roman"
},
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Abstention",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/748",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"prenom": "Bernard",
"nom": "Roman"
},
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Invalide",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/747",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"prenom": "Bernard",
"nom": "Roman"
},
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Pour",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/nonexisting",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/nonexisting",
"prenom": "Bernard",
"nom": "Roman"
},
{
"chambre": "AN",
"groupe": "Whatever",
"division": "Pour",
"scrutin_uri": "/scrutins/detail/(legislature)/14/(num)/747",
"scrutin_url": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"prenom": "Non",
"nom": "Existing"
}
......
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('representatives_votes', '0008_unique_proposal_title'),
]
operations = [
migrations.AddField(
model_name='dossier',
name='ext_link',
field=models.URLField(default=b'', blank=True),
),
]
......@@ -11,6 +11,7 @@ class Dossier(HashableModel, TimeStampedModel):
reference = models.CharField(max_length=200, unique=True)
text = models.TextField(blank=True, default='')
link = models.URLField()
ext_link = models.URLField(blank=True, default='')
hashable_fields = ['title', 'reference']
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment