Commit 6cac0520 authored by Nicolas Joyard's avatar Nicolas Joyard

Add FranceData import scripts

parent 1e3bf2c9
......@@ -14,6 +14,9 @@ script:
- django-admin migrate
- flake8 representatives_votes/ --exclude migrations --ignore E128
- py.test
- cat representatives_votes/contrib/francedata/tests/dossiers_input.json | francedata_import_dossiers
- cat representatives_votes/contrib/francedata/tests/scrutins_input.json | francedata_import_scrutins
- cat representatives_votes/contrib/francedata/tests/votes_input.json | francedata_import_votes
- cat representatives_votes/contrib/parltrack/tests/dossiers_fixture.json | parltrack_import_dossiers
- cat representatives_votes/contrib/parltrack/tests/votes_fixture.json | parltrack_import_votes
after_success:
......
# coding: utf-8
import sys
import ijson
import logging
import django
from django.apps import apps
from representatives_votes.models import Dossier
logger = logging.getLogger(__name__)
def parse_dossier_data(data):
changed = False
ref = data['uri']
try:
dossier = Dossier.objects.get(reference=ref)
except Dossier.DoesNotExist:
dossier = Dossier(reference=ref)
logger.debug('Created dossier %s' % ref)
changed = True
title = data['titre']
if dossier.title != title:
logger.debug('Changed dossier title to %s' % title)
dossier.title = title
changed = True
source = data['url']
if dossier.link != source:
logger.debug('Changed dossier link to %s' % source)
dossier.link = source
changed = True
if changed:
logger.debug('Saved dossier %s' % ref)
dossier.save()
def main(stream=None):
if not apps.ready:
django.setup()
for data in ijson.items(stream or sys.stdin, 'item'):
parse_dossier_data(data)
# coding: utf-8
from datetime import datetime
import ijson
import logging
from pytz import timezone as date_timezone
import sys
import django
from django.apps import apps
from django.utils.timezone import make_aware as date_make_aware
from representatives_votes.models import Dossier, Proposal
logger = logging.getLogger(__name__)
def _parse_date(date_str):
return date_make_aware(
datetime.strptime(date_str, "%Y-%m-%d"),
date_timezone('Europe/Paris')
)
def _get_unique_title(proposal_pk, candidate):
title = candidate
try:
exists = Proposal.objects.get(title=title)
except Proposal.DoesNotExist:
exists = None
if exists and exists.pk != proposal_pk:
num = 1
while exists and exists.pk != proposal_pk:
title = '%s (%d)' % (candidate, num)
try:
exists = Proposal.objects.get(title=title)
except Proposal.DoesNotExist:
exists = None
num = num + 1
logger.debug('Made unique title %s' % title)
return title
class ScrutinImporter:
dossiers = None
def get_dossier(self, ref):
if self.dossiers is None:
self.dossiers = {
d[0]: d[1] for d in Dossier.objects.values_list('reference',
'pk')
}
return self.dossiers.get(ref, None)
def parse_scrutin_data(self, data):
ref = data['uri']
if 'dossier_uri' not in data:
logger.debug('Cannot create proposal without dossier')
return
dossier = self.get_dossier(data['dossier_uri'])
if dossier is None:
logger.debug('Cannot create proposal for unknown dossier %s'
% data['dossier_uri'])
return
changed = False
try:
proposal = Proposal.objects.get(reference=ref)
except Proposal.DoesNotExist:
proposal = Proposal(reference=ref, total_for=0, total_against=0,
total_abstain=0)
logger.debug('Created proposal %s' % ref)
changed = True
values = dict(
title=_get_unique_title(proposal.pk, data["objet"]),
datetime=_parse_date(data["date"]),
dossier_id=self.get_dossier(data['dossier_uri']),
kind='dossier'
)
for key, value in values.items():
if value != getattr(proposal, key, None):
logger.debug('Changed proposal %s to %s' % (key, value))
setattr(proposal, key, value)
changed = True
if changed:
logger.debug('Updated proposal %s' % ref)
proposal.save()
def main(stream=None):
if not apps.ready:
django.setup()
importer = ScrutinImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
importer.parse_scrutin_data(data)
# coding: utf-8
import ijson
import logging
import sys
import django
from django.apps import apps
from django.utils.text import slugify
from representatives_votes.models import Proposal, Representative, Vote
logger = logging.getLogger(__name__)
class VotesImporter:
deputes = None
scrutins = None
touched = []
positions = dict(
pour="for",
contre="against",
abstention="abstain"
)
def get_depute(self, prenom, nom):
if self.deputes is None:
self.deputes = {
slugify(r[0]): r[1] for r in
Representative.objects.values_list('full_name', 'pk')
}
full = (u'%s %s' % (prenom, nom)).replace(u' ', ' ')
return self.deputes.get(slugify(full), None)
def get_scrutin(self, ref):
if self.scrutins is None:
self.scrutins = {
s[0]: s[1] for s in Proposal.objects.values_list('reference',
'pk')
}
return self.scrutins.get(ref, None)
def parse_vote_data(self, data):
scrutin = self.get_scrutin(data['scrutin_uri'])
if scrutin is None:
logger.debug('Cannot import vote for unknown scrutin %s'
% data['scrutin_uri'])
return
depute = self.get_depute(data['prenom'], data['nom'])
if depute is None:
logger.debug('Cannot import vote by unknown rep %s %s'
% (data['prenom'], data['nom']))
return
if not data['division'].lower() in self.positions:
logger.debug('Cannot import vote for invalid position %s'
% data['division'])
return
position = self.positions[data['division'].lower()]
changed = False
try:
vote = Vote.objects.get(representative_id=depute,
proposal_id=scrutin)
except Vote.DoesNotExist:
vote = Vote(representative_id=depute, proposal_id=scrutin)
logger.debug('Created vote for rep %s on %s' % (depute, scrutin))
changed = True
if vote.position != position:
logger.debug('Changed vote position to %s' % position)
changed = True
vote.position = position
if changed:
logger.debug('Updated vote for rep %s on %s' % (depute, scrutin))
self.touched.append(scrutin)
vote.save()
def update_totals(self):
proposals = [Proposal.objects.get(pk=pk) for pk in self.touched]
for proposal in proposals:
changed = False
for pos in self.positions.values():
count = Vote.objects.filter(proposal_id=proposal.pk,
position=pos).count()
if getattr(proposal, 'total_%s' % pos, None) != count:
logger.debug('Changed %s count for proposal %s to %s' % (
pos, proposal.pk, count))
setattr(proposal, 'total_%s' % pos, count)
changed = True
if changed:
logger.debug('Updated proposal %s' % proposal.pk)
proposal.save()
def main(stream=None):
if not apps.ready:
django.setup()
importer = VotesImporter()
for data in ijson.items(stream or sys.stdin, 'item'):
importer.parse_vote_data(data)
importer.update_totals()
[
{
"fields": {
"updated": "2016-02-14T13:16:31.417Z",
"reference": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"text": "",
"created": "2016-02-14T13:16:31.417Z",
"link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"fingerprint": "5d1707e6663bb28d0308cdb36e9e91c5f235f8a1"
},
"model": "representatives_votes.dossier",
"pk": 1
},
{
"fields": {
"updated": "2016-02-14T13:16:31.428Z",
"reference": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"text": "",
"created": "2016-02-14T13:16:31.428Z",
"link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"fingerprint": "c03f5e32f66e5f03ebe0a5d100f2f4ade941accc"
},
"model": "representatives_votes.dossier",
"pk": 2
}
]
[
{
"url": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"titre": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp"
},
{
"url": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"titre": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"uri": "/14/dossiers/action_publique_territoriale_metropoles.asp"
}
]
\ No newline at end of file
[
{
"fields": {
"updated": "2016-02-14T14:01:37.343Z",
"last_name": "",
"photo": "http://www.nosdeputes.fr/depute/photo/bernard-roman",
"created": "2016-02-14T14:01:37.343Z",
"gender": 2,
"remote_id": "2611",
"first_name": "",
"cv": "",
"active": true,
"birth_place": "Lille (Nord)",
"full_name": "Bernard Roman",
"fingerprint": "e28b45cced3c89ad3835fbdf261367ebea91b180",
"birth_date": "1952-07-15",
"slug": "bernard-roman"
},
"model": "representatives.representative",
"pk": 1
}
]
[
{
"fields": {
"updated": "2016-02-14T13:44:37.550Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:44:37.550Z",
"kind": "dossier",
"datetime": "2013-12-04T23:00:00Z",
"total_against": 0,
"fingerprint": "40bb927c36b00bb688c1d7e7f4be5b9a1aae4af3",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 1
},
{
"fields": {
"updated": "2016-02-14T13:44:37.578Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:44:37.578Z",
"kind": "dossier",
"datetime": "2013-12-05T23:00:00Z",
"total_against": 0,
"fingerprint": "a8709fb12e8e6e4a5f46931d855bf70453dd7fd2",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 2
},
{
"fields": {
"updated": "2016-02-14T13:44:37.587Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.587Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"fingerprint": "abf1dbdff878fa750f6ffb33fb362cb734e553e3",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 3
},
{
"fields": {
"updated": "2016-02-14T13:44:37.596Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/747",
"title": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.596Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"fingerprint": "3b75e49e1be0c8efc3706b9896cc1cb1f76dd9a7",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 4
}
]
[
{
"objet": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"uri": "/scrutins/detail/(legislature)/14/(num)/740",
"numero": "740",
"dossier_uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"date": "2013-12-05"
},
{
"objet": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"uri": "/scrutins/detail/(legislature)/14/(num)/740-2",
"numero": "740-2",
"dossier_uri": "/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp",
"date": "2013-12-06"
},
{
"objet": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"uri": "/scrutins/detail/(legislature)/14/(num)/748",
"numero": "748",
"dossier_uri": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"date": "2013-12-11"
},
{
"objet": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"url": "http://www2.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"uri": "/scrutins/detail/(legislature)/14/(num)/747",
"numero": "747",
"dossier_uri": "/14/dossiers/action_publique_territoriale_metropoles.asp",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp",
"date": "2013-12-11"
},
{
"objet": "Dossier inexistant.",
"url": "http://www2.assemblee-nationale.fr/404",
"uri": "/scrutins/404",
"numero": "000",
"dossier_uri": "/14/dossiers/inexistant",
"dossier_url": "http://www2.assemblee-nationale.fr/14/dossiers/inexistant",
"date": "2099-12-11"
}
]
\ No newline at end of file
import copy
import os
import pytest
from django.core.serializers.json import Deserializer
from django.core.management import call_command
from representatives.models import Representative
from representatives_votes.contrib.francedata import import_dossiers
from representatives_votes.contrib.francedata import import_scrutins
from representatives_votes.contrib.francedata import import_votes
from representatives_votes.models import Dossier, Proposal, Vote
def _get_testdata(filename):
return os.path.join(os.path.dirname(__file__), filename)
def _test_import(fixtures, scenario, callback):
for model in (Representative, Dossier, Proposal, Vote):
model.objects.all().delete()
for fix in fixtures:
call_command('loaddata', fix)
inputfile = _get_testdata('%s_input.json' % scenario)
expected = _get_testdata('%s_expected.json' % scenario)
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(inputfile, 'r') as f:
callback(f)
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for f in exclude:
if f in compare:
compare.pop(f)
type(obj.object).objects.get(**compare)
@pytest.mark.django_db
def test_francedata_import_dossiers():
fixtures = []
_test_import(fixtures, 'dossiers', import_dossiers.main)
@pytest.mark.django_db
def test_francedata_import_scrutins():
fixtures = [
_get_testdata('dossiers_expected.json')
]
_test_import(fixtures, 'scrutins', import_scrutins.main)
@pytest.mark.django_db
def test_francedata_import_votes():
fixtures = [
_get_testdata('dossiers_expected.json'),
_get_testdata('scrutins_expected.json'),
_get_testdata('rep_fixture.json')
]
_test_import(fixtures, 'votes', import_votes.main)
[
{
"fields": {
"updated": "2016-02-14T13:58:46.991Z",
"total_for": 1,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:58:00.536Z",
"kind": "dossier",
"datetime": "2013-12-04T23:00:00Z",
"total_against": 0,
"fingerprint": "73ca11e690cc6db1d806f927538d05b49eb6fd9d",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 1
},
{
"fields": {
"updated": "2016-02-14T13:58:47.003Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:58:00.551Z",
"kind": "dossier",
"datetime": "2013-12-05T23:00:00Z",
"total_against": 1,
"fingerprint": "92ece2396838f1612ada0d896cbdce8fe5deaf13",
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 2
},
{
"fields": {
"updated": "2016-02-14T13:58:47.016Z",
"total_for": 0,
"description": "",
"reference": "/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:58:00.561Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"fingerprint": "4379d073c5ffd05ecbf9a0d7e69c4e519050a90a",
"total_abstain": 1
},
"model": "representatives_votes.proposal",
"pk": 3
},