Skip to content

GitLab

Commit 714aab07 authored by Nicolas Joyard's avatar Nicolas Joyard
Browse files

Fix rep votes tests with natural keys as well

parent a2782dd8
Hide whitespace changes
Inline Side-by-side
src/representatives_votes/contrib/francedata/tests/dossiers_expected.json
View file @ 714aab07
[
{
"fields" : {
"fields": {
"abbreviation": "AN",
"country": 1095,
"country": [
"FR"
],
"name": "Assembl\u00e9e nationale"
},
"model": "representatives.chamber",
"pk": 2
"model": "representatives.chamber"
},
{
"fields": {
"abbreviation": "SEN",
"country": 1095,
"country": [
"FR"
],
"name": "S\u00e9nat"
},
"model": "representatives.chamber",
"pk": 3
"model": "representatives.chamber"
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.303Z",
"updated": "2016-10-05T09:33:06.310Z",
"title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9",
"reference": "14/liberte_maires_rythmes_scolaires_premier_degre",
"created": "2016-07-07T20:23:24.302Z"
"created": "2016-10-05T09:33:06.310Z"
},
"model": "representatives_votes.dossier",
"pk": 1
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.365Z",
"updated": "2016-10-05T09:33:06.325Z",
"title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles",
"reference": "14/action_publique_territoriale_metropoles",
"created": "2016-07-07T20:23:24.332Z"
"created": "2016-10-05T09:33:06.318Z"
},
"model": "representatives_votes.dossier",
"pk": 2
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",
"updated": "2016-07-07T20:23:24.410Z",
"updated": "2016-10-05T09:33:06.338Z",
"title": "Protection de l'enfant",
"reference": "ppl13-799",
"created": "2016-07-07T20:23:24.410Z"
"created": "2016-10-05T09:33:06.338Z"
},
"model": "representatives_votes.dossier",
"pk": 3
"model": "representatives_votes.dossier"
},
{
"fields": {
"updated": "2016-07-07T20:23:24.307Z",
"updated": "2016-10-05T09:33:06.314Z",
"title": "",
"dossier": 1,
"created": "2016-07-07T20:23:24.307Z",
"dossier": [
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"created": "2016-10-05T09:33:06.314Z",
"kind": "procedure-file",
"chamber": 2,
"chamber": [
"AN"
],
"link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp"
},
"model": "representatives_votes.document",
......@@ -65,12 +68,16 @@
},
{
"fields": {
"updated": "2016-07-07T20:23:24.335Z",
"updated": "2016-10-05T09:33:06.320Z",
"title": "",
"dossier": 2,
"created": "2016-07-07T20:23:24.335Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:33:06.320Z",
"kind": "procedure-file",
"chamber": 3,
"chamber": [
"SEN"
],
"link": "http://www.senat.fr/dossier-legislatif/pjl12-495.html"
},
"model": "representatives_votes.document",
......@@ -78,12 +85,16 @@
},
{
"fields": {
"updated": "2016-07-07T20:23:24.371Z",
"updated": "2016-10-05T09:33:06.332Z",
"title": "",
"dossier": 2,
"created": "2016-07-07T20:23:24.371Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:33:06.331Z",
"kind": "procedure-file",
"chamber": 2,
"chamber": [
"AN"
],
"link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp"
},
"model": "representatives_votes.document",
......@@ -91,12 +102,16 @@
},
{
"fields": {
"updated": "2016-07-07T20:23:24.415Z",
"updated": "2016-10-05T09:33:06.342Z",
"title": "",
"dossier": 3,
"created": "2016-07-07T20:23:24.415Z",
"dossier": [
"ppl13-799"
],
"created": "2016-10-05T09:33:06.342Z",
"kind": "procedure-file",
"chamber": 3,
"chamber": [
"SEN"
],
"link": "http://www.senat.fr/dossier-legislatif/ppl13-799.html"
},
"model": "representatives_votes.document",
......
src/representatives_votes/contrib/francedata/tests/scrutins_expected.json
View file @ 714aab07
[
{
"fields": {
"updated": "2016-02-14T13:44:37.550Z",
"updated": "2016-10-05T09:41:46.342Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:44:37.550Z",
"dossier": [
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"created": "2016-10-05T09:41:46.342Z",
"kind": "dossier",
"datetime": "2013-12-04T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 1
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.578Z",
"updated": "2016-10-05T09:41:46.349Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:44:37.578Z",
"dossier": [
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"created": "2016-10-05T09:41:46.349Z",
"kind": "dossier",
"datetime": "2013-12-05T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 2
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.587Z",
"updated": "2016-10-05T09:41:46.363Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.587Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:41:46.363Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 3
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.596Z",
"updated": "2016-10-05T09:41:46.375Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"title": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.596Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:41:46.375Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 4
"model": "representatives_votes.proposal"
}
]
src/representatives_votes/contrib/francedata/tests/test_francedata_import.py
View file @ 714aab07
import copy
import os
import pytest
from django.core.serializers.json import Deserializer
from django.core.management import call_command
from representatives.models import Representative
from representatives.tests.base import TestBase
from representatives_votes.contrib.francedata import import_dossiers
from representatives_votes.contrib.francedata import import_scrutins
from representatives_votes.contrib.francedata import import_votes
......@@ -16,55 +14,39 @@ def _get_testdata(filename):
return os.path.join(os.path.dirname(__file__), filename)
def _test_import(fixtures, scenario, callback):
for model in (Representative, Dossier, Proposal, Vote):
model.objects.all().delete()
class FranceDataVotesTest(TestBase):
def _test_import(self, fixtures, scenario, callback):
for model in (Representative, Dossier, Proposal, Vote):
model.objects.all().delete()
for fix in fixtures:
call_command('loaddata', fix)
for fix in fixtures:
call_command('loaddata', fix)
inputfile = _get_testdata('%s_input.json' % scenario)
expected = _get_testdata('%s_expected.json' % scenario)
inputfile = _get_testdata('%s_input.json' % scenario)
expected = _get_testdata('%s_expected.json' % scenario)
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(inputfile, 'r') as f:
callback(f)
with open(inputfile, 'r') as f:
callback(f)
self.assertObjectsFromFixture(expected)
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
def test_francedata_import_dossiers(self):
fixtures = []
for f in exclude:
if f in compare:
compare.pop(f)
self._test_import(fixtures, 'dossiers', import_dossiers.main)
type(obj.object).objects.get(**compare)
def test_francedata_import_scrutins(self):
fixtures = [
_get_testdata('dossiers_expected.json')
]
self._test_import(fixtures, 'scrutins', import_scrutins.main)
@pytest.mark.django_db
def test_francedata_import_dossiers():
fixtures = []
def test_francedata_import_votes(self):
fixtures = [
_get_testdata('dossiers_expected.json'),
_get_testdata('scrutins_expected.json'),
_get_testdata('rep_fixture.json')
]
_test_import(fixtures, 'dossiers', import_dossiers.main)
@pytest.mark.django_db
def test_francedata_import_scrutins():
fixtures = [
_get_testdata('dossiers_expected.json')
]
_test_import(fixtures, 'scrutins', import_scrutins.main)
@pytest.mark.django_db
def test_francedata_import_votes():
fixtures = [
_get_testdata('dossiers_expected.json'),
_get_testdata('scrutins_expected.json'),
_get_testdata('rep_fixture.json')
]
_test_import(fixtures, 'votes', import_votes.main)
self._test_import(fixtures, 'votes', import_votes.main)
src/representatives_votes/contrib/francedata/tests/votes_expected.json
View file @ 714aab07
[
{
"fields": {
"updated": "2016-02-14T13:44:37.550Z",
"updated": "2016-10-05T09:43:34.405Z",
"total_for": 1,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9.",
"dossier": 1,
"created": "2016-02-14T13:44:37.550Z",
"dossier": [
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"created": "2016-10-05T09:41:46.342Z",
"kind": "dossier",
"datetime": "2013-12-04T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 1
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.578Z",
"updated": "2016-10-05T09:43:34.427Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"title": "La motion de rejet pr\u00e9alable, pr\u00e9sent\u00e9e par m. le roux, de la proposition de loi permettant le libre choix des maires concernant les rythmes scolaires dans l'enseignement du premier degr\u00e9. (1)",
"dossier": 1,
"created": "2016-02-14T13:44:37.578Z",
"dossier": [
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"created": "2016-10-05T09:41:46.349Z",
"kind": "dossier",
"datetime": "2013-12-05T23:00:00Z",
"total_against": 1,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 2
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.587Z",
"updated": "2016-10-05T09:43:34.434Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"title": "L'amendement n\u00b0 381 de m. dolez \u00e0 l'article 2 du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.587Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:41:46.363Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"total_abstain": 1
},
"model": "representatives_votes.proposal",
"pk": 3
"model": "representatives_votes.proposal"
},
{
"fields": {
"updated": "2016-02-14T13:44:37.596Z",
"updated": "2016-10-05T09:41:46.375Z",
"total_for": 0,
"description": "",
"reference": "http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/747",
"title": "L'amendement n\u00b0 379 de m. dolez \u00e0 l'article 1er a du projet de loi de modernisation de l'action publique territoriale et d'affirmation des m\u00e9tropoles.",
"dossier": 2,
"created": "2016-02-14T13:44:37.596Z",
"dossier": [
"14/action_publique_territoriale_metropoles"
],
"created": "2016-10-05T09:41:46.375Z",
"kind": "dossier",
"datetime": "2013-12-10T23:00:00Z",
"total_against": 0,
"total_abstain": 0
},
"model": "representatives_votes.proposal",
"pk": 4
"model": "representatives_votes.proposal"
},
{
"fields": {
"representative_name": "",
"position": "for",
"proposal": 1,
"representative": 1
"proposal": [
"http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740",
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"representative": [
"bernard-roman-1952-07-15"
]
},
"model": "representatives_votes.vote",
"pk": 1
......@@ -81,8 +90,13 @@
"fields": {
"representative_name": "",
"position": "against",
"proposal": 2,
"representative": 1
"proposal": [
"http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/740-2",
"14/liberte_maires_rythmes_scolaires_premier_degre"
],
"representative": [
"bernard-roman-1952-07-15"
]
},
"model": "representatives_votes.vote",
"pk": 2
......@@ -91,8 +105,13 @@
"fields": {
"representative_name": "",
"position": "abstain",
"proposal": 3,
"representative": 1
"proposal": [
"http://www.assemblee-nationale.fr/scrutins/detail/(legislature)/14/(num)/748",
"14/action_publique_territoriale_metropoles"
],
"representative": [
"bernard-roman-1952-07-15"
]
},
"model": "representatives_votes.vote",
"pk": 3
......
src/representatives_votes/contrib/parltrack/import_votes.py
View file @ 714aab07
......@@ -36,6 +36,15 @@ class Command(object):
self.index_representatives()
self.index_dossiers()
def should_skip(self, proposal_data):
responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
for receiver, response in responses:
if response is False:
return True
return False
def parse_vote_data(self, vote_data):
"""
Parse data from parltrack votes db dumps (1 proposal)
......@@ -100,14 +109,11 @@ class Command(object):
if changed:
proposal.save()
responses = vote_pre_import.send(sender=self, vote_data=proposal_data)
for receiver, response in responses:
if response is False:
logger.debug(
'Skipping dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
if self.should_skip(proposal_data):
logger.debug(
'Skipping dossier %s', proposal_data.get(
'epref', proposal_data['title']))
return
positions = ['For', 'Abstain', 'Against']
logger.info(
......
src/representatives_votes/contrib/parltrack/tests/dossiers_expected.json
View file @ 714aab07
......@@ -2,66 +2,65 @@
{
"fields": {
"text": "",
"updated": "2016-07-08T05:17:40.580Z",
"updated": "2016-10-05T09:49:40.469Z",
"title": "Agenda for change: the future of EU development policy",
"reference": "2012/2002(INI)",
"created": "2016-07-08T05:17:40.580Z"
"created": "2016-10-05T09:49:40.469Z"
},
"model": "representatives_votes.dossier",
"pk": 1
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",
"updated": "2016-07-08T05:17:40.617Z",
"updated": "2016-10-05T09:49:40.477Z",
"title": "2016 general budget: all sections",
"reference": "2015/2132(BUD)",
"created": "2016-07-08T05:17:40.616Z"
"created": "2016-10-05T09:49:40.477Z"
},
"model": "representatives_votes.dossier",
"pk": 2
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",
"updated": "2016-07-08T05:17:40.644Z",
"updated": "2016-10-05T09:49:40.482Z",
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"reference": "2013/2857(DEA)",
"created": "2016-07-08T05:17:40.644Z"
"created": "2016-10-05T09:49:40.482Z"
},
"model": "representatives_votes.dossier",
"pk": 3
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",
"updated": "2016-07-08T05:17:40.682Z",
"updated": "2016-10-05T09:49:40.487Z",
"title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries",
"reference": "2015/2623(DEA)",
"created": "2016-07-08T05:17:40.682Z"
"created": "2016-10-05T09:49:40.487Z"
},
"model": "representatives_votes.dossier",
"pk": 4
"model": "representatives_votes.dossier"
},
{
"fields": {
"text": "",