From 82a3b1dcb9887d92b55b82efa4326a1653f7630c Mon Sep 17 00:00:00 2001 From: Nicolas Joyard Date: Wed, 5 Oct 2016 10:56:18 +0200 Subject: [PATCH] Fix rep import tests (using natural keys) --- .../tests/representatives_expected.json | 532 ++++++++++------ .../test_francedata_import_representatives.py | 41 +- .../tests/representatives_expected.json | 589 ++++++++++++------ .../tests/test_import_representatives.py | 41 +- src/representatives/models.py | 94 ++- src/representatives/tests/base.py | 38 ++ 6 files changed, 912 insertions(+), 423 deletions(-) create mode 100644 src/representatives/tests/base.py diff --git a/src/representatives/contrib/francedata/tests/representatives_expected.json b/src/representatives/contrib/francedata/tests/representatives_expected.json index 68d99a2..bfe99ef 100644 --- a/src/representatives/contrib/francedata/tests/representatives_expected.json +++ b/src/representatives/contrib/francedata/tests/representatives_expected.json @@ -12,8 +12,7 @@ "birth_date": "1952-07-15", "slug": "bernard-roman-1952-07-15" }, - "model": "representatives.representative", - "pk": 1 + "model": "representatives.representative" }, { "fields": { @@ -28,13 +27,14 @@ "birth_date": "1959-06-16", "slug": "david-assouline-1959-06-16" }, - "model": "representatives.representative", - "pk": 2 + "model": "representatives.representative" }, { "fields": { "email": "contact@bernard-roman.org", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "kind": "other" }, "model": "representatives.email", @@ -43,7 +43,9 @@ { "fields": { "email": "broman@assemblee-nationale.fr", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "kind": "official" }, "model": "representatives.email", @@ -52,7 +54,9 @@ { "fields": { "email": "d.assouline@senat.fr", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "official" }, "model": "representatives.email", @@ -61,7 +65,9 @@ { "fields": { "url": "http://www2.assemblee-nationale.fr/deputes/fiche/OMC_PA2611", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "kind": "AN" }, "model": "representatives.website", @@ -70,7 +76,9 @@ { "fields": { "url": "http://www.bernard-roman.net", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "kind": "" }, "model": "representatives.website", @@ -79,7 +87,9 @@ { "fields": { "url": "http://twitter.com/bernardroman59", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "kind": "twitter" }, "model": "representatives.website", @@ -88,7 +98,9 @@ { "fields": { "url": "http://www.senat.fr/senateur/assouline_david04059m.html", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "SEN" }, "model": "representatives.website", @@ -97,7 +109,9 @@ { "fields": { "url": "http://www.david-assouline.net", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "" }, "model": "representatives.website", @@ -106,7 +120,9 @@ { "fields": { "url": "https://www.facebook.com/DavAssouline", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "facebook" }, "model": "representatives.website", @@ -115,7 +131,9 @@ { "fields": { "url": "http://david-assouline.net", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "" }, "model": "representatives.website", @@ -124,7 +142,9 @@ { "fields": { "url": "http://twitter.com/dassouline", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "kind": "twitter" }, "model": "representatives.website", @@ -134,12 +154,16 @@ "fields": { "city": "Paris", "name": "Assembl\u00e9e nationale", - "country": 1095, + "country": [ + "FR" + ], "floor": "", "number": "126", "street": "Rue de l'Universit\u00e9", "office_number": "", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "location": "", "kind": "official", "postcode": "75355" @@ -151,12 +175,16 @@ "fields": { "city": "Lille", "name": "Permanence", - "country": 1095, + "country": [ + "FR" + ], "floor": "", "number": "20", "street": "Rue d'Arras", "office_number": "", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "location": "", "kind": "", "postcode": "59000" @@ -168,12 +196,16 @@ "fields": { "city": "Lille", "name": "", - "country": 1095, + "country": [ + "FR" + ], "floor": "", "number": "", "street": "", "office_number": "", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "location": "", "kind": "", "postcode": "59000" @@ -185,12 +217,16 @@ "fields": { "city": "Paris", "name": "Palais du Luxembourg", - "country": 1095, + "country": [ + "FR" + ], "floor": "", "number": "15", "street": "Rue de Vaugirard", "office_number": "", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "location": "", "kind": "official", "postcode": "75291" @@ -201,7 +237,9 @@ { "fields": { "number": "03 20 52 09 20", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "address": 2, "kind": "" }, @@ -214,36 +252,38 @@ "country": null, "name": "European Parliament" }, - "model": "representatives.chamber", - "pk": 1 + "model": "representatives.chamber" }, { "fields": { "abbreviation": "AN", - "country": 1095, + "country": [ + "FR" + ], "name": "Assembl\u00e9e nationale" }, - "model": "representatives.chamber", - "pk": 2 + "model": "representatives.chamber" }, { "fields": { "abbreviation": "SEN", - "country": 1095, + "country": [ + "FR" + ], "name": "S\u00e9nat" }, - "model": "representatives.chamber", - "pk": 3 + "model": "representatives.chamber" }, { "fields": { "name": "European Parliament", "kind": "chamber", "abbreviation": "EP", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 1 + "model": "representatives.group" }, { "fields": { @@ -252,28 +292,29 @@ "abbreviation": "FR", "chamber": null }, - "model": "representatives.group", - "pk": 2 + "model": "representatives.group" }, { "fields": { "name": "Assembl\u00e9e nationale", "kind": "chamber", "abbreviation": "AN", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 3 + "model": "representatives.group" }, { "fields": { "name": "Socialiste, r\u00e9publicain et citoyen", "kind": "group", "abbreviation": "SRC", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 4 + "model": "representatives.group" }, { "fields": { @@ -282,8 +323,7 @@ "abbreviation": "59", "chamber": null }, - "model": "representatives.group", - "pk": 5 + "model": "representatives.group" }, { "fields": { @@ -292,78 +332,84 @@ "abbreviation": "59-1", "chamber": null }, - "model": "representatives.group", - "pk": 6 + "model": "representatives.group" }, { "fields": { "name": "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique", "kind": "committee", "abbreviation": "Lois", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 7 + "model": "representatives.group" }, { "fields": { "name": "Bureau de l'assembl\u00e9e nationale", "kind": "delegation", "abbreviation": "", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 8 + "model": "representatives.group" }, { "fields": { "name": "D\u00e9l\u00e9gation charg\u00e9e de la communication et de la presse", "kind": "delegation", "abbreviation": "", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 9 + "model": "representatives.group" }, { "fields": { "name": "Groupe d'amiti\u00e9 france-mexique", "kind": "delegation", "abbreviation": "", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 10 + "model": "representatives.group" }, { "fields": { "name": "Groupe d'amiti\u00e9 france-argentine", "kind": "delegation", "abbreviation": "", - "chamber": 2 + "chamber": [ + "AN" + ] }, - "model": "representatives.group", - "pk": 11 + "model": "representatives.group" }, { "fields": { "name": "S\u00e9nat", "kind": "chamber", "abbreviation": "SEN", - "chamber": 3 + "chamber": [ + "SEN" + ] }, - "model": "representatives.group", - "pk": 12 + "model": "representatives.group" }, { "fields": { "name": "Socialiste et r\u00e9publicain", "kind": "group", "abbreviation": "SOC", - "chamber": 3 + "chamber": [ + "SEN" + ] }, - "model": "representatives.group", - "pk": 13 + "model": "representatives.group" }, { "fields": { @@ -372,8 +418,7 @@ "abbreviation": "75", "chamber": null }, - "model": "representatives.group", - "pk": 14 + "model": "representatives.group" }, { "fields": { @@ -382,280 +427,419 @@ "abbreviation": "75-nd", "chamber": null }, - "model": "representatives.group", - "pk": 15 + "model": "representatives.group" }, { "fields": { "name": "Commission de la culture, de l'\u00e9ducation et de la communication", "kind": "committee", "abbreviation": "Culture", - "chamber": 3 + "chamber": [ + "SEN" + ] }, - "model": "representatives.group", - "pk": 16 + "model": "representatives.group" }, { "fields": { "name": "Groupe France-Japon", "kind": "delegation", "abbreviation": "", - "chamber": 3 + "chamber": [ + "SEN" + ] }, - "model": "representatives.group", - "pk": 17 + "model": "representatives.group" }, { "fields": { "country": null, "name": "European Parliament" }, - "model": "representatives.constituency", - "pk": 1 + "model": "representatives.constituency" }, { "fields": { - "country": 1095, + "country": [ + "FR" + ], "name": "Assembl\u00e9e nationale" }, - "model": "representatives.constituency", - "pk": 2 + "model": "representatives.constituency" }, { "fields": { - "country": 1095, + "country": [ + "FR" + ], "name": "S\u00e9nat" }, - "model": "representatives.constituency", - "pk": 3 + "model": "representatives.constituency" }, { "fields": { - "country": 1095, + "country": [ + "FR" + ], "name": "Parti socialiste" }, - "model": "representatives.constituency", - "pk": 4 + "model": "representatives.constituency" }, { "fields": { - "group": 2, + "group": [ + "France", + "country", + null + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": null, - "constituency": 4 + "constituency": [ + "Parti socialiste", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 1 + "model": "representatives.mandate" }, { "fields": { - "group": 3, + "group": [ + "Assembl\u00e9e nationale", + "chamber", + "AN" + ], "end_date": null, "role": "D\u00e9put\u00e9", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 2 + "model": "representatives.mandate" }, { "fields": { - "group": 4, + "group": [ + "Socialiste, r\u00e9publicain et citoyen", + "group", + "AN" + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 3 + "model": "representatives.mandate" }, { "fields": { - "group": 5, + "group": [ + "Nord", + "department", + null + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 4 + "model": "representatives.mandate" }, { "fields": { - "group": 6, + "group": [ + "Nord (1\u00e8re circonscription)", + "district", + null + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 5 + "model": "representatives.mandate" }, { "fields": { - "group": 7, + "group": [ + "Commission des lois constitutionnelles, de la l\u00e9gislation et de l'administration g\u00e9n\u00e9rale de la r\u00e9publique", + "committee", + "AN" + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 6 + "model": "representatives.mandate" }, { "fields": { - "group": 8, + "group": [ + "Bureau de l'assembl\u00e9e nationale", + "delegation", + "AN" + ], "end_date": null, "role": "questeur", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 7 + "model": "representatives.mandate" }, { "fields": { - "group": 9, + "group": [ + "D\u00e9l\u00e9gation charg\u00e9e de la communication et de la presse", + "delegation", + "AN" + ], "end_date": null, "role": "membre", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 8 + "model": "representatives.mandate" }, { "fields": { - "group": 10, + "group": [ + "Groupe d'amiti\u00e9 france-mexique", + "delegation", + "AN" + ], "end_date": null, "role": "vice-pr\u00e9sident", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 9 + "model": "representatives.mandate" }, { "fields": { - "group": 11, + "group": [ + "Groupe d'amiti\u00e9 france-argentine", + "delegation", + "AN" + ], "end_date": null, "role": "vice-pr\u00e9sident", - "representative": 1, + "representative": [ + "bernard-roman-1952-07-15" + ], "link": "", "begin_date": "2012-06-20", - "constituency": 2 + "constituency": [ + "Assembl\u00e9e nationale", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 10 + "model": "representatives.mandate" }, { "fields": { - "group": 2, + "group": [ + "France", + "country", + null + ], "end_date": null, "role": "membre", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": null, - "constituency": 4 + "constituency": [ + "Parti socialiste", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 11 + "model": "representatives.mandate" }, { "fields": { - "group": 12, + "group": [ + "S\u00e9nat", + "chamber", + "SEN" + ], "end_date": "2012-01-01", "role": "S\u00e9nateur", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 12 + "model": "representatives.mandate" }, { "fields": { - "group": 13, + "group": [ + "Socialiste et r\u00e9publicain", + "group", + "SEN" + ], "end_date": "2012-01-01", "role": "membre", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 13 + "model": "representatives.mandate" }, { "fields": { - "group": 14, + "group": [ + "Paris", + "department", + null + ], "end_date": "2012-01-01", "role": "membre", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 14 + "model": "representatives.mandate" }, { "fields": { - "group": 15, + "group": [ + "Paris", + "district", + null + ], "end_date": "2012-01-01", "role": "membre", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 15 + "model": "representatives.mandate" }, { "fields": { - "group": 16, + "group": [ + "Commission de la culture, de l'\u00e9ducation et de la communication", + "committee", + "SEN" + ], "end_date": "2012-01-01", "role": "vice-pr\u00e9sident", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 16 + "model": "representatives.mandate" }, { "fields": { - "group": 17, + "group": [ + "Groupe France-Japon", + "delegation", + "SEN" + ], "end_date": "2012-01-01", "role": "pr\u00e9sident", - "representative": 2, + "representative": [ + "david-assouline-1959-06-16" + ], "link": "", "begin_date": "2004-09-26", - "constituency": 3 + "constituency": [ + "S\u00e9nat", + "FR" + ] }, - "model": "representatives.mandate", - "pk": 17 + "model": "representatives.mandate" } ] diff --git a/src/representatives/contrib/francedata/tests/test_francedata_import_representatives.py b/src/representatives/contrib/francedata/tests/test_francedata_import_representatives.py index eb15c3d..c52724f 100644 --- a/src/representatives/contrib/francedata/tests/test_francedata_import_representatives.py +++ b/src/representatives/contrib/francedata/tests/test_francedata_import_representatives.py @@ -1,38 +1,19 @@ -import pytest import os -import copy -from django.core.serializers.json import Deserializer from representatives.models import Representative +from representatives.tests.base import TestBase from representatives.contrib.francedata import import_representatives -@pytest.mark.django_db -def test_francedata_import_representatives(): - inputjson = os.path.join(os.path.dirname(__file__), - 'representatives_input.json') - expected = os.path.join(os.path.dirname(__file__), - 'representatives_expected.json') +class FranceDataRepresentativesTest(TestBase): + def test_francedata_import_representatives(self): + inputjson = os.path.join(os.path.dirname(__file__), + 'representatives_input.json') + expected = os.path.join(os.path.dirname(__file__), + 'representatives_expected.json') - # Disable django auto fields - exclude = ('id', '_state', 'created', 'updated', 'fingerprint') + with open(inputjson, 'r') as f: + import_representatives.main(f) - with open(inputjson, 'r') as f: - import_representatives.main(f) - - missing = [] - with open(expected, 'r') as f: - for obj in Deserializer(f.read()): - compare = copy.copy(obj.object.__dict__) - - for field in exclude: - if field in compare: - compare.pop(field) - - try: - type(obj.object).objects.get(**compare) - except: - missing.append(compare) - - assert len(missing) is 0 - assert Representative.objects.count() == 2 + self.assertObjectsFromFixture(expected) + assert Representative.objects.count() == 2 diff --git a/src/representatives/contrib/parltrack/tests/representatives_expected.json b/src/representatives/contrib/parltrack/tests/representatives_expected.json index 80693ac..fe547d2 100644 --- a/src/representatives/contrib/parltrack/tests/representatives_expected.json +++ b/src/representatives/contrib/parltrack/tests/representatives_expected.json @@ -2,39 +2,39 @@ { "fields": { "last_name": "PIRKER", - "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg", "gender": 2, "first_name": "Hubert", "cv": "Transport and security spokesman, \u00d6VP Delegation, European Parliament;\nsecurity spokesman, \u00d6VP Delegation, European Parliament (2006-2009); security spokesman (coordinator), EPP Group (1999-2004); Deputy Head of \u00d6VP Delegation, European Parliament (1996-2004);", "active": false, "birth_place": "Gries", "full_name": "Hubert PIRKER", + "photo": "http://www.europarl.europa.eu/mepphoto/2307.jpg", "birth_date": "1948-10-03", "slug": "hubert-pirker-1948-10-03" }, - "model": "representatives.representative", - "pk": 1 + "model": "representatives.representative" }, { "fields": { "last_name": "LUDVIGSSON", - "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg", "gender": 2, "first_name": "Olle", "cv": "", "active": true, "birth_place": "H\u00e4ls\u00f6", "full_name": "Olle LUDVIGSSON", + "photo": "http://www.europarl.europa.eu/mepphoto/96673.jpg", "birth_date": "1948-10-28", "slug": "olle-ludvigsson-1948-10-28" }, - "model": "representatives.representative", - "pk": 2 + "model": "representatives.representative" }, { "fields": { "email": "olle.ludvigsson@europarl.europa.eu", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "kind": "official" }, "model": "representatives.email", @@ -43,7 +43,9 @@ { "fields": { "url": "http://www.europarl.europa.eu/meps/en/2307/_home.html", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "kind": "EP" }, "model": "representatives.website", @@ -52,7 +54,9 @@ { "fields": { "url": "http://www.europarl.europa.eu/meps/en/96673/_home.html", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "kind": "EP" }, "model": "representatives.website", @@ -61,7 +65,9 @@ { "fields": { "url": "http://www.sap.se/olle", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "kind": "" }, "model": "representatives.website", @@ -70,7 +76,9 @@ { "fields": { "url": "http://twitter.com/olleludvigsson", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "kind": "twitter" }, "model": "representatives.website", @@ -79,7 +87,9 @@ { "fields": { "url": "https://www.facebook.com/olle.ludvigsson", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "kind": "facebook" }, "model": "representatives.website", @@ -89,41 +99,53 @@ "fields": { "city": "Brussels", "name": "Brussels European Parliament", - "country": 1050, + "country": [ + "BE" + ], "floor": "14G", "number": "60", "street": "rue Wiertz / Wiertzstraat", "office_number": "257", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "location": "", "kind": "official", "postcode": "1047" }, - "model": "representatives.address", - "pk": 1 + "model": "representatives.address" }, { "fields": { "city": "Strasbourg", "name": "Strasbourg European Parliament", - "country": 1095, + "country": [ + "FR" + ], "floor": "T07", "number": "1", "street": "Av. du Pr\u00e9sident Robert Schuman - CS 91024", "office_number": "070", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "location": "", "kind": "official", "postcode": "67070" }, - "model": "representatives.address", - "pk": 2 + "model": "representatives.address" }, { "fields": { "number": "+322 28 45442", - "representative": 2, - "address": 1, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], + "address": [ + "Brussels European Parliament", + "official", + "olle-ludvigsson-1948-10-28" + ], "kind": "office phone" }, "model": "representatives.phone", @@ -132,8 +154,14 @@ { "fields": { "number": "+333 88 1 75442", - "representative": 2, - "address": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], + "address": [ + "Strasbourg European Parliament", + "official", + "olle-ludvigsson-1948-10-28" + ], "kind": "office phone" }, "model": "representatives.phone", @@ -141,72 +169,77 @@ }, { "fields": { + "abbreviation": "EP", "country": null, - "name": "European Parliament", - "abbreviation": "EP" + "name": "European Parliament" }, - "model": "representatives.chamber", - "pk": 1 + "model": "representatives.chamber" }, { "fields": { "name": "European Parliament", "kind": "chamber", "abbreviation": "EP", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 1 + "model": "representatives.group" }, { "fields": { "name": "Committee on Employment and Social Affairs", "kind": "committee", "abbreviation": "EMPL", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 2 + "model": "representatives.group" }, { "fields": { "name": "Delegation for relations with the countries of Southeast Asia and the Association of Southeast Asian Nations (ASEAN)", "kind": "delegation", "abbreviation": "", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 3 + "model": "representatives.group" }, { "fields": { "name": "Delegation for relations with the Member States of ASEAN, South-east Asia and the Republic of Korea", "kind": "delegation", "abbreviation": "", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 4 + "model": "representatives.group" }, { "fields": { "name": "Group of the European People's Party (Christian Democrats) and European Democrats", "kind": "group", "abbreviation": "PPE-DE", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 5 + "model": "representatives.group" }, { "fields": { "name": "Group of the European People's Party (Christian-Democratic Group)", "kind": "group", "abbreviation": "EPP", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 6 + "model": "representatives.group" }, { "fields": { @@ -215,8 +248,7 @@ "abbreviation": "AT", "chamber": null }, - "model": "representatives.group", - "pk": 7 + "model": "representatives.group" }, { "fields": { @@ -225,68 +257,73 @@ "abbreviation": "", "chamber": null }, - "model": "representatives.group", - "pk": 8 + "model": "representatives.group" }, { "fields": { "name": "Committee on Economic and Monetary Affairs", "kind": "committee", "abbreviation": "ECON", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 9 + "model": "representatives.group" }, { "fields": { "name": "Committee on Industry, Research and Energy", "kind": "committee", "abbreviation": "ITRE", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 10 + "model": "representatives.group" }, { "fields": { "name": "Delegation to the EU-Serbia Stabilisation and Association Parliamentary Committee", "kind": "delegation", "abbreviation": "", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 11 + "model": "representatives.group" }, { "fields": { "name": "Delegation for relations with Bosnia and Herzegovina, and Kosovo", "kind": "delegation", "abbreviation": "", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 12 + "model": "representatives.group" }, { "fields": { "name": "Delegation for relations with Australia and New Zealand", "kind": "delegation", "abbreviation": "", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 13 + "model": "representatives.group" }, { "fields": { "name": "Group of the Progressive Alliance of Socialists and Democrats in the European Parliament", "kind": "group", "abbreviation": "SD", - "chamber": 1 + "chamber": [ + "EP" + ] }, - "model": "representatives.group", - "pk": 14 + "model": "representatives.group" }, { "fields": { @@ -295,317 +332,493 @@ "abbreviation": "SE", "chamber": null }, - "model": "representatives.group", - "pk": 15 + "model": "representatives.group" }, { "fields": { "country": null, "name": "European Parliament" }, - "model": "representatives.constituency", - "pk": 1 + "model": "representatives.constituency" }, { "fields": { - "country": 1043, + "country": [ + "AT" + ], "name": "\u00d6sterreichische Volkspartei" }, - "model": "representatives.constituency", - "pk": 2 + "model": "representatives.constituency" }, { "fields": { - "country": 1202, + "country": [ + "SE" + ], "name": "Arbetarepartiet- Socialdemokraterna" }, - "model": "representatives.constituency", - "pk": 3 + "model": "representatives.constituency" }, { "fields": { - "group": 2, + "group": [ + "Committee on Employment and Social Affairs", + "committee", + "EP" + ], "end_date": "1999-07-19", "role": "Substitute", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1997-01-16", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 1 + "model": "representatives.mandate" }, { "fields": { - "group": 3, + "group": [ + "Delegation for relations with the countries of Southeast Asia and the Association of Southeast Asian Nations (ASEAN)", + "delegation", + "EP" + ], "end_date": "2014-06-30", "role": "Member", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "2013-10-09", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 2 + "model": "representatives.mandate" }, { "fields": { - "group": 4, + "group": [ + "Delegation for relations with the Member States of ASEAN, South-east Asia and the Republic of Korea", + "delegation", + "EP" + ], "end_date": "1997-01-15", "role": "Member", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1996-11-14", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 3 + "model": "representatives.mandate" }, { "fields": { - "group": 5, + "group": [ + "Group of the European People's Party (Christian Democrats) and European Democrats", + "group", + "EP" + ], "end_date": "2004-07-19", "role": "Member", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1999-07-20", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 4 + "model": "representatives.mandate" }, { "fields": { - "group": 6, + "group": [ + "Group of the European People's Party (Christian-Democratic Group)", + "group", + "EP" + ], "end_date": "1999-07-19", "role": "Member", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1996-11-11", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 5 + "model": "representatives.mandate" }, { "fields": { - "group": 7, + "group": [ + "Austria", + "country", + null + ], "end_date": "2004-07-19", "role": "", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1999-07-20", - "constituency": 2 + "constituency": [ + "\u00d6sterreichische Volkspartei", + "AT" + ] }, - "model": "representatives.mandate", - "pk": 6 + "model": "representatives.mandate" }, { "fields": { - "group": 1, + "group": [ + "European Parliament", + "chamber", + "EP" + ], "end_date": "2004-07-19", "role": "", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1999-07-20", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 7 + "model": "representatives.mandate" }, { "fields": { - "group": 7, + "group": [ + "Austria", + "country", + null + ], "end_date": "1999-07-19", "role": "", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1996-11-11", - "constituency": 2 + "constituency": [ + "\u00d6sterreichische Volkspartei", + "AT" + ] }, - "model": "representatives.mandate", - "pk": 8 + "model": "representatives.mandate" }, { "fields": { - "group": 1, + "group": [ + "European Parliament", + "chamber", + "EP" + ], "end_date": "1999-07-19", "role": "", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "1996-11-11", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 9 + "model": "representatives.mandate" }, { "fields": { - "group": 8, + "group": [ + "Conference of Delegation Chairs", + "organization", + null + ], "end_date": "2009-07-13", "role": "Member", - "representative": 1, + "representative": [ + "hubert-pirker-1948-10-03" + ], "link": "", "begin_date": "2006-03-21", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 10 + "model": "representatives.mandate" }, { "fields": { - "group": 9, + "group": [ + "Committee on Economic and Monetary Affairs", + "committee", + "EP" + ], "end_date": "9999-12-31", "role": "Member", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-01", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 11 + "model": "representatives.mandate" }, { "fields": { - "group": 10, + "group": [ + "Committee on Industry, Research and Energy", + "committee", + "EP" + ], "end_date": "9999-12-31", "role": "Substitute", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-08", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 12 + "model": "representatives.mandate" }, { "fields": { - "group": 9, + "group": [ + "Committee on Economic and Monetary Affairs", + "committee", + "EP" + ], "end_date": "2012-01-18", "role": "Substitute", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2009-07-16", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 13 + "model": "representatives.mandate" }, { "fields": { - "group": 11, + "group": [ + "Delegation to the EU-Serbia Stabilisation and Association Parliamentary Committee", + "delegation", + "EP" + ], "end_date": "9999-12-31", "role": "Member", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-14", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 14 + "model": "representatives.mandate" }, { "fields": { - "group": 12, + "group": [ + "Delegation for relations with Bosnia and Herzegovina, and Kosovo", + "delegation", + "EP" + ], "end_date": "9999-12-31", "role": "Substitute", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2015-05-18", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 15 + "model": "representatives.mandate" }, { "fields": { - "group": 13, + "group": [ + "Delegation for relations with Australia and New Zealand", + "delegation", + "EP" + ], "end_date": "2013-01-10", "role": "Substitute", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2009-09-17", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 16 + "model": "representatives.mandate" }, { "fields": { - "group": 14, + "group": [ + "Group of the Progressive Alliance of Socialists and Democrats in the European Parliament", + "group", + "EP" + ], "end_date": "9999-12-31", "role": "Member", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-01", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 17 + "model": "representatives.mandate" }, { "fields": { - "group": 14, + "group": [ + "Group of the Progressive Alliance of Socialists and Democrats in the European Parliament", + "group", + "EP" + ], "end_date": "2014-06-30", "role": "Member", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2009-07-14", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 18 + "model": "representatives.mandate" }, { "fields": { - "group": 15, + "group": [ + "Sweden", + "country", + null + ], "end_date": "9999-12-31", "role": "", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-01", - "constituency": 3 + "constituency": [ + "Arbetarepartiet- Socialdemokraterna", + "SE" + ] }, - "model": "representatives.mandate", - "pk": 19 + "model": "representatives.mandate" }, { "fields": { - "group": 1, + "group": [ + "European Parliament", + "chamber", + "EP" + ], "end_date": "9999-12-31", "role": "", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2014-07-01", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 20 + "model": "representatives.mandate" }, { "fields": { - "group": 15, + "group": [ + "Sweden", + "country", + null + ], "end_date": "2014-06-30", "role": "", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2009-07-14", - "constituency": 3 + "constituency": [ + "Arbetarepartiet- Socialdemokraterna", + "SE" + ] }, - "model": "representatives.mandate", - "pk": 21 + "model": "representatives.mandate" }, { "fields": { - "group": 1, + "group": [ + "European Parliament", + "chamber", + "EP" + ], "end_date": "2014-06-30", "role": "", - "representative": 2, + "representative": [ + "olle-ludvigsson-1948-10-28" + ], "link": "", "begin_date": "2009-07-14", - "constituency": 1 + "constituency": [ + "European Parliament", + null + ] }, - "model": "representatives.mandate", - "pk": 22 + "model": "representatives.mandate" } ] diff --git a/src/representatives/contrib/parltrack/tests/test_import_representatives.py b/src/representatives/contrib/parltrack/tests/test_import_representatives.py index 3ee2658..cdb4557 100644 --- a/src/representatives/contrib/parltrack/tests/test_import_representatives.py +++ b/src/representatives/contrib/parltrack/tests/test_import_representatives.py @@ -1,38 +1,19 @@ -import pytest import os -import copy -from django.core.serializers.json import Deserializer from representatives.models import Representative +from representatives.tests.base import TestBase from representatives.contrib.parltrack import import_representatives -@pytest.mark.django_db -def test_parltrack_import_representatives(): - fixture = os.path.join(os.path.dirname(__file__), - 'representatives_fixture.json') - expected = os.path.join(os.path.dirname(__file__), - 'representatives_expected.json') +class ParltracRepresentativesTest(TestBase): + def test_parltrack_import_representatives(self): + fixture = os.path.join(os.path.dirname(__file__), + 'representatives_fixture.json') + expected = os.path.join(os.path.dirname(__file__), + 'representatives_expected.json') - # Disable django auto fields - exclude = ('id', '_state', 'created', 'updated', 'fingerprint') + with open(fixture, 'r') as f: + import_representatives.main(f) - with open(fixture, 'r') as f: - import_representatives.main(f) - - missing = [] - with open(expected, 'r') as f: - for obj in Deserializer(f.read()): - compare = copy.copy(obj.object.__dict__) - - for field in exclude: - if field in compare: - compare.pop(field) - - try: - type(obj.object).objects.get(**compare) - except: - missing.append(compare) - - assert len(missing) is 0 - assert Representative.objects.count() == 2 + self.assertObjectsFromFixture(expected) + assert Representative.objects.count() == 2 diff --git a/src/representatives/models.py b/src/representatives/models.py index 74cf0be..628189e 100644 --- a/src/representatives/models.py +++ b/src/representatives/models.py @@ -20,19 +20,36 @@ class TimeStampedModel(models.Model): abstract = True +class CountryManager(models.Manager): + def get_by_natural_key(self, code): + return self.get(code=code) + + class Country(models.Model): + objects = CountryManager() + name = models.CharField(max_length=255) code = models.CharField(max_length=2, unique=True) def __unicode__(self): return u'{} [{}]'.format(self.name, self.code) + def natural_key(self): + return (self.code,) + + +class RepresentativeManager(models.Manager): + def get_by_natural_key(self, slug): + return self.get(slug=slug) + class Representative(TimeStampedModel): """ Base model for representatives """ + objects = RepresentativeManager() + slug = models.SlugField(max_length=100, unique=True) first_name = models.CharField(max_length=255, blank=True, default='') last_name = models.CharField(max_length=255, blank=True, default='') @@ -52,6 +69,9 @@ class Representative(TimeStampedModel): def __unicode__(self): return smart_unicode(self.full_name) + def natural_key(self): + return (self.slug,) + def gender_as_str(self): genders = {0: 'N/A', 1: 'F', 2: 'M'} return genders[self.gender] @@ -79,7 +99,15 @@ class WebSite(Contact): kind = models.CharField(max_length=255, blank=True, default='') +class AddressManager(models.Manager): + def get_by_natural_key(self, name, kind, representative_slug): + representative = Representative.objects.get(slug=representative_slug) + return self.get(name=name, kind=kind, representative=representative) + + class Address(Contact): + objects = AddressManager() + country = models.ForeignKey(Country) city = models.CharField(max_length=255, blank=True, default='') street = models.CharField(max_length=255, blank=True, default='') @@ -91,6 +119,10 @@ class Address(Contact): name = models.CharField(max_length=255, blank=True, default='') location = models.CharField(max_length=255, blank=True, default='') + def natural_key(self): + representative_nk = self.representative.natural_key() + return (self.name, self.kind) + representative_nk + class Phone(Contact): number = models.CharField(max_length=255, blank=True, default='') @@ -98,10 +130,17 @@ class Phone(Contact): address = models.ForeignKey(Address, null=True, related_name='phones') +class ChamberManager(models.Manager): + def get_by_natural_key(self, abbreviation): + return self.get(abbreviation=abbreviation) + + class Chamber(models.Model): """ A representative chamber """ + objects = ChamberManager() + name = models.CharField(max_length=255) country = models.ForeignKey('Country', null=True, related_name='chambers') abbreviation = models.CharField(max_length=10, blank=True, default='', @@ -110,11 +149,26 @@ class Chamber(models.Model): def __unicode__(self): return u'{} [{}]'.format(self.name, self.abbreviation) + def natural_key(self): + return (self.abbreviation,) + + +class GroupManager(models.Manager): + def get_by_natural_key(self, name, kind, chamber_nk): + if chamber_nk: + chamber = Chamber.objects.get_by_natural_key(chamber_nk) + else: + chamber = None + + return self.get(name=name, kind=kind, chamber=chamber) + class Group(TimeStampedModel): """ An entity represented by a representative through a mandate """ + objects = GroupManager() + name = models.CharField(max_length=511, db_index=True) abbreviation = models.CharField(max_length=10, blank=True, default='', db_index=True) @@ -128,14 +182,30 @@ class Group(TimeStampedModel): def __unicode__(self): return unicode(self.name) + def natural_key(self): + chamber_nk = self.chamber.natural_key() if self.chamber else (None,) + return (self.name, self.kind) + chamber_nk + class Meta: ordering = ('name',) +class ConstituencyManager(models.Manager): + def get_by_natural_key(self, name, country_nk): + if country_nk: + country = Country.objects.get_by_natural_key(country_nk) + else: + country = None + + return self.get(name=name, country=country) + + class Constituency(TimeStampedModel): """ An authority for which a representative has a mandate """ + objects = ConstituencyManager() + name = models.CharField(max_length=255) country = models.ForeignKey('Country', null=True, blank=True, related_name='constituencies') @@ -147,6 +217,10 @@ class Constituency(TimeStampedModel): def __unicode__(self): return unicode(self.name) + def natural_key(self): + country_nk = self.country.natural_key() if self.country else (None,) + return (self.name,) + country_nk + class MandateManager(models.Manager): """ This satisfies repr(Mandate) """ @@ -158,8 +232,18 @@ class MandateManager(models.Manager): 'constituency') -class Mandate(TimeStampedModel): +class MandateManager(models.Manager): + def get_by_natural_key(self, begin_date, end_date, representative_slug, + group_name, group_kind, group_chamber_nk): + representative = Representative.objects.get_by_natural_key( + representative_slug) + group = Group.objects.get_by_natural_key(group_name, group_kind, + group_chamber_nk) + return self.get(begin_date=begin_date, end_date=end_date, + representative=representative, group=group) + +class Mandate(TimeStampedModel): objects = MandateManager() group = models.ForeignKey(Group, null=True, related_name='mandates') @@ -190,5 +274,13 @@ class Mandate(TimeStampedModel): constituency=self.constituency, group=self.group) + def natural_key(self): + rep_nk = self.representative.natural_key() \ + if self.representative else (None,) + group_nk = self.group.natural_key() \ + if self.group else (None, None, None) + + return (self.begin_date, self.end_date) + rep_nk + group_nk + class Meta: ordering = ('-end_date',) diff --git a/src/representatives/tests/base.py b/src/representatives/tests/base.py new file mode 100644 index 0000000..8bc6d59 --- /dev/null +++ b/src/representatives/tests/base.py @@ -0,0 +1,38 @@ +import copy +import logging +import re + +from django import test +from django.core.serializers.json import Deserializer + + +class TestBase(test.TestCase): + logger = logging.getLogger('representatives') + + _exclude = ('id', '_state', 'created', 'updated', 'fingerprint') + _exclude_re = (re.compile(r'^_.*_cache$'),) + + def assertObjectsFromFixture(self, fixture): + missing = [] + + with open(fixture, 'r') as f: + for obj in Deserializer(f.read()): + compare = copy.copy(obj.object.__dict__) + + for field in self._exclude: + if field in compare: + compare.pop(field) + + for re in self._exclude_re: + for field in compare.keys(): + if re.match(field): + compare.pop(field) + + try: + type(obj.object).objects.get(**compare) + except: + self.logger.warn( + 'MISSING %s\n %s' % (type(obj.object), compare)) + missing.append(compare) + + assert len(missing) is 0 -- GitLab