diff --git a/representatives_votes/admin.py b/representatives_votes/admin.py index 917a44f6132edcabeb727731ccba63916f555f3b..c664ccbc2f58f987ab8c8090d38e486ff248e2fa 100644 --- a/representatives_votes/admin.py +++ b/representatives_votes/admin.py @@ -2,14 +2,22 @@ from django.contrib import admin -from .models import Dossier, Proposal, Vote +from .models import Dossier, Document, Proposal, Vote class DossierAdmin(admin.ModelAdmin): - list_display = ('id', 'reference', 'title', 'link') + list_display = ('id', 'reference', 'title') search_fields = ('reference', 'title') +class DocumentAdmin(admin.ModelAdmin): + list_display = ('dossier_reference', 'kind', 'title', 'link') + search_fields = ('reference', 'dossier__reference', 'title') + + def dossier_reference(self, obj): + return obj.dossier.reference + + class ProposalAdmin(admin.ModelAdmin): list_display = ( 'reference', @@ -49,5 +57,6 @@ class VoteAdmin(admin.ModelAdmin): return obj.proposal.reference admin.site.register(Dossier, DossierAdmin) +admin.site.register(Document, DocumentAdmin) admin.site.register(Proposal, ProposalAdmin) admin.site.register(Vote, VoteAdmin) diff --git a/representatives_votes/api.py b/representatives_votes/api.py index 0cbddf66d9b6e3c647864b5a2975cf1d0696f013..7bb21681bf5fe894a4c8f5975bdf3d3badfb0319 100644 --- a/representatives_votes/api.py +++ b/representatives_votes/api.py @@ -51,7 +51,8 @@ class DossierViewSet(viewsets.ReadOnlyModelViewSet): def retrieve(self, request, pk=None): self.serializer_class = DossierDetailSerializer - self.queryset = self.queryset.prefetch_related('proposals') + self.queryset = self.queryset.prefetch_related('proposals', + 'documents') return super(DossierViewSet, self).retrieve(request, pk) diff --git a/representatives_votes/contrib/francedata/import_dossiers.py b/representatives_votes/contrib/francedata/import_dossiers.py index caba0efc7575e886560d16c3bcdb8442f54b49b7..5e5f640b522bc957466a75bec3087b18b0fe976d 100644 --- a/representatives_votes/contrib/francedata/import_dossiers.py +++ b/representatives_votes/contrib/francedata/import_dossiers.py @@ -3,27 +3,48 @@ import sys import ijson import logging +import re import django from django.apps import apps +from django.db import transaction -from representatives_votes.models import Dossier +from representatives.contrib.francedata.import_representatives import \ + ensure_chambers +from representatives.models import Chamber +from representatives_votes.models import Document, Dossier logger = logging.getLogger(__name__) +def extract_reference(url): + m = re.search(r'/dossier-legislatif/([^./]+)\.html', url) + if m: + return m.group(1) + + m = re.search(r'/(\d+)/dossiers/([^./]+)\.asp', url) + if m: + return '%s/%s' % (m.group(1), m.group(2)) + + m = re.search(r'/dossiers/([^./]+)\.asp', url) + if m: + return m.group(1) + + return None + + def find_dossier(data): ''' - Find dossier with reference matching either 'url_an' or 'url_sen', - create it if not found. Ensure its reference and source are 'url_an' if - both fields are present. + Find dossier with reference matching either 'ref_an' or 'ref_sen', + create it if not found. Ensure its reference is 'ref_an' if both fields + are present. ''' changed = False dossier = None reffield = None - for field in [k for k in ('url_an', 'url_sen') if k in data]: + for field in [k for k in ('ref_an', 'ref_sen') if k in data]: try: dossier = Dossier.objects.get(reference=data[field]) reffield = field @@ -32,50 +53,85 @@ def find_dossier(data): pass if dossier is None: - reffield = 'url_an' if 'url_an' in data else 'url_sen' + reffield = 'ref_an' if 'ref_an' in data else 'ref_sen' dossier = Dossier(reference=data[reffield]) logger.debug('Created dossier %s' % data[reffield]) changed = True - if 'url_an' in data and reffield != 'url_an': - logger.debug('Changed dossier reference to %s' % data['url_an']) - dossier.reference = data['url_an'] + if 'ref_an' in data and reffield != 'ref_an': + logger.debug('Changed dossier reference to %s' % data['ref_an']) + dossier.reference = data['ref_an'] changed = True return dossier, changed -def parse_dossier_data(data): - dossier, changed = find_dossier(data) +def handle_document(dossier, chamber, url): + doc_changed = False + try: + doc = Document.objects.get(chamber=chamber, dossier=dossier, + kind='procedure-file') + except Document.DoesNotExist: + doc = Document(chamber=chamber, dossier=dossier, kind='procedure-file') + logger.debug('Created %s document for dossier %s' % + (chamber.abbreviation, dossier.title)) + doc_changed = True + + if doc.link != url: + logger.debug('Changing %s url from %s to %s' % + (chamber.abbreviation, doc.link, url)) + doc.link = url + doc_changed = True + + if doc_changed: + doc.save() + + +def parse_dossier_data(data, an, sen): + if 'url_an' in data: + ref_an = extract_reference(data['url_an']) + if ref_an is None: + logger.warn('No reference for dossier %s' % data['url_an']) + return + else: + data['ref_an'] = ref_an + + if 'url_sen' in data: + ref_sen = extract_reference(data['url_sen']) + if ref_sen is None: + logger.warn('No reference for dossier %s' % data['url_sen']) + return + else: + data['ref_sen'] = ref_sen - thisurl = data['url_an' if data['chambre'] == 'AN' else 'url_sen'] + dossier, changed = find_dossier(data) - if dossier.reference != dossier.link: - logger.debug('Changed dossier link to %s' % dossier.reference) - dossier.link = dossier.reference - changed = True + thisref = data['ref_an' if data['chambre'] == 'AN' else 'ref_sen'] title = data['titre'] - if dossier.reference == thisurl and dossier.title != title: + if dossier.reference == thisref and dossier.title != title: logger.debug('Changed dossier title to %s' % title) dossier.title = title changed = True - if 'url_an' in data and 'url_sen' in data: - ext_link = data['url_sen'] - if dossier.ext_link != ext_link: - logger.debug('Changed dossier ext. link to %s' % ext_link) - dossier.ext_link = ext_link - changed = True + with transaction.atomic(): + if changed: + logger.debug('Saved dossier %s' % dossier.reference) + dossier.save() + + if 'url_an' in data: + handle_document(dossier, an, data['url_an']) - if changed: - logger.debug('Saved dossier %s' % dossier.reference) - dossier.save() + if 'url_sen' in data: + handle_document(dossier, sen, data['url_sen']) def main(stream=None): if not apps.ready: django.setup() + ensure_chambers() + an = Chamber.objects.get(abbreviation='AN') + sen = Chamber.objects.get(abbreviation='SEN') for data in ijson.items(stream or sys.stdin, 'item'): - parse_dossier_data(data) + parse_dossier_data(data, an, sen) diff --git a/representatives_votes/contrib/francedata/import_scrutins.py b/representatives_votes/contrib/francedata/import_scrutins.py index 2233bef1203a988914104b0b4411e664880637db..3ce820843a93763fff2743f43c8c4ca4fa9fa3cf 100644 --- a/representatives_votes/contrib/francedata/import_scrutins.py +++ b/representatives_votes/contrib/francedata/import_scrutins.py @@ -48,23 +48,16 @@ def _get_unique_title(proposal_pk, candidate): class ScrutinImporter: - dossiers_ref = None - dossiers_ext = None + dossiers = {} def get_dossier(self, url): - if self.dossiers_ref is None: - self.dossiers_ref = { - d[0]: d[1] for d in Dossier.objects.values_list('reference', - 'pk') - } - - if self.dossiers_ext is None: - self.dossiers_ext = { - d[0]: d[1] for d in Dossier.objects.exclude(ext_link='') - .values_list('ext_link', 'pk') - } + if url not in self.dossiers: + try: + self.dossiers[url] = Dossier.objects.get(documents__link=url) + except Dossier.DoesNotExist: + return None - return self.dossiers_ref.get(url, self.dossiers_ext.get(url, None)) + return self.dossiers[url] def parse_scrutin_data(self, data): ref = data['url'] @@ -91,7 +84,7 @@ class ScrutinImporter: values = dict( title=_get_unique_title(proposal.pk, data["objet"]), datetime=_parse_date(data["date"]), - dossier_id=dossier, + dossier_id=dossier.pk, kind='dossier' ) diff --git a/representatives_votes/contrib/francedata/tests/dossiers_expected.json b/representatives_votes/contrib/francedata/tests/dossiers_expected.json index bf3a458b9b0f97313631508d94e35f63b56a814e..bae5aa5690e9a3f8756e8053cf00f4c9b0622554 100644 --- a/representatives_votes/contrib/francedata/tests/dossiers_expected.json +++ b/representatives_votes/contrib/francedata/tests/dossiers_expected.json @@ -1,41 +1,105 @@ [ +{ + "fields" : { + "abbreviation": "AN", + "country": 1095, + "name": "Assembl\u00e9e nationale" + }, + "model": "representatives.chamber", + "pk": 2 +}, +{ + "fields": { + "abbreviation": "SEN", + "country": 1095, + "name": "S\u00e9nat" + }, + "model": "representatives.chamber", + "pk": 3 +}, { "fields": { - "updated": "2016-02-14T13:16:31.417Z", - "reference": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp", - "title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9", "text": "", - "created": "2016-02-14T13:16:31.417Z", - "link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp", - "ext_link": "" + "updated": "2016-07-07T20:23:24.303Z", + "title": "Education : libre choix des maires concernant les rythmes scolaires dans le premier degr\u00e9", + "reference": "14/liberte_maires_rythmes_scolaires_premier_degre", + "created": "2016-07-07T20:23:24.302Z" }, "model": "representatives_votes.dossier", "pk": 1 }, { "fields": { - "updated": "2016-02-14T13:16:31.428Z", - "reference": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp", - "title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles", "text": "", - "created": "2016-02-14T13:16:31.428Z", - "link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp", - "ext_link": "http://www.senat.fr/dossier-legislatif/pjl12-495.html" + "updated": "2016-07-07T20:23:24.365Z", + "title": "Collectivit\u00e9s territoriales : action publique territoriale et m\u00e9tropoles", + "reference": "14/action_publique_territoriale_metropoles", + "created": "2016-07-07T20:23:24.332Z" }, "model": "representatives_votes.dossier", "pk": 2 }, { "fields": { - "updated": "2016-02-21T14:34:35.721Z", - "reference": "http://www.senat.fr/dossier-legislatif/ppl13-799.html", - "title": "Protection de l'enfant", "text": "", - "created": "2016-02-21T14:34:35.721Z", - "link": "http://www.senat.fr/dossier-legislatif/ppl13-799.html", - "ext_link": "" + "updated": "2016-07-07T20:23:24.410Z", + "title": "Protection de l'enfant", + "reference": "ppl13-799", + "created": "2016-07-07T20:23:24.410Z" }, "model": "representatives_votes.dossier", "pk": 3 +}, +{ + "fields": { + "updated": "2016-07-07T20:23:24.307Z", + "title": "", + "dossier": 1, + "created": "2016-07-07T20:23:24.307Z", + "kind": "procedure-file", + "chamber": 2, + "link": "http://www.assemblee-nationale.fr/14/dossiers/liberte_maires_rythmes_scolaires_premier_degre.asp" + }, + "model": "representatives_votes.document", + "pk": 1 +}, +{ + "fields": { + "updated": "2016-07-07T20:23:24.335Z", + "title": "", + "dossier": 2, + "created": "2016-07-07T20:23:24.335Z", + "kind": "procedure-file", + "chamber": 3, + "link": "http://www.senat.fr/dossier-legislatif/pjl12-495.html" + }, + "model": "representatives_votes.document", + "pk": 2 +}, +{ + "fields": { + "updated": "2016-07-07T20:23:24.371Z", + "title": "", + "dossier": 2, + "created": "2016-07-07T20:23:24.371Z", + "kind": "procedure-file", + "chamber": 2, + "link": "http://www.assemblee-nationale.fr/14/dossiers/action_publique_territoriale_metropoles.asp" + }, + "model": "representatives_votes.document", + "pk": 3 +}, +{ + "fields": { + "updated": "2016-07-07T20:23:24.415Z", + "title": "", + "dossier": 3, + "created": "2016-07-07T20:23:24.415Z", + "kind": "procedure-file", + "chamber": 3, + "link": "http://www.senat.fr/dossier-legislatif/ppl13-799.html" + }, + "model": "representatives_votes.document", + "pk": 4 } ] diff --git a/representatives_votes/contrib/parltrack/import_dossiers.py b/representatives_votes/contrib/parltrack/import_dossiers.py index dd9fdf8ef025d2f7d84c52258a3c4b64a60b6e78..3728674256bc833ec9ba9a251df9122b03fcfeab 100644 --- a/representatives_votes/contrib/parltrack/import_dossiers.py +++ b/representatives_votes/contrib/parltrack/import_dossiers.py @@ -6,8 +6,10 @@ import urllib2 import ijson import django from django.apps import apps +from django.db import transaction -from representatives_votes.models import Dossier +from representatives.models import Chamber +from representatives_votes.models import Dossier, Document from .import_votes import Command logger = logging.getLogger(__name__) @@ -17,38 +19,51 @@ URL = 'http://parltrack.euwiki.org/dumps/ep_dossiers.json.xz' LOCAL_PATH = 'ep_dossiers.json.xz' -def parse_dossier_data(data): +def parse_dossier_data(data, ep): """Parse data from parltarck dossier export (1 dossier) Update dossier if it existed before, this function goal is to import and update a dossier, not to import all parltrack data """ changed = False + doc_changed = False ref = data['procedure']['reference'] logger.debug('Processing dossier %s', ref) - try: - dossier = Dossier.objects.get(reference=ref) - except Dossier.DoesNotExist: - dossier = Dossier(reference=ref) - logger.debug('Dossier did not exist') - changed = True - - if dossier.title != data['procedure']['title']: - logger.debug('Title changed from "%s" to "%s"', dossier.title, - data['procedure']['title']) - dossier.title = data['procedure']['title'] - changed = True - - source = data['meta']['source'].replace('&l=en', '') - if dossier.link != source: - logger.debug('Source changed from "%s" to "%s"', dossier.link, source) - dossier.link = source - changed = True - - if changed: - logger.info('Updated dossier %s', ref) - dossier.save() + with transaction.atomic(): + try: + dossier = Dossier.objects.get(reference=ref) + except Dossier.DoesNotExist: + dossier = Dossier(reference=ref) + logger.debug('Dossier did not exist') + changed = True + + if dossier.title != data['procedure']['title']: + logger.debug('Title changed from "%s" to "%s"', dossier.title, + data['procedure']['title']) + dossier.title = data['procedure']['title'] + changed = True + + if changed: + logger.info('Updated dossier %s', ref) + dossier.save() + + source = data['meta']['source'].replace('&l=en', '') + try: + doc = Document.objects.get(dossier=dossier, kind='procedure-file') + except Document.DoesNotExist: + doc = Document(dossier=dossier, kind='procedure-file', chamber=ep) + logger.debug('Document for dossier %s did not exist', ref) + doc_changed = True + + if doc.link != source: + logger.debug('Link changed from %s to %s', doc.link, source) + doc.link = source + doc_changed = True + + if doc_changed: + logger.info('Updated document %s for dossier %s', doc.link, ref) + doc.save() if 'votes' in data.keys() and 'epref' in data['votes']: command = Command() @@ -68,13 +83,15 @@ def import_single(stream): if not apps.ready: django.setup() + ep = Chamber.objects.get(abbreviation='EP') for data in ijson.items(stream, ''): - parse_dossier_data(data) + parse_dossier_data(data, ep) def main(stream=None): if not apps.ready: django.setup() + ep = Chamber.objects.get(abbreviation='EP') for data in ijson.items(stream or sys.stdin, 'item'): - parse_dossier_data(data) + parse_dossier_data(data, ep) diff --git a/representatives_votes/contrib/parltrack/tests/dossiers_expected.json b/representatives_votes/contrib/parltrack/tests/dossiers_expected.json index 493e4a5a96192b9aba69dab9eefc7a066ace1bba..d6e17d7617ce90d485cfc7da4939b25203c564ed 100644 --- a/representatives_votes/contrib/parltrack/tests/dossiers_expected.json +++ b/representatives_votes/contrib/parltrack/tests/dossiers_expected.json @@ -1,62 +1,122 @@ [ { "fields": { - "updated": "2015-12-13T10:11:31.369Z", - "reference": "2012/2002(INI)", - "title": "Agenda for change: the future of EU development policy", "text": "", - "created": "2015-12-13T10:11:31.369Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + "updated": "2016-07-08T05:17:40.580Z", + "title": "Agenda for change: the future of EU development policy", + "reference": "2012/2002(INI)", + "created": "2016-07-08T05:17:40.580Z" }, "model": "representatives_votes.dossier", "pk": 1 }, { "fields": { - "updated": "2015-12-13T10:11:31.378Z", - "reference": "2015/2132(BUD)", - "title": "2016 general budget: all sections", "text": "", - "created": "2015-12-13T10:11:31.378Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + "updated": "2016-07-08T05:17:40.617Z", + "title": "2016 general budget: all sections", + "reference": "2015/2132(BUD)", + "created": "2016-07-08T05:17:40.616Z" }, "model": "representatives_votes.dossier", "pk": 2 }, { "fields": { - "updated": "2015-12-13T10:11:31.388Z", - "reference": "2013/2857(DEA)", - "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", "text": "", - "created": "2015-12-13T10:11:31.388Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)" + "updated": "2016-07-08T05:17:40.644Z", + "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", + "reference": "2013/2857(DEA)", + "created": "2016-07-08T05:17:40.644Z" }, "model": "representatives_votes.dossier", "pk": 3 }, { "fields": { - "updated": "2015-12-13T10:11:31.398Z", - "reference": "2015/2623(DEA)", - "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", "text": "", - "created": "2015-12-13T10:11:31.398Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)" + "updated": "2016-07-08T05:17:40.682Z", + "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", + "reference": "2015/2623(DEA)", + "created": "2016-07-08T05:17:40.682Z" }, "model": "representatives_votes.dossier", "pk": 4 }, { "fields": { - "updated": "2015-12-13T10:11:31.408Z", - "reference": "2009/0051(COD)", - "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", "text": "", - "created": "2015-12-13T10:11:31.408Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)" + "updated": "2016-07-08T05:17:40.719Z", + "title": "Scheme of control and enforcement applicable in the area covered by the Convention on future multilateral cooperation in the North-East Atlantic fisheries", + "reference": "2009/0051(COD)", + "created": "2016-07-08T05:17:40.719Z" }, "model": "representatives_votes.dossier", "pk": 5 +}, +{ + "fields": { + "updated": "2016-07-08T05:17:40.582Z", + "title": "", + "dossier": 1, + "created": "2016-07-08T05:17:40.582Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + }, + "model": "representatives_votes.document", + "pk": 1 +}, +{ + "fields": { + "updated": "2016-07-08T05:17:40.619Z", + "title": "", + "dossier": 2, + "created": "2016-07-08T05:17:40.619Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + }, + "model": "representatives_votes.document", + "pk": 2 +}, +{ + "fields": { + "updated": "2016-07-08T05:17:40.646Z", + "title": "", + "dossier": 3, + "created": "2016-07-08T05:17:40.646Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2013/2857(DEA)" + }, + "model": "representatives_votes.document", + "pk": 3 +}, +{ + "fields": { + "updated": "2016-07-08T05:17:40.684Z", + "title": "", + "dossier": 4, + "created": "2016-07-08T05:17:40.684Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2623(DEA)" + }, + "model": "representatives_votes.document", + "pk": 4 +}, +{ + "fields": { + "updated": "2016-07-08T05:17:40.724Z", + "title": "", + "dossier": 5, + "created": "2016-07-08T05:17:40.724Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2009/0051(COD)" + }, + "model": "representatives_votes.document", + "pk": 5 } ] diff --git a/representatives_votes/contrib/parltrack/tests/single_expected.json b/representatives_votes/contrib/parltrack/tests/single_expected.json index 7f21fc5a90305d73c9ddfa46857e511b257207e7..678e8de268771783dd469984813877e911021325 100644 --- a/representatives_votes/contrib/parltrack/tests/single_expected.json +++ b/representatives_votes/contrib/parltrack/tests/single_expected.json @@ -1,16 +1,45 @@ [ { "fields": { - "updated": "2015-12-13T10:11:31.378Z", - "reference": "2015/2132(BUD)", - "title": "2016 general budget: all sections", "text": "", - "created": "2015-12-13T10:11:31.378Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + "updated": "2016-07-08T05:20:11.662Z", + "title": "2016 general budget: all sections", + "reference": "2015/2132(BUD)", + "created": "2016-07-08T05:20:11.662Z" }, "model": "representatives_votes.dossier", "pk": 1 }, +{ + "fields": { + "updated": "2016-07-08T05:20:11.664Z", + "title": "", + "dossier": 1, + "created": "2016-07-08T05:20:11.664Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + }, + "model": "representatives_votes.document", + "pk": 1 +}, +{ + "fields": { + "updated": "2016-07-08T05:21:34.553Z", + "total_for": 1, + "description": "", + "reference": "A8-0298/2015", + "title": "A8-0298/2015 - Jos\u00e9 Manuel Fernandes et G\u00e9rard Deprez - Am 29", + "dossier": 1, + "created": "2016-07-08T05:20:11.696Z", + "kind": "Am 29", + "datetime": "2015-10-28T12:00:12Z", + "total_against": 1, + "total_abstain": 0 + }, + "model": "representatives_votes.proposal", + "pk": 1 +}, { "fields": { "representative_name": "", @@ -19,7 +48,7 @@ "representative": 1 }, "model": "representatives_votes.vote", - "pk": 2 + "pk": 1 }, { "fields": { @@ -29,6 +58,6 @@ "representative": 2 }, "model": "representatives_votes.vote", - "pk": 3 + "pk": 2 } ] diff --git a/representatives_votes/contrib/parltrack/tests/single_fixture.json b/representatives_votes/contrib/parltrack/tests/single_fixture.json index 7bdc7bb98f06734fd1c0f2c9bcadc79f2ad32315..4cf45ac3ec769d2188e6de52a225c592136bf65d 100644 --- a/representatives_votes/contrib/parltrack/tests/single_fixture.json +++ b/representatives_votes/contrib/parltrack/tests/single_fixture.json @@ -66,4 +66,4 @@ "url": "http://www.europarl.europa.eu/RegData/seance_pleniere/proces_verbal/2015/10-28/votes_nominaux/xml/P8_PV(2015)10-28(RCV)_XC.xml", "voteid": "59146" } -} +} \ No newline at end of file diff --git a/representatives_votes/contrib/parltrack/tests/sync_expected.json b/representatives_votes/contrib/parltrack/tests/sync_expected.json index 6840d6979a65ac9957e7240119232c59fa62f020..1c61f15c764680dc006c229cb78e1860720d7cf7 100644 --- a/representatives_votes/contrib/parltrack/tests/sync_expected.json +++ b/representatives_votes/contrib/parltrack/tests/sync_expected.json @@ -5,8 +5,7 @@ "reference": "2012/2002(INI)", "title": "Agenda for change: the future of EU development policy", "text": "", - "created": "2015-12-13T10:11:31.369Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + "created": "2015-12-13T10:11:31.369Z" }, "model": "representatives_votes.dossier", "pk": 1 diff --git a/representatives_votes/contrib/parltrack/tests/sync_fixture.json b/representatives_votes/contrib/parltrack/tests/sync_fixture.json index fc2725c367c13179b48ea810b1ddc4eac895fe92..0f22eef6db9c5b45ccfc201faf3ba2cdc94706e0 100644 --- a/representatives_votes/contrib/parltrack/tests/sync_fixture.json +++ b/representatives_votes/contrib/parltrack/tests/sync_fixture.json @@ -5,8 +5,7 @@ "reference": "2012/2002(INI)", "title": "initial title", "text": "", - "created": "2015-12-13T10:11:31.369Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + "created": "2015-12-13T10:11:31.369Z" }, "model": "representatives_votes.dossier", "pk": 1 diff --git a/representatives_votes/contrib/parltrack/tests/test_parltrack_import.py b/representatives_votes/contrib/parltrack/tests/test_parltrack_import.py index 079f7b8384560aaebdc24af10b230ac70791b99e..a8255f48bd5e21562f85587f93d4ec35502d0e94 100644 --- a/representatives_votes/contrib/parltrack/tests/test_parltrack_import.py +++ b/representatives_votes/contrib/parltrack/tests/test_parltrack_import.py @@ -64,7 +64,7 @@ class DossierTest(TestCase): representatives.__path__[0]), 'fixtures', 'representatives_test.json')) - with self.assertNumQueries(15): + with self.assertNumQueries(22): _test_import('single', import_dossiers.import_single) def test_parltrack_sync_dossier(self): @@ -87,7 +87,7 @@ class DossierTest(TestCase): with open(mock_file, 'r') as mock_stream: urlopen.return_value = mock_stream - with self.assertNumQueries(3): + with self.assertNumQueries(8): _test_import('sync', callback) urlopen.assert_called_with(expected_url) diff --git a/representatives_votes/contrib/parltrack/tests/votes_initial.json b/representatives_votes/contrib/parltrack/tests/votes_initial.json index 41ad35d65e0102afa52efe4e955f010516cec4ca..c55721694227fb4c3ea91ba41f6aae27fbe8fd44 100644 --- a/representatives_votes/contrib/parltrack/tests/votes_initial.json +++ b/representatives_votes/contrib/parltrack/tests/votes_initial.json @@ -17,8 +17,7 @@ "reference": "2015/2132(BUD)", "title": "2016 general budget: all sections", "text": "", - "created": "2015-12-13T01:10:09.698Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + "created": "2015-12-13T01:10:09.698Z" }, "model": "representatives_votes.dossier", "pk": 65 diff --git a/representatives_votes/fixtures/representatives_votes_test.json b/representatives_votes/fixtures/representatives_votes_test.json index ae48c5a63a7f4ee5bbf3f70eb46977328c5e309a..491860d28d6eed2165d735b991e3b8d52e9f27f4 100644 --- a/representatives_votes/fixtures/representatives_votes_test.json +++ b/representatives_votes/fixtures/representatives_votes_test.json @@ -5,8 +5,7 @@ "reference": "2012/2002(INI)", "title": "Agenda for change: the future of EU development policy", "text": "", - "created": "2015-12-27T11:51:14.770Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + "created": "2015-12-27T11:51:14.770Z" }, "model": "representatives_votes.dossier", "pk": 1 @@ -17,12 +16,37 @@ "reference": "2015/2132(BUD)", "title": "2016 general budget: all sections", "text": "", - "created": "2015-12-27T11:51:14.781Z", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + "created": "2015-12-27T11:51:14.781Z" }, "model": "representatives_votes.dossier", "pk": 2 }, +{ + "fields": { + "updated": "2016-07-08T05:20:11.664Z", + "title": "", + "dossier": 1, + "created": "2016-07-08T05:20:11.664Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + }, + "model": "representatives_votes.document", + "pk": 1 +}, +{ + "fields": { + "updated": "2016-07-08T05:20:11.664Z", + "title": "", + "dossier": 2, + "created": "2016-07-08T05:20:11.664Z", + "kind": "procedure-file", + "chamber": 1, + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)" + }, + "model": "representatives_votes.document", + "pk": 2 +}, { "fields": { "updated": "2015-12-27T11:51:24.327Z", diff --git a/representatives_votes/migrations/0012_document.py b/representatives_votes/migrations/0012_document.py new file mode 100644 index 0000000000000000000000000000000000000000..a88cf97fb9efbafb7e60e70e537181fa12e2dada --- /dev/null +++ b/representatives_votes/migrations/0012_document.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import logging + +from django.db import migrations, models + + +def migrate_dossier_links(apps, schema_editor): + logger = logging.getLogger('migrate_dossier_links') + + # Get model managers + Chamber = apps.get_model("representatives", "Chamber") + Dossier = apps.get_model("representatives_votes", "Dossier") + Document = apps.get_model("representatives_votes", "Document") + + docs = [] + + # EP dossiers + ep_chamber = Chamber.objects.get(abbreviation='EP') + ep_link = 'europarl.europa.eu' + for dossier in Dossier.objects.filter(link__icontains=ep_link): + logger.info('Create document %s for dossier %s' % (dossier.link, + dossier.reference)) + docs.append(Document(chamber=ep_chamber, dossier=dossier, + link=dossier.link, kind='procedure-file')) + + # France dossiers + try: + an_chamber = Chamber.objects.get(abbreviation='AN') + sen_chamber = Chamber.objects.get(abbreviation='SEN') + except Chamber.DoesNotExist: + return + + an_link = 'assemblee-nationale.fr' + sen_link = 'senat.fr' + + for dossier in Dossier.objects.filter(link__icontains=an_link): + logger.info('Create document %s for dossier %s' % (dossier.link, + dossier.reference)) + docs.append(Document(chamber=an_chamber, dossier=dossier, + link=dossier.link, kind='procedure-file')) + + for dossier in Dossier.objects.filter(ext_link__icontains=an_link): + logger.info('Create document %s for dossier %s' % (dossier.link, + dossier.reference)) + docs.append(Document(chamber=an_chamber, dossier=dossier, + link=dossier.ext_link, kind='procedure-file')) + + for dossier in Dossier.objects.filter(link__icontains=sen_link): + logger.info('Create document %s for dossier %s' % (dossier.link, + dossier.reference)) + docs.append(Document(chamber=sen_chamber, dossier=dossier, + link=dossier.link, kind='procedure-file')) + + for dossier in Dossier.objects.filter(ext_link__icontains=an_link): + logger.info('Create document %s for dossier %s' % (dossier.link, + dossier.reference)) + docs.append(Document(chamber=sen_chamber, dossier=dossier, + link=dossier.ext_link, kind='procedure-file')) + + # Create all dossiers + logger.info('Saving %s documents...' % len(docs)) + Document.objects.bulk_create(docs) + + +class Migration(migrations.Migration): + + dependencies = [ + ('representatives', '0019_remove_fingerprints'), + ('representatives_votes', '0011_remove_fingerprints'), + ] + + operations = [ + migrations.CreateModel( + name='Document', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ('title', models.CharField(max_length=1000)), + ('kind', models.CharField(default=b'', max_length=255, blank=True)), + ('link', models.URLField(max_length=1000)), + ('chamber', models.ForeignKey(to='representatives.Chamber')), + ('dossier', models.ForeignKey(related_name='documents', to='representatives_votes.Dossier')), + ], + options={ + 'abstract': False, + }, + ), + + migrations.RunPython(migrate_dossier_links), + + migrations.RemoveField( + model_name='dossier', + name='link', + ), + + migrations.RemoveField( + model_name='dossier', + name='ext_link', + ), + ] diff --git a/representatives_votes/models.py b/representatives_votes/models.py index 14f9f19da626444e219040b10d08689b0bf011b0..35e332509c46c35e50414c36c6b7ae849d8bc027 100644 --- a/representatives_votes/models.py +++ b/representatives_votes/models.py @@ -1,15 +1,13 @@ # coding: utf-8 from django.db import models -from representatives.models import Representative, TimeStampedModel +from representatives.models import Chamber, Representative, TimeStampedModel class Dossier(TimeStampedModel): title = models.CharField(max_length=1000) reference = models.CharField(max_length=200, unique=True) text = models.TextField(blank=True, default='') - link = models.URLField() - ext_link = models.URLField(blank=True, default='') class Meta: unique_together = (('title', 'reference')) @@ -18,6 +16,14 @@ class Dossier(TimeStampedModel): return unicode(self.title) +class Document(TimeStampedModel): + dossier = models.ForeignKey(Dossier, related_name='documents') + chamber = models.ForeignKey(Chamber) + title = models.CharField(max_length=1000) + kind = models.CharField(max_length=255, blank=True, default='') + link = models.URLField(max_length=1000) + + class Proposal(TimeStampedModel): dossier = models.ForeignKey(Dossier, related_name='proposals') title = models.CharField(max_length=1000, unique=True) diff --git a/representatives_votes/serializers.py b/representatives_votes/serializers.py index 66814ab19bde28358944689d277e0ed8c3c8e82f..48c86425a7be53fad19276f907fe809fead7dc84 100644 --- a/representatives_votes/serializers.py +++ b/representatives_votes/serializers.py @@ -46,6 +46,19 @@ class ProposalDetailSerializer(ProposalSerializer): fields = ProposalSerializer.Meta.fields + ('votes',) +class DocumentSerializer(serializers.HyperlinkedModelSerializer): + """ Base document serializer """ + + class Meta: + model = models.Document + fields = ( + 'dossier', + 'chamber', + 'kind', + 'link' + ) + + class DossierSerializer(serializers.HyperlinkedModelSerializer): """ Base dossier serializer """ @@ -55,7 +68,6 @@ class DossierSerializer(serializers.HyperlinkedModelSerializer): 'title', 'reference', 'text', - 'link', 'url', ) @@ -66,7 +78,8 @@ class DossierDetailSerializer(DossierSerializer): """ proposals = ProposalSerializer(many=True) + documents = DocumentSerializer(many=True) class Meta: model = models.Dossier - field = DossierSerializer.Meta.fields + ('proposals',) + field = DossierSerializer.Meta.fields + ('proposals', 'documents') diff --git a/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossier.content b/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossier.content index e7ea07f96f4150ae5240782c1b386cccbe2ee94d..cd4f3e31226b9b1ed631ec97ed03787bcb939439 100644 --- a/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossier.content +++ b/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossier.content @@ -26,11 +26,17 @@ "url": "http://testserver/api/proposals/2/" } ], + "documents": [ + { + "dossier": "http://testserver/api/dossiers/1/", + "chamber": "http://testserver/api/chambers/1/", + "kind": "procedure-file", + "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)" + } + ], "created": "2015-12-27T11:51:14.770000Z", "updated": "2015-12-27T11:51:14.770000Z", "title": "Agenda for change: the future of EU development policy", "reference": "2012/2002(INI)", - "text": "", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)", - "ext_link": "" + "text": "" } \ No newline at end of file diff --git a/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossiers.content b/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossiers.content index 82d079caa96b704c721918c16455518adc918319..4c5985a1ea64451953337905f6ba3e5a576ca1de 100644 --- a/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossiers.content +++ b/representatives_votes/tests/response_fixtures/RepresentativeManagerTest.test_dossiers.content @@ -3,14 +3,12 @@ "title": "Agenda for change: the future of EU development policy", "reference": "2012/2002(INI)", "text": "", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2012/2002(INI)", "url": "http://testserver/api/dossiers/1/" }, { "title": "2016 general budget: all sections", "reference": "2015/2132(BUD)", "text": "", - "link": "http://www.europarl.europa.eu/oeil/popups/ficheprocedure.do?reference=2015/2132(BUD)", "url": "http://testserver/api/dossiers/2/" } ] \ No newline at end of file diff --git a/representatives_votes/tests/test_views.py b/representatives_votes/tests/test_views.py index 5b631deb1802017847d463e42fdb0068aecd995a..3bd4e0da1b30fffe4928e5cb778954a876682110 100644 --- a/representatives_votes/tests/test_views.py +++ b/representatives_votes/tests/test_views.py @@ -15,8 +15,10 @@ class RepresentativeManagerTest(test.TestCase): Response.for_test(self).assertNoDiff(result) def test_dossier(self): - # One for dossier + 1 for proposals - self.functional_test(2, '/api/dossiers/1/') + # One for dossier + # One for proposals + # One for documents + self.functional_test(3, '/api/dossiers/1/') def test_dossiers(self): self.functional_test(1, '/api/dossiers/') diff --git a/representatives_votes/tests/urls.py b/representatives_votes/tests/urls.py index 9f9ae4cb7bea9cf43c6b42f9fc3ef944540f7a05..5e058de686c1356d7f9ef21a4cbc1b4fb2db48a6 100644 --- a/representatives_votes/tests/urls.py +++ b/representatives_votes/tests/urls.py @@ -9,6 +9,7 @@ from representatives_votes.api import ( ) from representatives.api import ( + ChamberViewSet, ConstituencyViewSet, GroupViewSet, MandateViewSet, @@ -16,6 +17,7 @@ from representatives.api import ( ) router = routers.DefaultRouter() +router.register(r'chambers', ChamberViewSet) router.register(r'constituencies', ConstituencyViewSet) router.register(r'groups', GroupViewSet) router.register(r'mandates', MandateViewSet) diff --git a/setup.py b/setup.py index 0dcbe5ed085c320063bff3b6d079f8770ff91433..311ffde7a38738c7bae181069c68957951f5dc0b 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( keywords='django government parliament votes', install_requires=[ 'django>1.8,<1.9', - 'django-representatives>=0.0.27', + 'django-representatives>=0.0.29', 'py-dateutil>=2,<3', 'ijson>=2,<3', 'pytz', # Always use up-to-date TZ data