Commit e1e714be authored by Jamesie Pic's avatar Jamesie Pic

Fresh start on the data model

This is completely backward-incompatible and was rushed before the
release - nobody was going to pay the interrest on the technical debt
this commit remove to support migrations since nobody has an instance in
production.

Memopol{Representative,Vote,Dossier} subclasses are gone. Django model
inheritance is fun but comes at a price - hacks like
create_child_instance_from_parent would end up everywhere. Use a simple
one to one relation when you don't intend to have several subclasses of
a model which is the case here.

Country and current mandates are not pre-calculated and stored in the
database anymore, instead they are pre-fetched which requires
django-representatives>=0.0.7

Score related code was rewritten in votes.models, along with the
Recommendation model. We're still not calculating it on the fly here
because that would require some SQL backflips which will take a bit of
time to do properly.
parent 9ac4be5d
......@@ -4,18 +4,21 @@ env:
language: python
python:
- '2.7'
before_install:
- pip install codecov
install:
- pip install -e .
- pip install flake8 pep8
- pip install -e .[testing]
before_script:
- bin/install_client_deps.sh
script:
- ! lesscpy -N static/less/base.less 2>&1 | grep Error
- pep8 . --exclude '*/migrations,docs,static' --ignore E128
- bash -c '! lesscpy -N static/less/base.less 2>&1 | grep Error'
- flake8 . --exclude '*/migrations,docs,static' --ignore E128
- django-admin test positions votes core legislature
- py.test memopol representatives_positions representatives_recommendations
- rm -rf db.sqlite
- django-admin migrate
- django-admin update_score
after_success:
- codecov
deploy:
- provider: openshift
user: jamespic@gmail.com
......
[![Build Status](https://travis-ci.org/political-memory/political_memory.svg?branch=master)](https://travis-ci.org/political-memory/political_memory)
[![codecov.io](https://codecov.io/github/political-memory/political_memory/coverage.svg?branch=master)](https://codecov.io/github/political-memory/political_memory?branch=master)
git clone git@github.com:political-memory/political_memory.git
cd political_memory
cp memopol/config.json.sample memopol/config.json
# Create a throwable virtualenv
virtualenv ve
source ve/bin/activate
......
......@@ -2,19 +2,21 @@
from __future__ import unicode_literals
from django.db import migrations, models
from django.conf import settings
def set_site_name(apps, schema_editor):
Site = apps.get_model('sites', 'Site')
Site.objects.filter(pk=settings.SITE_ID).update(
name=settings.SITE_NAME, domain=settings.SITE_DOMAIN)
class Migration(migrations.Migration):
dependencies = [
('legislature', '0002_memopolrepresentative_main_mandate'),
('sites', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='memopolrepresentative',
name='main_mandate',
field=models.ForeignKey(
default=None, to='representatives.Mandate', null=True),
),
migrations.RunPython(set_site_name),
]
/ -load memopol_tags cache
- load i18n
- load cache
- load staticfiles
#header.container-fluid
%a{href: "/", id: 'logo'}
%img{src: '{% static "images/logo.png" %}'}
%h1
%a#header_banner{href: "/"}
-trans "Political Memory"
%p.organization
=config.ORGANIZATION_NAME
#nav.container-fluid
-include "core/blocks/navigation.html"
%ul.nav
%li
%a{href: "{% url 'legislature:representative-index' %}"}
Representatives
%li
%a{href: "{% url 'legislature:group-index' kind='country' %}"}
Countries
%li
%a{href: "{% url 'legislature:group-index' kind='group' %}"}
Parties
%li
%a{href: "{% url 'legislature:group-index' kind='delegation' %}"}
Delegations
%li
%a{href: "{% url 'legislature:group-index' kind='committee' %}"}
Committees
%ul.nav
%li
%a{href: "{% url 'votes:dossier-index' %}"}
Votes
......@@ -2,15 +2,20 @@
%nav
%ul.pagination.pagination-sm
- if page.has_previous
- if page_obj.has_previous
%li
%a{'href': '?={queries.urlencode}&page=={page.previous_page_number}',
%a{'href': '?={queries.urlencode}&page=1',
'aria-label': 'First'}
<i aria-hidden="true" class="fa fa-chevron-circle-left"></i>
%li
%a{'href': '?={queries.urlencode}&page=={page_obj.previous_page_number}',
'aria-label': 'Previous'}
<i aria-hidden="true" class="fa fa-chevron-left"></i>
- for p in page.pages
- for p in page_range
- if p
- if p == page.number
- if p == page_obj.number
%li.active
%a{'href': ''}
{{ p }}
......@@ -18,17 +23,18 @@
%li
%a{'href': '?={queries.urlencode}&page=={p}'}
{{ p }}
- else
%li.disabled
%a{'href': ''}
- if page.has_next
- if page_obj.has_next
%li
%a{'href': '?={queries.urlencode}&page=={page.next_page_number}',
%a{'href': '?={queries.urlencode}&page=={page_obj.next_page_number}',
'aria-label': 'Next'}
<i aria-hidden="true" class="fa fa-chevron-right"></i>
%li
%a{'href': '?={queries.urlencode}&page=={paginator.num_pages}',
'aria-label': 'Last'}
<i aria-hidden="true" class="fa fa-chevron-circle-right"></i>
%div.count
Number of results : {{ paginator.count }}
%br
......@@ -36,9 +42,10 @@
{{ paginator.per_page }}
(
- for limit in pagination_limits
%a{'href': '?limit={{ limit }}'}
%a{'href': '?paginate_by={{ limit }}'}
{{ limit }}
- if not forloop.last
\/
)
- include 'core/blocks/grid-list.html'
- if grid_list
- include 'core/blocks/grid-list.html'
- extends "base.html"
- block content
.row
.col-md-8
%p
Memopol is reachable only in <b>reduced functionality mode</b>.
By the way, you could access to
<a href="{% url 'legislature:representative-index' %}">the list of MEPs</a>.
%p
You can help on building the new Memopol by <a href="https://wiki.laquadrature.net/Projects/Memopol/Roadmap">coding, translating, de signing, funding, etc...</a>.
.col-md-4
.panel.panel-default
.panel-body
%p
<a href="http://memopol.org">Memopol Blog</a> is available as well as the new
<a href="http://git.laquadrature.net/memopol/memopol/issues">
bugtracking system</a>
%h3
What is memopol?
%p
Political Memory is a tool designed by La Quadrature du Net to help
European citizens to reach members of European Parliament (MEPs) and
track their voting records on issues related to fundamental
freedoms online. <em><a href="">More...</a></em>
from __future__ import absolute_import
from pure_pagination import EmptyPage
from pure_pagination import Paginator
from django.shortcuts import render
def create_child_instance_from_parent(child_cls, parent_instance):
"""
Create a child model instance from a parent instance
"""
parent_cls = parent_instance.__class__
field = child_cls._meta.get_ancestor_link(parent_cls).column
child_instance = child_cls(**{
field: parent_instance.pk
})
child_instance.__dict__.update(parent_instance.__dict__)
child_instance.save()
return child_instance
def render_paginate_list(request, object_list, template_name):
"""
Render a paginated list of representatives
"""
pagination_limits = (12, 24, 48, 96)
num_by_page = request.GET.get('limit', unicode(pagination_limits[0]))
num_by_page = int(num_by_page) if num_by_page.isdigit() else 1
paginator = Paginator(object_list, num_by_page)
number = request.GET.get('page', '1')
number = int(number) if number.isdigit() else 1
try:
page = paginator.page(number)
except EmptyPage:
page = paginator.page(paginator.num_pages)
context = {}
context['paginator'] = paginator
context['page'] = page
context['object_list'] = context['page'].object_list
context['pagination_limits'] = pagination_limits
return render(
request,
template_name,
context
)
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from django.views.generic.base import TemplateView
class HomeView(TemplateView):
template_name = "core/home.html"
class PaginationMixin(object):
pagination_limits = (12, 24, 48, 96)
def get(self, *args, **kwargs):
self.set_paginate_by()
return super(PaginationMixin, self).get(*args, **kwargs)
def set_paginate_by(self):
if 'paginate_by' in self.request.GET:
self.request.session['paginate_by'] = \
self.request.GET['paginate_by']
elif 'paginate_by' not in self.request.session:
self.request.session['paginate_by'] = 12
def get_paginate_by(self, queryset):
return self.request.session['paginate_by']
def get_page_range(self, page):
pages = []
if page.paginator.num_pages != 1:
for i in page.paginator.page_range:
if page.number - 4 < i < page.number + 4:
pages.append(i)
return pages
def get_context_data(self, **kwargs):
c = super(PaginationMixin, self).get_context_data(**kwargs)
c['pagination_limits'] = self.pagination_limits
c['paginate_by'] = self.request.session['paginate_by']
c['page_range'] = self.get_page_range(c['page_obj'])
return c
class GridListMixin(object):
def set_session_display(self):
if self.request.GET.get('display') in ('grid', 'list'):
self.request.session['display'] = self.request.GET.get('display')
if 'display' not in self.request.session:
self.request.session['display'] = 'grid'
def get(self, *args, **kwargs):
self.set_session_display()
return super(GridListMixin, self).get(*args, **kwargs)
def get_template_names(self):
return [t.replace('_list', '_%s' % self.request.session['display'])
for t in super(GridListMixin, self).get_template_names()]
def get_context_data(self, **kwargs):
c = super(GridListMixin, self).get_context_data(**kwargs)
c['grid_list'] = True
return c
No preview for this file type
......@@ -13,3 +13,24 @@ Adding random recommendations
In [4]: for p in Proposal.objects.all(): Recommendation.objects.create(proposal=p, recommendation='for', weight=random.randint(1,10))
Creating test fixtures
======================
The largest test fixtures are, the longer it takes to load them, the longer the
test run is.
To create test fixtures for representatives_positions, insert some Position
objects, and reduce the database with::
./manage.py remove_representatives_without_position
./manage.py remove_groups_without_mandate
./manage.py remove_countries_without_group
For representatives_recommendations::
./manage.py remove_proposals_without_recommendation
./manage.py remove_dossiers_without_proposal
./manage.py remove_representatives_without_vote
./manage.py remove_groups_without_mandate
./manage.py remove_countries_without_group
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from __future__ import absolute_import
from django.contrib import admin
from representatives.models import Address, Country, Email, Phone, WebSite
from .models import MemopolRepresentative
class EmailInline(admin.TabularInline):
model = Email
extra = 0
class WebsiteInline(admin.TabularInline):
model = WebSite
extra = 0
class AdressInline(admin.StackedInline):
model = Address
extra = 0
class PhoneInline(admin.TabularInline):
model = Phone
extra = 0
class CountryInline(admin.TabularInline):
model = Country
extra = 0
class MemopolRepresentativeAdmin(admin.ModelAdmin):
list_display = ('full_name', 'country', 'score', 'main_mandate')
search_fields = ('first_name', 'last_name', 'birth_place')
list_filter = ('gender', 'active')
inlines = [
PhoneInline,
EmailInline,
WebsiteInline,
AdressInline,
]
admin.site.register(MemopolRepresentative, MemopolRepresentativeAdmin)
# coding: utf-8
# This file is part of mempol.
#
# mempol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# mempol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from __future__ import absolute_import
import django_filters
from .models import MemopolRepresentative
class RepresentativeFilter(django_filters.FilterSet):
class Meta:
model = MemopolRepresentative
# fields = ['full_name', 'country', 'score']
fields = {
'full_name': ['icontains', 'exact'],
'slug': ['exact'],
'remote_id': ['exact'],
'gender': ['exact'],
'active': ['exact'],
'country__name': ['exact'],
'country__code': ['exact']
}
order_by = ['score', 'full_name']
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('representatives', '0003_auto_20150702_1827'),
]
operations = [
migrations.CreateModel(
name='MemopolRepresentative',
fields=[
('representative_ptr', models.OneToOneField(parent_link=True, auto_created=True,
primary_key=True, serialize=False, to='representatives.Representative')),
('score', models.IntegerField(default=0)),
('country', models.ForeignKey(
to='representatives.Country', null=True)),
],
options={
'abstract': False,
},
bases=('representatives.representative',),
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('representatives', '0004_auto_20150709_1601'),
('legislature', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='memopolrepresentative',
name='main_mandate',
field=models.ForeignKey(
default=True, to='representatives.Mandate', null=True),
),
]
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from datetime import datetime
from django.db import models
from representatives.contrib.parltrack.import_representatives import \
representative_post_save
from representatives.models import Country, Mandate, Representative
from votes.models import MemopolVote
# from django.utils.functional import cached_property
class MemopolRepresentative(Representative):
country = models.ForeignKey(Country, null=True)
score = models.IntegerField(default=0)
main_mandate = models.ForeignKey(Mandate, null=True, default=None)
def update_score(self):
score = 0
for vote in MemopolVote.objects.filter(representative=self):
score += vote.absolute_score
self.score = score
self.save()
def update_all(self):
self.update_score()
def active_mandates(self):
return self.mandates.filter(
end_date__gte=datetime.now()
)
def former_mandates(self):
return self.mandates.filter(
end_date__lte=datetime.now()
)
def votes_with_proposal(self):
return MemopolVote.objects.select_related(
'proposal',
'proposal__recommendation'
).filter(representative=self)
def mempol_representative(sender, representative, data, **kwargs):
update = False
try:
memopol_representative = MemopolRepresentative.objects.get(
representative_ptr=representative)
except MemopolRepresentative.DoesNotExist:
memopol_representative = MemopolRepresentative(
representative_ptr=representative)
# Please forgive the horror your are about to witness, but this is
# really necessary. Django wants to update the parent model when we
# save a child model.
memopol_representative.__dict__.update(representative.__dict__)
try:
country = sorted(data.get('Constituencies', []),
key=lambda c: c.get('end') if c is not None else 1
)[-1]['country']
except IndexError:
pass
else:
if sender.cache.get('countries', None) is None:
sender.cache['countries'] = {c.name: c.pk for c in
Country.objects.all()}
country_id = sender.cache['countries'].get(country)
if memopol_representative.country_id != country_id:
memopol_representative.country_id = country_id
update = True
if sender.mep_cache['groups']:
main_mandate = sorted(sender.mep_cache['groups'],
key=lambda m: m.end_date)[-1]
if memopol_representative.main_mandate_id != main_mandate.pk:
memopol_representative.main_mandate_id = main_mandate.pk
update = True
if update:
memopol_representative.save()
representative_post_save.connect(mempol_representative)
# coding: utf-8
# This file is part of memopol.
#
# memopol is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of
# the License, or any later version.
#
# memopol is distributed in the hope that it will
# be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU General Affero Public
# License along with django-representatives.
# If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
from datetime import datetime
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.db.models import Q
from django.http import Http404
from django.shortcuts import get_object_or_404, render
from legislature.models import MemopolRepresentative
from representatives.models import Group
def retrieve(request, pk=None, name=None):
if pk:
representative = get_object_or_404(
MemopolRepresentative,
id=pk
)
elif name:
representative = get_object_or_404(
MemopolRepresentative,
full_name=name
)
else:
return Http404()
return render(
request,
'legislature/representative_view.html',
{'representative': representative}
)
def representatives_by_group(request, group_kind, group_abbr=None,
group_name=None, search=None, group_id=None):
if group_id:
representative_list = MemopolRepresentative.objects.filter(
mandates__group_id=group_id,
mandates__end_date__gte=datetime.now()
)
elif group_abbr:
representative_list = MemopolRepresentative.objects.filter(
mandates__group__abbreviation=group_abbr,
mandates__group__kind=group_kind,
mandates__end_date__gte=datetime.now()
)
elif group_name:
representative_list = MemopolRepresentative.objects.filter(
Q(mandates__group__name__icontains=group_name),