Commit be45d3d8 authored by Nicolas Joyard's avatar Nicolas Joyard

Rework scores calculations using pseudo-materialized views

parent 24a91eae
......@@ -18,8 +18,11 @@ from representatives_votes.api import (
)
from representatives_recommendations.api import (
RecommendationViewSet
)
from memopol_scores.api import (
DossierScoreViewSet,
RecommendationViewSet,
RepresentativeScoreViewSet,
VoteScoreViewSet
)
......
......@@ -61,7 +61,7 @@ class RepresentativeFilter(FilterSet):
return qs
try:
return qs.filter(score__score__gte=int(value))
return qs.filter(representative_score__score__gte=int(value))
except ValueError:
return qs
......@@ -70,7 +70,7 @@ class RepresentativeFilter(FilterSet):
return qs
try:
return qs.filter(score__score__lte=int(value))
return qs.filter(representative_score__score__lte=int(value))
except ValueError:
return qs
......
......@@ -91,6 +91,7 @@ INSTALLED_APPS = (
# ---
'core',
'memopol',
'memopol_scores',
'memopol_settings',
'memopol_themes',
'representatives',
......
......@@ -97,6 +97,8 @@ def proposal_status_label(status, recommendation=None):
@register.filter
def score_badge(score, tooltip=None):
score = int(score or 0)
if score > 0:
color = 'success'
elif score < 0:
......
from django import test
from responsediff.test import ResponseDiffTestMixin
from memopol_scores.models import RepresentativeScore
class BaseTest(ResponseDiffTestMixin, test.TestCase):
......@@ -19,6 +20,9 @@ class BaseTest(ResponseDiffTestMixin, test.TestCase):
"""
left_pane_queries = 7
def setUp(self):
RepresentativeScore.refresh()
def request_test(self, url=None):
self.assertResponseDiffEmpty(self.client.get(url or self.url))
......
......@@ -13,7 +13,7 @@ from representatives_positions.views import PositionFormMixin
class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin,
generic.DetailView):
queryset = Representative.objects.select_related('score')
queryset = Representative.objects.select_related('representative_score')
def get_queryset(self):
qs = super(RepresentativeDetailBase, self).get_queryset()
......
......@@ -20,7 +20,7 @@ class RepresentativeDetailPositions(RepresentativeDetailBase):
.order_by('-datetime', 'pk')
),
'positions__themes',
'positions__positionscore'
'positions__position_score'
)
return qs
......
......@@ -2,8 +2,7 @@
from django.db import models
from representatives_recommendations.models import VoteScore
from representatives_votes.models import Proposal
from representatives_votes.models import Vote
from .representative_detail_base import RepresentativeDetailBase
......@@ -17,14 +16,15 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
qs = qs.prefetch_related(
models.Prefetch(
'votes',
queryset=VoteScore.objects.filter(
proposal__in=Proposal.objects.exclude(recommendation=None),
queryset=Vote.objects.exclude(
proposal__recommendation=None
).select_related(
'vote_score',
'proposal__dossier',
'proposal__recommendation'
).order_by('-proposal__datetime', 'proposal__title')
),
'dossierscores'
'dossier_scores'
)
return qs
......@@ -32,7 +32,7 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
def get_context_data(self, **kwargs):
c = super(RepresentativeDetailVotes, self).get_context_data(**kwargs)
ds = c['object'].dossierscores.all()
ds = c['object'].dossier_scores.all()
dossiers = {}
for vote in c['object'].votes.all():
......
......@@ -18,7 +18,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
SortMixin, PositionFormMixin, generic.ListView):
csv_name = 'representatives'
queryset = Representative.objects.select_related('score')
queryset = Representative.objects.select_related('representative_score')
current_filter = None
sort_modes = {
'name-asc': {
......@@ -34,12 +34,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
'score-asc': {
'order': 2,
'label': 'Best score',
'fields': ['-score__score']
'fields': ['-representative_score__score']
},
'score-desc': {
'order': 2,
'label': 'Worst score',
'fields': ['score__score']
'fields': ['representative_score__score']
}
}
sort_default = 'name-asc'
......
# coding: utf-8
from django.views import generic
from representatives.models import Representative
from .representative_mixin import RepresentativeViewMixin
class RepresentativeMandates(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_mandates'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativeMandates, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativeMandates, self).get_context_data(**kwargs)
self.add_representative_country_and_main_mandate(c['object'])
c['mandates'] = c['object'].mandates.all()
return c
# coding: utf-8
from django.db import models
from django.views import generic
from representatives.models import Representative
from representatives_positions.forms import PositionForm
from representatives_positions.models import Position
from .representative_mixin import RepresentativeViewMixin
class RepresentativePositions(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_positions'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativePositions, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
qs = qs.prefetch_related(
models.Prefetch(
'positions',
queryset=Position.objects.filter(published=True)
.order_by('-datetime', 'pk')
)
)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativePositions, self).get_context_data(**kwargs)
c['position_form'] = PositionForm(
initial={'representative': self.object.pk})
self.add_representative_country_and_main_mandate(c['object'])
return c
# coding: utf-8
from django.db import models
from django.views import generic
from representatives.models import Representative
from representatives_recommendations.models import VoteScore
from representatives_votes.models import Proposal
from .representative_mixin import RepresentativeViewMixin
class RepresentativeVotes(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_votes'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativeVotes, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
qs = qs.prefetch_related(
models.Prefetch(
'votes',
queryset=VoteScore.objects.filter(
proposal__in=Proposal.objects.exclude(recommendation=None),
).select_related('proposal__recommendation').order_by(
'-proposal__datetime')
)
)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativeVotes, self).get_context_data(**kwargs)
self.add_representative_country_and_main_mandate(c['object'])
c['votes'] = c['object'].votes.all()
return c
......@@ -9,7 +9,7 @@ class ThemeDetailPositions(ThemeDetailBase):
def get_queryset(self):
qs = super(ThemeDetailPositions, self).get_queryset()
qs = qs.prefetch_related('positions__representative',
'positions__positionscore')
'positions__position_score')
return qs
def get_context_data(self, **kwargs):
......
from rest_framework import (
filters,
viewsets,
)
from rql_filter.backend import RQLFilterBackend
from representatives.api import DefaultWebPagination
from .models import (
DossierScore,
RepresentativeScore,
VoteScore
)
from .serializers import (
DossierScoreSerializer,
RepresentativeScoreSerializer,
VoteScoreSerializer
)
class DossierScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view representative score contribution for each dossier
"""
queryset = DossierScore.objects.all()
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'dossier': ['exact'],
'representative': ['exact'],
'score': ['exact', 'gte', 'lte']
}
search_fields = ('dossier', 'representative')
ordering_fields = ('representative', 'dossier')
pagination_class = DefaultWebPagination
serializer_class = DossierScoreSerializer
class RepresentativeScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view representative scores
"""
queryset = RepresentativeScore.objects.select_related('representative')
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'representative': ['exact'],
'score': ['exact', 'gte', 'lte']
}
search_fields = ('representative', 'score')
ordering_fields = ('representative', 'score')
pagination_class = DefaultWebPagination
serializer_class = RepresentativeScoreSerializer
class VoteScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view votes with their score impact.
This endpoint only shows votes that have a matching recommendation.
"""
queryset = VoteScore.objects.select_related(
'vote__representative',
'vote__proposal',
'vote__proposal__dossier',
'vote__proposal__recommendation'
).filter(
vote__proposal__recommendation__isnull=False
)
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'vote__representative': ['exact'],
'vote__proposal': ['exact'],
'vote__proposal__dossier': ['exact']
}
pagination_class = DefaultWebPagination
serializer_class = VoteScoreSerializer
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('representatives', '0020_rep_unique_slug_remove_remoteid'),
('representatives_votes', '0012_document'),
('representatives_positions', '0007_remove_positionscore'),
('representatives_recommendations', '0010_remove_views'),
('memopol_themes', '0003_remove_themescore'),
('memopol_settings', '0002_score_settings'),
]
operations = [
migrations.CreateModel(
name='DossierScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('dossier', models.ForeignKey(to='representatives_votes.Dossier')),
('representative', models.ForeignKey(related_name='dossier_scores', to='representatives.Representative')),
],
),
migrations.CreateModel(
name='PositionScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('position', models.OneToOneField(related_name='position_score', to='representatives_positions.Position')),
],
),
migrations.CreateModel(
name='RepresentativeScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('representative', models.OneToOneField(related_name='representative_score', to='representatives.Representative')),
],
),
migrations.CreateModel(
name='VoteScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('vote', models.OneToOneField(related_name='vote_score', to='representatives_votes.Vote')),
],
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('memopol_scores', '0001_initial'),
]
operations = [
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_vote_score"
AS SELECT
"representatives_votes_vote"."id" AS "vote_id",
decay_score(
CAST(CASE
WHEN "representatives_votes_vote"."position"::text = "representatives_recommendations_recommendation"."recommendation"::text
THEN "representatives_recommendations_recommendation"."weight"
ELSE 0 - "representatives_recommendations_recommendation"."weight"
END AS NUMERIC),
"representatives_votes_proposal"."datetime",
"decay_num"."value",
"decay_denom"."value",
"exponent"."value",
"decimals"."value"
) AS "score"
FROM "representatives_votes_vote"
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1
JOIN "representatives_votes_proposal" ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
LEFT JOIN "representatives_recommendations_recommendation" ON "representatives_votes_proposal"."id" = "representatives_recommendations_recommendation"."proposal_id"
WHERE "representatives_recommendations_recommendation"."id" IS NOT NULL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_dossier_score"
AS SELECT
"representatives_votes_vote"."representative_id" AS "representative_id",
"representatives_votes_proposal"."dossier_id" AS "dossier_id",
SUM("memopol_scores_votescore"."score") AS "score"
FROM
"memopol_scores_votescore"
INNER JOIN "representatives_votes_vote"
ON "memopol_scores_votescore"."vote_id" = "representatives_votes_vote"."id"
INNER JOIN "representatives_votes_proposal"
ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
GROUP BY
"representatives_votes_vote"."representative_id",
"representatives_votes_proposal"."dossier_id"
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_position_score"
AS SELECT
"representatives_positions_position"."id" AS "position_id",
decay_score(
"representatives_positions_position"."score",
"representatives_positions_position"."datetime",
"decay_num"."value",
"decay_denom"."value",
"exponent"."value",
"decimals"."value"
) AS "score"
FROM
"representatives_positions_position"
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_representative_score"
AS SELECT
"source"."representative_id" AS "representative_id" ,
SUM("source"."score") AS "score"
FROM
(
SELECT
"memopol_scores_dossierscore"."representative_id" AS "representative_id",
"memopol_scores_dossierscore"."score" AS "score"
FROM "memopol_scores_dossierscore"
UNION ALL
SELECT
"representatives_positions_position"."representative_id" AS "representative_id",
"memopol_scores_positionscore"."score" AS "score"
FROM
"memopol_scores_positionscore"
INNER JOIN "representatives_positions_position"
ON "memopol_scores_positionscore"."position_id" = "representatives_positions_position"."id"
) "source"
GROUP BY
"source"."representative_id"
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION refresh_vote_scores()
RETURNS VOID AS $$
BEGIN
TRUNCATE TABLE "memopol_scores_votescore";
INSERT INTO "memopol_scores_votescore" ("vote_id", "score")
SELECT "vote_id", "score" FROM "memopol_scores_v_vote_score";
END;
$$ LANGUAGE PLPGSQL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION refresh_dossier_scores()
RETURNS VOID AS $$
BEGIN
TRUNCATE TABLE "memopol_scores_dossierscore";
PERFORM refresh_vote_scores();
INSERT INTO "memopol_scores_dossierscore" ("representative_id", "dossier_id", "score")
SELECT "representative_id", "dossier_id", "score" FROM "memopol_scores_v_dossier_score";
END;
$$ LANGUAGE PLPGSQL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION refresh_position_scores()
RETURNS VOID AS $$
BEGIN
TRUNCATE TABLE "memopol_scores_positionscore";
INSERT INTO "memopol_scores_positionscore" ("position_id", "score")
SELECT "position_id", "score" FROM "memopol_scores_v_position_score";
END;
$$ LANGUAGE PLPGSQL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION refresh_representative_scores()
RETURNS VOID AS $$
BEGIN
TRUNCATE TABLE "memopol_scores_representativescore";
PERFORM refresh_dossier_scores();
PERFORM refresh_position_scores();
INSERT INTO "memopol_scores_representativescore" ("representative_id", "score")
SELECT
"representatives_representative"."id",
COALESCE("memopol_scores_v_representative_score"."score", 0)
FROM
"representatives_representative"
LEFT OUTER JOIN "memopol_scores_v_representative_score"
ON "memopol_scores_v_representative_score"."representative_id" = "representatives_representative"."id";
END;
$$ LANGUAGE PLPGSQL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE FUNCTION refresh_scores()
RETURNS VOID AS $$
BEGIN
PERFORM refresh_representative_scores();
END;
$$ LANGUAGE PLPGSQL;
"""
),
]
from django.db import connection, models
from representatives.models import Representative
from representatives_votes.models import Dossier, Vote
from representatives_positions.models import Position
class VoteScore(models.Model):
vote = models.OneToOneField(Vote, related_name='vote_score')
score = models.FloatField()
@classmethod
def refresh(cls):
with connection.cursor() as cursor:
cursor.execute('SELECT refresh_vote_scores();')
class DossierScore(models.Model):
representative = models.ForeignKey(Representative,
related_name='dossier_scores')
dossier = models.ForeignKey(Dossier)
score = models.FloatField()