Commit be45d3d8 authored by Nicolas Joyard's avatar Nicolas Joyard
Browse files

Rework scores calculations using pseudo-materialized views

parent 24a91eae
...@@ -18,8 +18,11 @@ from representatives_votes.api import ( ...@@ -18,8 +18,11 @@ from representatives_votes.api import (
) )
from representatives_recommendations.api import ( from representatives_recommendations.api import (
RecommendationViewSet
)
from memopol_scores.api import (
DossierScoreViewSet, DossierScoreViewSet,
RecommendationViewSet,
RepresentativeScoreViewSet, RepresentativeScoreViewSet,
VoteScoreViewSet VoteScoreViewSet
) )
......
...@@ -61,7 +61,7 @@ class RepresentativeFilter(FilterSet): ...@@ -61,7 +61,7 @@ class RepresentativeFilter(FilterSet):
return qs return qs
try: try:
return qs.filter(score__score__gte=int(value)) return qs.filter(representative_score__score__gte=int(value))
except ValueError: except ValueError:
return qs return qs
...@@ -70,7 +70,7 @@ class RepresentativeFilter(FilterSet): ...@@ -70,7 +70,7 @@ class RepresentativeFilter(FilterSet):
return qs return qs
try: try:
return qs.filter(score__score__lte=int(value)) return qs.filter(representative_score__score__lte=int(value))
except ValueError: except ValueError:
return qs return qs
......
...@@ -91,6 +91,7 @@ INSTALLED_APPS = ( ...@@ -91,6 +91,7 @@ INSTALLED_APPS = (
# --- # ---
'core', 'core',
'memopol', 'memopol',
'memopol_scores',
'memopol_settings', 'memopol_settings',
'memopol_themes', 'memopol_themes',
'representatives', 'representatives',
......
...@@ -97,6 +97,8 @@ def proposal_status_label(status, recommendation=None): ...@@ -97,6 +97,8 @@ def proposal_status_label(status, recommendation=None):
@register.filter @register.filter
def score_badge(score, tooltip=None): def score_badge(score, tooltip=None):
score = int(score or 0)
if score > 0: if score > 0:
color = 'success' color = 'success'
elif score < 0: elif score < 0:
......
from django import test from django import test
from responsediff.test import ResponseDiffTestMixin from responsediff.test import ResponseDiffTestMixin
from memopol_scores.models import RepresentativeScore
class BaseTest(ResponseDiffTestMixin, test.TestCase): class BaseTest(ResponseDiffTestMixin, test.TestCase):
...@@ -19,6 +20,9 @@ class BaseTest(ResponseDiffTestMixin, test.TestCase): ...@@ -19,6 +20,9 @@ class BaseTest(ResponseDiffTestMixin, test.TestCase):
""" """
left_pane_queries = 7 left_pane_queries = 7
def setUp(self):
RepresentativeScore.refresh()
def request_test(self, url=None): def request_test(self, url=None):
self.assertResponseDiffEmpty(self.client.get(url or self.url)) self.assertResponseDiffEmpty(self.client.get(url or self.url))
......
...@@ -13,7 +13,7 @@ from representatives_positions.views import PositionFormMixin ...@@ -13,7 +13,7 @@ from representatives_positions.views import PositionFormMixin
class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin, class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin,
generic.DetailView): generic.DetailView):
queryset = Representative.objects.select_related('score') queryset = Representative.objects.select_related('representative_score')
def get_queryset(self): def get_queryset(self):
qs = super(RepresentativeDetailBase, self).get_queryset() qs = super(RepresentativeDetailBase, self).get_queryset()
......
...@@ -20,7 +20,7 @@ class RepresentativeDetailPositions(RepresentativeDetailBase): ...@@ -20,7 +20,7 @@ class RepresentativeDetailPositions(RepresentativeDetailBase):
.order_by('-datetime', 'pk') .order_by('-datetime', 'pk')
), ),
'positions__themes', 'positions__themes',
'positions__positionscore' 'positions__position_score'
) )
return qs return qs
......
...@@ -2,8 +2,7 @@ ...@@ -2,8 +2,7 @@
from django.db import models from django.db import models
from representatives_recommendations.models import VoteScore from representatives_votes.models import Vote
from representatives_votes.models import Proposal
from .representative_detail_base import RepresentativeDetailBase from .representative_detail_base import RepresentativeDetailBase
...@@ -17,14 +16,15 @@ class RepresentativeDetailVotes(RepresentativeDetailBase): ...@@ -17,14 +16,15 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
qs = qs.prefetch_related( qs = qs.prefetch_related(
models.Prefetch( models.Prefetch(
'votes', 'votes',
queryset=VoteScore.objects.filter( queryset=Vote.objects.exclude(
proposal__in=Proposal.objects.exclude(recommendation=None), proposal__recommendation=None
).select_related( ).select_related(
'vote_score',
'proposal__dossier', 'proposal__dossier',
'proposal__recommendation' 'proposal__recommendation'
).order_by('-proposal__datetime', 'proposal__title') ).order_by('-proposal__datetime', 'proposal__title')
), ),
'dossierscores' 'dossier_scores'
) )
return qs return qs
...@@ -32,7 +32,7 @@ class RepresentativeDetailVotes(RepresentativeDetailBase): ...@@ -32,7 +32,7 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
c = super(RepresentativeDetailVotes, self).get_context_data(**kwargs) c = super(RepresentativeDetailVotes, self).get_context_data(**kwargs)
ds = c['object'].dossierscores.all() ds = c['object'].dossier_scores.all()
dossiers = {} dossiers = {}
for vote in c['object'].votes.all(): for vote in c['object'].votes.all():
......
...@@ -18,7 +18,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin, ...@@ -18,7 +18,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
SortMixin, PositionFormMixin, generic.ListView): SortMixin, PositionFormMixin, generic.ListView):
csv_name = 'representatives' csv_name = 'representatives'
queryset = Representative.objects.select_related('score') queryset = Representative.objects.select_related('representative_score')
current_filter = None current_filter = None
sort_modes = { sort_modes = {
'name-asc': { 'name-asc': {
...@@ -34,12 +34,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin, ...@@ -34,12 +34,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
'score-asc': { 'score-asc': {
'order': 2, 'order': 2,
'label': 'Best score', 'label': 'Best score',
'fields': ['-score__score'] 'fields': ['-representative_score__score']
}, },
'score-desc': { 'score-desc': {
'order': 2, 'order': 2,
'label': 'Worst score', 'label': 'Worst score',
'fields': ['score__score'] 'fields': ['representative_score__score']
} }
} }
sort_default = 'name-asc' sort_default = 'name-asc'
......
# coding: utf-8
from django.views import generic
from representatives.models import Representative
from .representative_mixin import RepresentativeViewMixin
class RepresentativeMandates(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_mandates'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativeMandates, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativeMandates, self).get_context_data(**kwargs)
self.add_representative_country_and_main_mandate(c['object'])
c['mandates'] = c['object'].mandates.all()
return c
# coding: utf-8
from django.db import models
from django.views import generic
from representatives.models import Representative
from representatives_positions.forms import PositionForm
from representatives_positions.models import Position
from .representative_mixin import RepresentativeViewMixin
class RepresentativePositions(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_positions'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativePositions, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
qs = qs.prefetch_related(
models.Prefetch(
'positions',
queryset=Position.objects.filter(published=True)
.order_by('-datetime', 'pk')
)
)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativePositions, self).get_context_data(**kwargs)
c['position_form'] = PositionForm(
initial={'representative': self.object.pk})
self.add_representative_country_and_main_mandate(c['object'])
return c
# coding: utf-8
from django.db import models
from django.views import generic
from representatives.models import Representative
from representatives_recommendations.models import VoteScore
from representatives_votes.models import Proposal
from .representative_mixin import RepresentativeViewMixin
class RepresentativeVotes(RepresentativeViewMixin, generic.DetailView):
template_name = 'representatives/representative_votes'
queryset = Representative.objects.select_related('score')
def get_queryset(self):
qs = super(RepresentativeVotes, self).get_queryset()
qs = self.prefetch_for_representative_country_and_main_mandate(qs)
qs = qs.prefetch_related(
models.Prefetch(
'votes',
queryset=VoteScore.objects.filter(
proposal__in=Proposal.objects.exclude(recommendation=None),
).select_related('proposal__recommendation').order_by(
'-proposal__datetime')
)
)
return qs
def get_context_data(self, **kwargs):
c = super(RepresentativeVotes, self).get_context_data(**kwargs)
self.add_representative_country_and_main_mandate(c['object'])
c['votes'] = c['object'].votes.all()
return c
...@@ -9,7 +9,7 @@ class ThemeDetailPositions(ThemeDetailBase): ...@@ -9,7 +9,7 @@ class ThemeDetailPositions(ThemeDetailBase):
def get_queryset(self): def get_queryset(self):
qs = super(ThemeDetailPositions, self).get_queryset() qs = super(ThemeDetailPositions, self).get_queryset()
qs = qs.prefetch_related('positions__representative', qs = qs.prefetch_related('positions__representative',
'positions__positionscore') 'positions__position_score')
return qs return qs
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
......
from rest_framework import (
filters,
viewsets,
)
from rql_filter.backend import RQLFilterBackend
from representatives.api import DefaultWebPagination
from .models import (
DossierScore,
RepresentativeScore,
VoteScore
)
from .serializers import (
DossierScoreSerializer,
RepresentativeScoreSerializer,
VoteScoreSerializer
)
class DossierScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view representative score contribution for each dossier
"""
queryset = DossierScore.objects.all()
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'dossier': ['exact'],
'representative': ['exact'],
'score': ['exact', 'gte', 'lte']
}
search_fields = ('dossier', 'representative')
ordering_fields = ('representative', 'dossier')
pagination_class = DefaultWebPagination
serializer_class = DossierScoreSerializer
class RepresentativeScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view representative scores
"""
queryset = RepresentativeScore.objects.select_related('representative')
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'representative': ['exact'],
'score': ['exact', 'gte', 'lte']
}
search_fields = ('representative', 'score')
ordering_fields = ('representative', 'score')
pagination_class = DefaultWebPagination
serializer_class = RepresentativeScoreSerializer
class VoteScoreViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint to view votes with their score impact.
This endpoint only shows votes that have a matching recommendation.
"""
queryset = VoteScore.objects.select_related(
'vote__representative',
'vote__proposal',
'vote__proposal__dossier',
'vote__proposal__recommendation'
).filter(
vote__proposal__recommendation__isnull=False
)
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
RQLFilterBackend
)
filter_fields = {
'vote__representative': ['exact'],
'vote__proposal': ['exact'],
'vote__proposal__dossier': ['exact']
}
pagination_class = DefaultWebPagination
serializer_class = VoteScoreSerializer
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('representatives', '0020_rep_unique_slug_remove_remoteid'),
('representatives_votes', '0012_document'),
('representatives_positions', '0007_remove_positionscore'),
('representatives_recommendations', '0010_remove_views'),
('memopol_themes', '0003_remove_themescore'),
('memopol_settings', '0002_score_settings'),
]
operations = [
migrations.CreateModel(
name='DossierScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('dossier', models.ForeignKey(to='representatives_votes.Dossier')),
('representative', models.ForeignKey(related_name='dossier_scores', to='representatives.Representative')),
],
),
migrations.CreateModel(
name='PositionScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('position', models.OneToOneField(related_name='position_score', to='representatives_positions.Position')),
],
),
migrations.CreateModel(
name='RepresentativeScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('representative', models.OneToOneField(related_name='representative_score', to='representatives.Representative')),
],
),
migrations.CreateModel(
name='VoteScore',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('score', models.FloatField()),
('vote', models.OneToOneField(related_name='vote_score', to='representatives_votes.Vote')),
],
),
]
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('memopol_scores', '0001_initial'),
]
operations = [
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_vote_score"
AS SELECT
"representatives_votes_vote"."id" AS "vote_id",
decay_score(
CAST(CASE
WHEN "representatives_votes_vote"."position"::text = "representatives_recommendations_recommendation"."recommendation"::text
THEN "representatives_recommendations_recommendation"."weight"
ELSE 0 - "representatives_recommendations_recommendation"."weight"
END AS NUMERIC),
"representatives_votes_proposal"."datetime",
"decay_num"."value",
"decay_denom"."value",
"exponent"."value",
"decimals"."value"
) AS "score"
FROM "representatives_votes_vote"
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1
JOIN "representatives_votes_proposal" ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
LEFT JOIN "representatives_recommendations_recommendation" ON "representatives_votes_proposal"."id" = "representatives_recommendations_recommendation"."proposal_id"
WHERE "representatives_recommendations_recommendation"."id" IS NOT NULL;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_dossier_score"
AS SELECT
"representatives_votes_vote"."representative_id" AS "representative_id",
"representatives_votes_proposal"."dossier_id" AS "dossier_id",
SUM("memopol_scores_votescore"."score") AS "score"
FROM
"memopol_scores_votescore"
INNER JOIN "representatives_votes_vote"
ON "memopol_scores_votescore"."vote_id" = "representatives_votes_vote"."id"
INNER JOIN "representatives_votes_proposal"
ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
GROUP BY
"representatives_votes_vote"."representative_id",
"representatives_votes_proposal"."dossier_id"
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_position_score"
AS SELECT
"representatives_positions_position"."id" AS "position_id",
decay_score(
"representatives_positions_position"."score",
"representatives_positions_position"."datetime",
"decay_num"."value",
"decay_denom"."value",
"exponent"."value",
"decimals"."value"
) AS "score"
FROM
"representatives_positions_position"
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1;
"""
),
migrations.RunSQL(
"""
CREATE OR REPLACE VIEW "memopol_scores_v_representative_score"
AS SELECT
"source"."representative_id" AS "representative_id" ,
SUM("source"."score") AS "score"
FROM
(
SELECT
"memopol_scores_dossierscore"."representative_id" AS "representative_id",
"memopol_scores_dossierscore"."score" AS "score"
FROM "memopol_scores_dossierscore"
UNION ALL
SELECT
"representatives_positions_position"."representative_id" AS "representative_id",
"memopol_scores_positionscore"."score" AS "score"
FROM
"memopol_scores_positionscore"
INNER JOIN "representatives_positions_position"
ON "memopol_scores_positionscore"."position_id" = "representatives_positions_position"."id"
) "source"
GROUP BY
"source"."representative_id"
"""
),
migrations.RunSQL(
"""