Rework scores calculations using pseudo-materialized views

memopol/api.py

@@ -18,8 +18,11 @@ from representatives_votes.api import (
 )
 
 from representatives_recommendations.api import (
+    RecommendationViewSet
+)
+
+from memopol_scores.api import (
     DossierScoreViewSet,
-    RecommendationViewSet,
     RepresentativeScoreViewSet,
     VoteScoreViewSet
 )

memopol/filters.py

@@ -61,7 +61,7 @@ class RepresentativeFilter(FilterSet):
             return qs
 
         try:
-            return qs.filter(score__score__gte=int(value))
+            return qs.filter(representative_score__score__gte=int(value))
         except ValueError:
             return qs
 
@@ -70,7 +70,7 @@ class RepresentativeFilter(FilterSet):
             return qs
 
         try:
-            return qs.filter(score__score__lte=int(value))
+            return qs.filter(representative_score__score__lte=int(value))
         except ValueError:
             return qs
 

memopol/settings.py

@@ -91,6 +91,7 @@ INSTALLED_APPS = (
     # ---
     'core',
     'memopol',
+    'memopol_scores',
     'memopol_settings',
     'memopol_themes',
     'representatives',

memopol/templatetags/memopol_tags.py

@@ -97,6 +97,8 @@ def proposal_status_label(status, recommendation=None):
 
 @register.filter
 def score_badge(score, tooltip=None):
+    score = int(score or 0)
+
     if score > 0:
         color = 'success'
     elif score < 0:

memopol/tests/base.py

 from django import test
 from responsediff.test import ResponseDiffTestMixin
+from memopol_scores.models import RepresentativeScore
 
 
 class BaseTest(ResponseDiffTestMixin, test.TestCase):
@@ -19,6 +20,9 @@ class BaseTest(ResponseDiffTestMixin, test.TestCase):
     """
     left_pane_queries = 7
 
+    def setUp(self):
+        RepresentativeScore.refresh()
+
     def request_test(self, url=None):
         self.assertResponseDiffEmpty(self.client.get(url or self.url))
 

memopol/views/representative_detail_base.py

@@ -13,7 +13,7 @@ from representatives_positions.views import PositionFormMixin
 class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin,
                                generic.DetailView):
 
-    queryset = Representative.objects.select_related('score')
+    queryset = Representative.objects.select_related('representative_score')
 
     def get_queryset(self):
         qs = super(RepresentativeDetailBase, self).get_queryset()

memopol/views/representative_detail_positions.py

@@ -20,7 +20,7 @@ class RepresentativeDetailPositions(RepresentativeDetailBase):
                 .order_by('-datetime', 'pk')
             ),
             'positions__themes',
-            'positions__positionscore'
+            'positions__position_score'
         )
 
         return qs

memopol/views/representative_detail_votes.py

@@ -2,8 +2,7 @@
 
 from django.db import models
 
-from representatives_recommendations.models import VoteScore
-from representatives_votes.models import Proposal
+from representatives_votes.models import Vote
 
 from .representative_detail_base import RepresentativeDetailBase
 
@@ -17,14 +16,15 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
         qs = qs.prefetch_related(
             models.Prefetch(
                 'votes',
-                queryset=VoteScore.objects.filter(
-                    proposal__in=Proposal.objects.exclude(recommendation=None),
+                queryset=Vote.objects.exclude(
+                    proposal__recommendation=None
                 ).select_related(
+                    'vote_score',
                     'proposal__dossier',
                     'proposal__recommendation'
                 ).order_by('-proposal__datetime', 'proposal__title')
             ),
-            'dossierscores'
+            'dossier_scores'
         )
 
         return qs
@@ -32,7 +32,7 @@ class RepresentativeDetailVotes(RepresentativeDetailBase):
     def get_context_data(self, **kwargs):
         c = super(RepresentativeDetailVotes, self).get_context_data(**kwargs)
 
-        ds = c['object'].dossierscores.all()
+        ds = c['object'].dossier_scores.all()
 
         dossiers = {}
         for vote in c['object'].votes.all():

memopol/views/representative_list.py

@@ -18,7 +18,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
                          SortMixin, PositionFormMixin, generic.ListView):
 
     csv_name = 'representatives'
-    queryset = Representative.objects.select_related('score')
+    queryset = Representative.objects.select_related('representative_score')
     current_filter = None
     sort_modes = {
         'name-asc': {
@@ -34,12 +34,12 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
         'score-asc': {
             'order': 2,
             'label': 'Best score',
-            'fields': ['-score__score']
+            'fields': ['-representative_score__score']
         },
         'score-desc': {
             'order': 2,
             'label': 'Worst score',
-            'fields': ['score__score']
+            'fields': ['representative_score__score']
         }
     }
     sort_default = 'name-asc'

memopol/views/representative_mandates.py

-# coding: utf-8
-
-from django.views import generic
-
-from representatives.models import Representative
-
-from .representative_mixin import RepresentativeViewMixin
-
-
-class RepresentativeMandates(RepresentativeViewMixin, generic.DetailView):
-    template_name = 'representatives/representative_mandates'
-
-    queryset = Representative.objects.select_related('score')
-
-    def get_queryset(self):
-        qs = super(RepresentativeMandates, self).get_queryset()
-
-        qs = self.prefetch_for_representative_country_and_main_mandate(qs)
-
-        return qs
-
-    def get_context_data(self, **kwargs):
-        c = super(RepresentativeMandates, self).get_context_data(**kwargs)
-
-        self.add_representative_country_and_main_mandate(c['object'])
-
-        c['mandates'] = c['object'].mandates.all()
-
-        return c

memopol/views/representative_positions.py

-# coding: utf-8
-
-from django.db import models
-from django.views import generic
-
-from representatives.models import Representative
-from representatives_positions.forms import PositionForm
-from representatives_positions.models import Position
-
-from .representative_mixin import RepresentativeViewMixin
-
-
-class RepresentativePositions(RepresentativeViewMixin, generic.DetailView):
-    template_name = 'representatives/representative_positions'
-
-    queryset = Representative.objects.select_related('score')
-
-    def get_queryset(self):
-        qs = super(RepresentativePositions, self).get_queryset()
-
-        qs = self.prefetch_for_representative_country_and_main_mandate(qs)
-
-        qs = qs.prefetch_related(
-            models.Prefetch(
-                'positions',
-                queryset=Position.objects.filter(published=True)
-                .order_by('-datetime', 'pk')
-            )
-        )
-
-        return qs
-
-    def get_context_data(self, **kwargs):
-        c = super(RepresentativePositions, self).get_context_data(**kwargs)
-
-        c['position_form'] = PositionForm(
-            initial={'representative': self.object.pk})
-        self.add_representative_country_and_main_mandate(c['object'])
-
-        return c

memopol/views/representative_votes.py

-# coding: utf-8
-
-from django.db import models
-from django.views import generic
-
-from representatives.models import Representative
-from representatives_recommendations.models import VoteScore
-from representatives_votes.models import Proposal
-
-from .representative_mixin import RepresentativeViewMixin
-
-
-class RepresentativeVotes(RepresentativeViewMixin, generic.DetailView):
-    template_name = 'representatives/representative_votes'
-
-    queryset = Representative.objects.select_related('score')
-
-    def get_queryset(self):
-        qs = super(RepresentativeVotes, self).get_queryset()
-
-        qs = self.prefetch_for_representative_country_and_main_mandate(qs)
-
-        qs = qs.prefetch_related(
-            models.Prefetch(
-                'votes',
-                queryset=VoteScore.objects.filter(
-                    proposal__in=Proposal.objects.exclude(recommendation=None),
-                ).select_related('proposal__recommendation').order_by(
-                    '-proposal__datetime')
-            )
-        )
-
-        return qs
-
-    def get_context_data(self, **kwargs):
-        c = super(RepresentativeVotes, self).get_context_data(**kwargs)
-
-        self.add_representative_country_and_main_mandate(c['object'])
-
-        c['votes'] = c['object'].votes.all()
-
-        return c

memopol/views/theme_detail_positions.py

@@ -9,7 +9,7 @@ class ThemeDetailPositions(ThemeDetailBase):
     def get_queryset(self):
         qs = super(ThemeDetailPositions, self).get_queryset()
         qs = qs.prefetch_related('positions__representative',
-                                 'positions__positionscore')
+                                 'positions__position_score')
         return qs
 
     def get_context_data(self, **kwargs):

memopol_scores/api.py

+from rest_framework import (
+    filters,
+    viewsets,
+)
+
+from rql_filter.backend import RQLFilterBackend
+
+from representatives.api import DefaultWebPagination
+
+from .models import (
+    DossierScore,
+    RepresentativeScore,
+    VoteScore
+)
+
+from .serializers import (
+    DossierScoreSerializer,
+    RepresentativeScoreSerializer,
+    VoteScoreSerializer
+)
+
+
+class DossierScoreViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint to view representative score contribution for each dossier
+    """
+    queryset = DossierScore.objects.all()
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter,
+        RQLFilterBackend
+    )
+    filter_fields = {
+        'dossier': ['exact'],
+        'representative': ['exact'],
+        'score': ['exact', 'gte', 'lte']
+    }
+    search_fields = ('dossier', 'representative')
+    ordering_fields = ('representative', 'dossier')
+    pagination_class = DefaultWebPagination
+    serializer_class = DossierScoreSerializer
+
+
+class RepresentativeScoreViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint to view representative scores
+    """
+    queryset = RepresentativeScore.objects.select_related('representative')
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter,
+        RQLFilterBackend
+    )
+    filter_fields = {
+        'representative': ['exact'],
+        'score': ['exact', 'gte', 'lte']
+    }
+    search_fields = ('representative', 'score')
+    ordering_fields = ('representative', 'score')
+    pagination_class = DefaultWebPagination
+    serializer_class = RepresentativeScoreSerializer
+
+
+class VoteScoreViewSet(viewsets.ReadOnlyModelViewSet):
+    """
+    API endpoint to view votes with their score impact.
+    This endpoint only shows votes that have a matching recommendation.
+    """
+    queryset = VoteScore.objects.select_related(
+        'vote__representative',
+        'vote__proposal',
+        'vote__proposal__dossier',
+        'vote__proposal__recommendation'
+    ).filter(
+        vote__proposal__recommendation__isnull=False
+    )
+
+    filter_backends = (
+        filters.DjangoFilterBackend,
+        filters.SearchFilter,
+        filters.OrderingFilter,
+        RQLFilterBackend
+    )
+
+    filter_fields = {
+        'vote__representative': ['exact'],
+        'vote__proposal': ['exact'],
+        'vote__proposal__dossier': ['exact']
+    }
+
+    pagination_class = DefaultWebPagination
+    serializer_class = VoteScoreSerializer

memopol_scores/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('representatives', '0020_rep_unique_slug_remove_remoteid'),
+        ('representatives_votes', '0012_document'),
+        ('representatives_positions', '0007_remove_positionscore'),
+        ('representatives_recommendations', '0010_remove_views'),
+        ('memopol_themes', '0003_remove_themescore'),
+        ('memopol_settings', '0002_score_settings'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DossierScore',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('score', models.FloatField()),
+                ('dossier', models.ForeignKey(to='representatives_votes.Dossier')),
+                ('representative', models.ForeignKey(related_name='dossier_scores', to='representatives.Representative')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='PositionScore',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('score', models.FloatField()),
+                ('position', models.OneToOneField(related_name='position_score', to='representatives_positions.Position')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='RepresentativeScore',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('score', models.FloatField()),
+                ('representative', models.OneToOneField(related_name='representative_score', to='representatives.Representative')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='VoteScore',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('score', models.FloatField()),
+                ('vote', models.OneToOneField(related_name='vote_score', to='representatives_votes.Vote')),
+            ],
+        ),
+    ]

memopol_scores/migrations/0002_create_views.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('memopol_scores', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE VIEW "memopol_scores_v_vote_score"
+            AS SELECT
+                "representatives_votes_vote"."id" AS "vote_id",
+                decay_score(
+                    CAST(CASE
+                        WHEN "representatives_votes_vote"."position"::text = "representatives_recommendations_recommendation"."recommendation"::text
+                        THEN "representatives_recommendations_recommendation"."weight"
+                        ELSE 0 - "representatives_recommendations_recommendation"."weight"
+                    END AS NUMERIC),
+                    "representatives_votes_proposal"."datetime",
+                    "decay_num"."value",
+                    "decay_denom"."value",
+                    "exponent"."value",
+                    "decimals"."value"
+                ) AS "score"
+            FROM "representatives_votes_vote"
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1
+                JOIN "representatives_votes_proposal" ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
+                LEFT JOIN "representatives_recommendations_recommendation" ON "representatives_votes_proposal"."id" = "representatives_recommendations_recommendation"."proposal_id"
+            WHERE "representatives_recommendations_recommendation"."id" IS NOT NULL;
+            """
+        ),
+
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE VIEW "memopol_scores_v_dossier_score"
+            AS SELECT
+                "representatives_votes_vote"."representative_id" AS "representative_id",
+                "representatives_votes_proposal"."dossier_id" AS "dossier_id",
+                SUM("memopol_scores_votescore"."score") AS "score"
+            FROM
+                "memopol_scores_votescore"
+                INNER JOIN "representatives_votes_vote"
+                    ON "memopol_scores_votescore"."vote_id" = "representatives_votes_vote"."id"
+                INNER JOIN "representatives_votes_proposal"
+                    ON "representatives_votes_vote"."proposal_id" = "representatives_votes_proposal"."id"
+            GROUP BY
+                "representatives_votes_vote"."representative_id",
+                "representatives_votes_proposal"."dossier_id"
+            """
+        ),
+
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE VIEW "memopol_scores_v_position_score"
+            AS SELECT
+                "representatives_positions_position"."id" AS "position_id",
+                decay_score(
+                    "representatives_positions_position"."score",
+                    "representatives_positions_position"."datetime",
+                    "decay_num"."value",
+                    "decay_denom"."value",
+                    "exponent"."value",
+                    "decimals"."value"
+                ) AS "score"
+            FROM
+                "representatives_positions_position"
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_NUM') "decay_num" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECAY_DENOM') "decay_denom" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS NUMERIC) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_EXPONENT') "exponent" ON 1=1
+                JOIN (SELECT CAST(TO_NUMBER("value", '99999') AS INTEGER) AS "value" FROM "memopol_settings_setting" WHERE "key" = 'SCORE_DECIMALS') "decimals" ON 1=1;
+            """
+        ),
+
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE VIEW "memopol_scores_v_representative_score"
+            AS SELECT
+                "source"."representative_id" AS "representative_id" ,
+                SUM("source"."score") AS "score"
+            FROM
+                (
+                    SELECT
+                        "memopol_scores_dossierscore"."representative_id" AS "representative_id",
+                        "memopol_scores_dossierscore"."score" AS "score"
+                    FROM "memopol_scores_dossierscore"
+                    UNION ALL
+                    SELECT
+                        "representatives_positions_position"."representative_id" AS "representative_id",
+                        "memopol_scores_positionscore"."score" AS "score"
+                    FROM
+                        "memopol_scores_positionscore"
+                        INNER JOIN "representatives_positions_position"
+                            ON "memopol_scores_positionscore"."position_id" = "representatives_positions_position"."id"
+                ) "source"
+            GROUP BY
+                "source"."representative_id"
+            """
+        ),
+
+        migrations.RunSQL(
+            """
+            CREATE OR REPLACE FUNCTION refresh_vote_scores()