Commit d895d897 authored by James Pic's avatar James Pic Committed by GitHub

Merge pull request #159 from political-memory/ci-fixes

CI fixes
parents 2e8a891c 0558edff
sudo: false
<<<<<<< HEAD
env:
global:
- DJANGO_DEBUG=True
......
......@@ -17,10 +17,11 @@ virtualenv memopol_env
source memopol_env/bin/activate
# Install python dependencies
pip install -U pip setuptools
pip install -e .[testing]
# Install client dependencies
bin/install_client_deps.sh
src/memopol/bin/install_client_deps.sh
# Create pg user and database
if [ $(psql -c "select 'CNT=' || count(1) from pg_catalog.pg_user where usename='memopol';" -U postgres | grep CNT=1 | wc -l) -lt 1 ]; then
......
......@@ -35,8 +35,10 @@ allowed hosts. Setup your WSGI server to serve:
Initial memopol setup
=====================
From the repository root, install python dependencies::
From the repository root, install python dependencies (you may want to do that
in a virtualenv)::
$ pip install -U pip setuptools
$ pip install -Ue .
Install client libraries::
......@@ -45,11 +47,11 @@ Install client libraries::
Setup the database schema::
$ ./manage.py migrate --noinput
$ memopol migrate --noinput
Collect static files::
$ ./manage.py collectstatic --noinput
$ memopol collectstatic --noinput
Memopol should be ready to go.
......@@ -60,9 +62,9 @@ To update simply pull the repository and run setup commands again::
$ git pull
$ pip install -Ue .
$ bin/install_client_deps.sh
$ ./manage.py migrate --noinput
$ ./manage.py collectstatic --noinput
$ src/memopol/bin/install_client_deps.sh
$ memopol migrate --noinput
$ memopol collectstatic --noinput
Data provisionning
==================
......@@ -72,7 +74,7 @@ Set up two cron jobs:
* One to update data from parliaments, that runs ``bin/update_all``. This
script takes quite some time to run, so you should schedule it once every
night for example
* One to refresh scores, that runs ``./manage.py refresh_scores``. This one
* One to refresh scores, that runs ``memopol refresh_scores``. This one
runs quite quickly (a few seconds), you may want to run it after the update
job has completed (but you can run it more often).
......
......@@ -29,7 +29,6 @@ setup(name='political-memory',
'unicodecsv>=0.14,<0.15',
'pytz', # Always use up-to-date TZ data
'django-suit>=0.2,<0.3',
'sqlparse>=0.1,<0.2',
'psycopg2>=2,<3',
],
extras_require={
......@@ -43,11 +42,19 @@ setup(name='political-memory',
'pytest>=2,<3',
'pytest-django>=2,<3',
'pytest-cov>=2,<3',
'mock-2.0.0',
'mock==2.0.0',
'tox>=2.3,<3',
]
},
entry_points={
'console_scripts': [
'parltrack_import_representatives = representatives.contrib.parltrack.import_representatives:main', # noqa
'parltrack_import_dossiers = representatives_votes.contrib.parltrack.import_dossiers:main', # noqa
'parltrack_import_votes = representatives_votes.contrib.parltrack.import_votes:main', # noqa
'francedata_import_representatives = representatives.contrib.francedata.import_representatives:main', # noqa
'francedata_import_dossiers = representatives_votes.contrib.francedata.import_dossiers:main', # noqa
'francedata_import_scrutins = representatives_votes.contrib.francedata.import_scrutins:main', # noqa
'francedata_import_votes = representatives_votes.contrib.francedata.import_votes:main', # noqa
'memopol_import_positions = representatives_positions.contrib.import_positions:main', # noqa
'memopol_import_recommendations = representatives_recommendations.contrib.import_recommendations:main', # noqa
'memopol = memopol.manage:main',
......
<div class="form-group has-error"><label class="col-md-3 control-label" for="id_position-datetime">Datetime</label><div class="col-md-9">
<div class="input-group date" id="id_position-datetime">
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
<span class="input-group-addon"><span class="glyphicon glyphicon-remove"></span></span>
<span class="input-group-addon"><span class="glyphicon glyphicon-calendar"></span></span>
</div>
<script type="text/javascript">
$("#id_position-datetime").datetimepicker({minView: 2,
autoclose: true,
language: 'en',
startView: 2,
format: 'yyyy-mm-dd'}).find('input').addClass("form-control");
</script>
<span class="help-block">This field is required.
</span></div></div>
\ No newline at end of file
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
\ No newline at end of file
<div class="form-group has-error"><label class="col-md-3 control-label" for="id_position-link">Link</label><div class="col-md-9"><input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/><span class="help-block">This field is required.
</span></div></div>
\ No newline at end of file
<input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/>
\ No newline at end of file
<form action="" method="post">
<div class="modal-header">
<button aria-label="Close" class="close" data-dismiss="modal" type="button"><span aria-hidden="true">×</span></button>
<h4 class="modal-title">Add a representative public position</h4>
</div>
<div class="modal-body">
<input name="csrfmiddlewaretoken" type="hidden" value="csrftoken"/>
<div class="row">
<div class="col-sm-12">
<div class="well well-sm text-justify">
<p>
Use this form to submit a public position taken by a representative and
related to one of the themes followed on this instance of Political Memory.
Public positions may include blog or social network posts, interviews,
parliament interventions...
</p>
<p>
Be sure to include a relevant excerpt from the public position as well as
a valid link that refers to it. Note that positions will be reviewed by
the staff before publication.
</p>
</div>
</div>
</div>
<div class="row">
<div class="col-sm-6">
<div class="form-group"><label class="col-md-3 control-label" for="id_position-representative">Representative</label><div class="col-md-9"><select class="form-control" id="id_position-representative" name="position-representative" required="required" title="">
<option selected="selected" value="">---------</option>
<option value="4902">François Asensi</option>
<option value="4893">Thierry Benoit</option>
<option value="4898">Marcel Bonnot</option>
<option value="4891">Jean-Claude Bouchet</option>
<option value="22">Paul BRANNEN</option>
<option value="12">Udo BULLMANN</option>
<option value="4914">Jean-Paul Chanteguet</option>
<option value="4912">Jean-Louis Christ</option>
<option value="4900">Jean-Michel Couve</option>
<option value="21">Esther de LANGE</option>
<option value="9">Albert DESS</option>
<option value="4896">Marc Dolez</option>
<option value="4889">Dominique Dord</option>
<option value="4899">Olivier Dussopt</option>
<option value="4890">Daniel Fasquelle</option>
<option value="24">María Teresa GIMÉNEZ BARBAT</option>
<option value="4894">Claude Goasguen</option>
<option value="4885">Pascale Got</option>
<option value="13">Bolesław G. PIECHA</option>
<option value="20">Iveta GRIGULE</option>
<option value="1">Czesław HOC</option>
<option value="4910">Philippe Houillon</option>
<option value="7">Sylvia-Yvonne KAUFMANN</option>
<option value="18">Jan KELLER</option>
<option value="3">Dietmar KÖSTER</option>
<option value="29">Werner LANGEN</option>
<option value="23">Jo LEINEN</option>
<option value="4903">Pierre Lellouche</option>
<option value="4886">Annick Lepetit</option>
<option value="4895">Pierre Lequiller</option>
<option value="8">Arne LIETZ</option>
<option value="19">Verónica LOPE FONTAGNÉ</option>
<option value="4908">Jacqueline Maquet</option>
<option value="4907">Philippe Martin</option>
<option value="25">Gesine MEISSNER</option>
<option value="4904">Hervé Morin</option>
<option value="4913">Alain Moyne-Bressand</option>
<option value="16">Angelika NIEBLER</option>
<option value="14">Paul NUTTALL</option>
<option value="5">Patrick O'FLYNN</option>
<option value="4905">Martine Pinville</option>
<option value="15">Mirosław PIOTROWSKI</option>
<option value="4906">François Pupponi</option>
<option value="4911">Jean-Luc Reitzer</option>
<option value="4909">Franck Reynier</option>
<option value="4888">Marcel Rogemont</option>
<option value="4901">André Santini</option>
<option value="10">Annie SCHREIJER-PIERIK</option>
<option value="28">Joachim SCHUSTER</option>
<option value="26">Helga STEVENS</option>
<option value="17">László TŐKÉS</option>
<option value="4887">Jean-Jacques Urvoas</option>
<option value="4892">Alain Vidalies</option>
<option value="4897">Philippe Vigier</option>
<option value="6">Axel VOSS</option>
<option value="2">Renate WEBER</option>
<option value="11">Kerstin WESTPHAL</option>
<option value="4">Hermann WINKLER</option>
<option value="27">Damiano ZOFFOLI</option>
</select></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-datetime">Datetime</label><div class="col-md-9">
<div class="input-group date" id="id_position-datetime">
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
<span class="input-group-addon"><span class="glyphicon glyphicon-remove"></span></span>
<span class="input-group-addon"><span class="glyphicon glyphicon-calendar"></span></span>
</div>
<script type="text/javascript">
$("#id_position-datetime").datetimepicker({minView: 2,
autoclose: true,
language: 'en',
startView: 2,
format: 'yyyy-mm-dd'}).find('input').addClass("form-control");
</script>
</div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-link">Link</label><div class="col-md-9"><input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-title">Title</label><div class="col-md-9"><input class="form-control" id="id_position-title" maxlength="500" name="position-title" placeholder="Title" required="required" title="" type="text"/></div></div>
</div>
<div class="col-sm-6">
<div class="form-group"><label class="col-md-3 control-label" for="id_position-kind">Kind</label><div class="col-md-9"><select class="form-control" id="id_position-kind" name="position-kind" required="required" title="">
<option selected="selected" value="other">Other</option>
<option value="blog">Blog post</option>
<option value="social">Social network</option>
<option value="press">Press interview</option>
<option value="parliament">Parliament debate</option>
</select></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-themes_0">Themes</label><div class="col-md-9"><div id="id_position-themes"><div class="checkbox"><label for="id_position-themes_0"><input class="" id="id_position-themes_0" name="position-themes" title="" type="checkbox" value="1"/> Etat d'urgence</label></div>
<div class="checkbox"><label for="id_position-themes_1"><input class="" id="id_position-themes_1" name="position-themes" title="" type="checkbox" value="2"/> ACTA</label></div></div></div></div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<div class="col-sm-12">
<div class="form-group"><label class="control-label" for="id_position-text">Text</label><textarea class="form-control" cols="40" id="id_position-text" name="position-text" placeholder="Text" required="required" rows="10" title=""></textarea></div>
</div>
</div>
</div>
</div>
<div class="modal-footer">
<button class="btn btn-default" data-dismiss="modal" type="button">Close</button>
<button class="btn btn-primary" type="submit">Submit public position</button>
</div>
</form>
\ No newline at end of file
......@@ -58,16 +58,16 @@
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="1 Rue Sadi Carnot – Annonay 07100, France">
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="126 Rue de l'Université – Paris 75355, France">
<i class="fa fa-envelope" title=""></i>
Other address
Assemblée nationale
</span>
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="126 Rue de l'Université – Paris 75355, France">
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="1 Rue Sadi Carnot – Annonay 07100, France">
<i class="fa fa-envelope" title=""></i>
Assemblée nationale
Other address
</span>
......
......@@ -7,9 +7,9 @@
</li>
---
<li>
<a href="?&amp;sort=score-desc">Worst score</a>
<a href="?&amp;sort=score-asc">Best score</a>
</li>
---
<li>
<a href="?&amp;sort=score-asc">Best score</a>
<a href="?&amp;sort=score-desc">Worst score</a>
</li>
\ No newline at end of file
<li>
<a href="?&amp;sort=name-desc">Name Z-A</a>
</li>
---
<li class="disabled">
<a href="?&amp;sort=name-asc">Name A-Z</a>
</li>
---
<li>
<a href="?&amp;sort=name-desc">Name Z-A</a>
</li>
\ No newline at end of file
......@@ -50,7 +50,8 @@ class PositionFormTest(BaseTest):
fixture.pop('position-representative')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
def test_create_position_without_datetime(self):
......@@ -58,7 +59,8 @@ class PositionFormTest(BaseTest):
fixture.pop('position-datetime')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
def test_create_position_without_link(self):
......@@ -66,5 +68,6 @@ class PositionFormTest(BaseTest):
fixture.pop('position-link')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
......@@ -3,7 +3,8 @@
from django.db import models
from django.views import generic
from representatives.models import Chamber, Representative, Phone, WebSite
from representatives.models import (Address, Chamber, Representative, Phone,
WebSite)
from .representative_mixin import RepresentativeViewMixin
......@@ -30,22 +31,29 @@ class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin,
'email_set',
models.Prefetch(
'website_set',
queryset=WebSite.objects.filter(kind__in=social),
queryset=WebSite.objects.filter(kind__in=social)
.order_by('id'),
to_attr='social_websites'
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.filter(kind__in=chambers),
queryset=WebSite.objects.filter(kind__in=chambers)
.order_by('id'),
to_attr='chamber_websites'
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.exclude(kind__in=social)
.exclude(kind__in=chambers),
.exclude(kind__in=chambers)
.order_by('id'),
to_attr='other_websites'
),
'address_set__country',
'address_set__phones',
models.Prefetch(
'address_set',
queryset=Address.objects.select_related('country')
.prefetch_related('phones')
.order_by('id')
),
models.Prefetch(
'phone_set',
queryset=Phone.objects.filter(address__isnull=True)
......
......@@ -37,7 +37,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
'fields': ['-representative_score__score']
},
'score-desc': {
'order': 2,
'order': 3,
'label': 'Worst score',
'fields': ['representative_score__score']
}
......
......@@ -30,7 +30,7 @@ class ThemeList(PaginationMixin, SortMixin, PositionFormMixin,
'fields': ['name']
},
'name-desc': {
'order': 0,
'order': 1,
'label': 'Name Z-A',
'fields': ['-name']
}
......
......@@ -24,10 +24,12 @@ from .models import (
Chamber,
Constituency,
Country,
Email,
Group,
Mandate,
Phone,
Representative,
WebSite,
)
......@@ -49,7 +51,7 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows representatives to be viewed.
"""
queryset = Representative.objects.all()
queryset = Representative.objects.order_by('slug')
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
......@@ -68,23 +70,33 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
'birth_date': ['exact', 'gte', 'lte'],
}
search_fields = ('first_name', 'last_name', 'slug')
ordering_fields = ('id', 'birth_date', 'last_name', 'full_name')
pagination_class = DefaultWebPagination
def get_queryset(self):
qs = super(RepresentativeViewSet, self).get_queryset()
qs = qs.prefetch_related(
'email_set',
'website_set',
models.Prefetch(
'email_set',
queryset=Email.objects.order_by('id')
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.order_by('id')
),
models.Prefetch(
'address_set',
queryset=Address.objects.select_related('country')
.order_by('id')
),
models.Prefetch(
'phone_set',
queryset=Phone.objects.select_related('address__country')
.order_by('id')
),
'mandates',
models.Prefetch(
'mandates',
queryset=Mandate.objects.order_by('id')
)
)
return qs
......@@ -102,7 +114,8 @@ class MandateViewSet(viewsets.ReadOnlyModelViewSet):
API endpoint that allows mandates to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Mandate.objects.select_related('representative')
queryset = Mandate.objects.select_related('representative') \
.order_by('representative_id', 'id')
serializer_class = MandateSerializer
filter_backends = (
......@@ -121,7 +134,7 @@ class MandateViewSet(viewsets.ReadOnlyModelViewSet):
class ConstituencyViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Constituency.objects.all()
queryset = Constituency.objects.order_by('id')
serializer_class = ConstituencySerializer
filter_backends = (
......@@ -131,7 +144,7 @@ class ConstituencyViewSet(viewsets.ReadOnlyModelViewSet):
class GroupViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Group.objects.all()
queryset = Group.objects.order_by('id')
serializer_class = GroupSerializer
filter_backends = (
......@@ -141,7 +154,7 @@ class GroupViewSet(viewsets.ReadOnlyModelViewSet):
class ChamberViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Chamber.objects.all()
queryset = Chamber.objects.order_by('id')
serializer_class = ChamberSerializer
filter_backends = (
......@@ -151,7 +164,7 @@ class ChamberViewSet(viewsets.ReadOnlyModelViewSet):
class CountryViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Country.objects.all()
queryset = Country.objects.order_by('id')
serializer_class = CountrySerializer
filter_backends = (
......
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.tests.base import TestBase
from representatives.contrib.francedata import import_representatives
@pytest.mark.django_db
def test_francedata_import_representatives():
inputjson = os.path.join(os.path.dirname(__file__),
'representatives_input.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
class FranceDataRepresentativesTest(TestBase):
def test_francedata_import_representatives(self):
inputjson = os.path.join(os.path.dirname(__file__),
'representatives_input.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(inputjson, 'r') as f:
import_representatives.main(f)
with open(inputjson, 'r') as f:
import_representatives.main(f)
missing = []
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for field in exclude:
if field in compare:
compare.pop(field)
try:
type(obj.object).objects.get(**compare)
except:
missing.append(compare)
assert len(missing) is 0
assert Representative.objects.count() == 2
self.assertObjectsFromFixture(expected)
assert Representative.objects.count() == 2
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.tests.base import TestBase
from representatives.contrib.parltrack import import_representatives
@pytest.mark.django_db
def test_parltrack_import_representatives():
fixture = os.path.join(os.path.dirname(__file__),
'representatives_fixture.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
class ParltracRepresentativesTest(TestBase):
def test_parltrack_import_representatives(self):
fixture = os.path.join(os.path.dirname(__file__),
'representatives_fixture.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(fixture, 'r') as f:
import_representatives.main(f)
with open(fixture, 'r') as f:
import_representatives.main(f)
missing = []
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for field in exclude:
if field in compare:
compare.pop(field)
try:
type(obj.object).objects.get(**compare)
except:
missing.append(compare)
assert len(missing) is 0
assert Representative.objects.count() == 2
self.assertObjectsFromFixture(expected)
assert Representative.objects.count() == 2