Skip to content

GitLab

Commit d895d897 authored by James Pic's avatar James Pic Committed by GitHub
Browse files

Merge pull request #159 from political-memory/ci-fixes 

CI fixes
parents 2e8a891c 0558edff
Expand all Hide whitespace changes
Inline Side-by-side
.travis.yml
View file @ d895d897
sudo: false
<<<<<<< HEAD
env:
global:
- DJANGO_DEBUG=True
......
bin/quickstart.sh
View file @ d895d897
......@@ -17,10 +17,11 @@ virtualenv memopol_env
source memopol_env/bin/activate
# Install python dependencies
pip install -U pip setuptools
pip install -e .[testing]
# Install client dependencies
bin/install_client_deps.sh
src/memopol/bin/install_client_deps.sh
# Create pg user and database
if [ $(psql -c "select 'CNT=' || count(1) from pg_catalog.pg_user where usename='memopol';" -U postgres | grep CNT=1 | wc -l) -lt 1 ]; then
......
docs/deploy-custom.rst
View file @ d895d897
......@@ -35,8 +35,10 @@ allowed hosts. Setup your WSGI server to serve:
Initial memopol setup
=====================
From the repository root, install python dependencies::
From the repository root, install python dependencies (you may want to do that
in a virtualenv)::
$ pip install -U pip setuptools
$ pip install -Ue .
Install client libraries::
......@@ -45,11 +47,11 @@ Install client libraries::
Setup the database schema::
$ ./manage.py migrate --noinput
$ memopol migrate --noinput
Collect static files::
$ ./manage.py collectstatic --noinput
$ memopol collectstatic --noinput
Memopol should be ready to go.
......@@ -60,9 +62,9 @@ To update simply pull the repository and run setup commands again::
$ git pull
$ pip install -Ue .
$ bin/install_client_deps.sh
$ ./manage.py migrate --noinput
$ ./manage.py collectstatic --noinput
$ src/memopol/bin/install_client_deps.sh
$ memopol migrate --noinput
$ memopol collectstatic --noinput
Data provisionning
==================
......@@ -72,7 +74,7 @@ Set up two cron jobs:
* One to update data from parliaments, that runs ``bin/update_all``. This
script takes quite some time to run, so you should schedule it once every
night for example
* One to refresh scores, that runs ``./manage.py refresh_scores``. This one
* One to refresh scores, that runs ``memopol refresh_scores``. This one
runs quite quickly (a few seconds), you may want to run it after the update
job has completed (but you can run it more often).
......
setup.py
View file @ d895d897
......@@ -29,7 +29,6 @@ setup(name='political-memory',
'unicodecsv>=0.14,<0.15',
'pytz', # Always use up-to-date TZ data
'django-suit>=0.2,<0.3',
'sqlparse>=0.1,<0.2',
'psycopg2>=2,<3',
],
extras_require={
......@@ -43,11 +42,19 @@ setup(name='political-memory',
'pytest>=2,<3',
'pytest-django>=2,<3',
'pytest-cov>=2,<3',
'mock-2.0.0',
'mock==2.0.0',
'tox>=2.3,<3',
]
},
entry_points={
'console_scripts': [
'parltrack_import_representatives = representatives.contrib.parltrack.import_representatives:main', # noqa
'parltrack_import_dossiers = representatives_votes.contrib.parltrack.import_dossiers:main', # noqa
'parltrack_import_votes = representatives_votes.contrib.parltrack.import_votes:main', # noqa
'francedata_import_representatives = representatives.contrib.francedata.import_representatives:main', # noqa
'francedata_import_dossiers = representatives_votes.contrib.francedata.import_dossiers:main', # noqa
'francedata_import_scrutins = representatives_votes.contrib.francedata.import_scrutins:main', # noqa
'francedata_import_votes = representatives_votes.contrib.francedata.import_votes:main', # noqa
'memopol_import_positions = representatives_positions.contrib.import_positions:main', # noqa
'memopol_import_recommendations = representatives_recommendations.contrib.import_recommendations:main', # noqa
'memopol = memopol.manage:main',
......
src/memopol/tests/response_fixtures/PositionFormTest.test_create_position_without_datetime.content
View file @ d895d897
<div class="form-group has-error"><label class="col-md-3 control-label" for="id_position-datetime">Datetime</label><div class="col-md-9">
<div class="input-group date" id="id_position-datetime">
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
<span class="input-group-addon"><span class="glyphicon glyphicon-remove"></span></span>
<span class="input-group-addon"><span class="glyphicon glyphicon-calendar"></span></span>
</div>
<script type="text/javascript">
$("#id_position-datetime").datetimepicker({minView: 2,
autoclose: true,
language: 'en',
startView: 2,
format: 'yyyy-mm-dd'}).find('input').addClass("form-control");
</script>
<span class="help-block">This field is required.
</span></div></div>
\ No newline at end of file
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
\ No newline at end of file
src/memopol/tests/response_fixtures/PositionFormTest.test_create_position_without_link.content
View file @ d895d897
<div class="form-group has-error"><label class="col-md-3 control-label" for="id_position-link">Link</label><div class="col-md-9"><input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/><span class="help-block">This field is required.
</span></div></div>
\ No newline at end of file
<input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/>
\ No newline at end of file
src/memopol/tests/response_fixtures/PositionFormTest.test_position_form.content deleted 100644 → 0
View file @ 2e8a891c
<form action="" method="post">
<div class="modal-header">
<button aria-label="Close" class="close" data-dismiss="modal" type="button"><span aria-hidden="true">×</span></button>
<h4 class="modal-title">Add a representative public position</h4>
</div>
<div class="modal-body">
<input name="csrfmiddlewaretoken" type="hidden" value="csrftoken"/>
<div class="row">
<div class="col-sm-12">
<div class="well well-sm text-justify">
<p>
Use this form to submit a public position taken by a representative and
related to one of the themes followed on this instance of Political Memory.
Public positions may include blog or social network posts, interviews,
parliament interventions...
</p>
<p>
Be sure to include a relevant excerpt from the public position as well as
a valid link that refers to it. Note that positions will be reviewed by
the staff before publication.
</p>
</div>
</div>
</div>
<div class="row">
<div class="col-sm-6">
<div class="form-group"><label class="col-md-3 control-label" for="id_position-representative">Representative</label><div class="col-md-9"><select class="form-control" id="id_position-representative" name="position-representative" required="required" title="">
<option selected="selected" value="">---------</option>
<option value="4902">François Asensi</option>
<option value="4893">Thierry Benoit</option>
<option value="4898">Marcel Bonnot</option>
<option value="4891">Jean-Claude Bouchet</option>
<option value="22">Paul BRANNEN</option>
<option value="12">Udo BULLMANN</option>
<option value="4914">Jean-Paul Chanteguet</option>
<option value="4912">Jean-Louis Christ</option>
<option value="4900">Jean-Michel Couve</option>
<option value="21">Esther de LANGE</option>
<option value="9">Albert DESS</option>
<option value="4896">Marc Dolez</option>
<option value="4889">Dominique Dord</option>
<option value="4899">Olivier Dussopt</option>
<option value="4890">Daniel Fasquelle</option>
<option value="24">María Teresa GIMÉNEZ BARBAT</option>
<option value="4894">Claude Goasguen</option>
<option value="4885">Pascale Got</option>
<option value="13">Bolesław G. PIECHA</option>
<option value="20">Iveta GRIGULE</option>
<option value="1">Czesław HOC</option>
<option value="4910">Philippe Houillon</option>
<option value="7">Sylvia-Yvonne KAUFMANN</option>
<option value="18">Jan KELLER</option>
<option value="3">Dietmar KÖSTER</option>
<option value="29">Werner LANGEN</option>
<option value="23">Jo LEINEN</option>
<option value="4903">Pierre Lellouche</option>
<option value="4886">Annick Lepetit</option>
<option value="4895">Pierre Lequiller</option>
<option value="8">Arne LIETZ</option>
<option value="19">Verónica LOPE FONTAGNÉ</option>
<option value="4908">Jacqueline Maquet</option>
<option value="4907">Philippe Martin</option>
<option value="25">Gesine MEISSNER</option>
<option value="4904">Hervé Morin</option>
<option value="4913">Alain Moyne-Bressand</option>
<option value="16">Angelika NIEBLER</option>
<option value="14">Paul NUTTALL</option>
<option value="5">Patrick O'FLYNN</option>
<option value="4905">Martine Pinville</option>
<option value="15">Mirosław PIOTROWSKI</option>
<option value="4906">François Pupponi</option>
<option value="4911">Jean-Luc Reitzer</option>
<option value="4909">Franck Reynier</option>
<option value="4888">Marcel Rogemont</option>
<option value="4901">André Santini</option>
<option value="10">Annie SCHREIJER-PIERIK</option>
<option value="28">Joachim SCHUSTER</option>
<option value="26">Helga STEVENS</option>
<option value="17">László TŐKÉS</option>
<option value="4887">Jean-Jacques Urvoas</option>
<option value="4892">Alain Vidalies</option>
<option value="4897">Philippe Vigier</option>
<option value="6">Axel VOSS</option>
<option value="2">Renate WEBER</option>
<option value="11">Kerstin WESTPHAL</option>
<option value="4">Hermann WINKLER</option>
<option value="27">Damiano ZOFFOLI</option>
</select></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-datetime">Datetime</label><div class="col-md-9">
<div class="input-group date" id="id_position-datetime">
<input class="form-control" id="id_position-datetime" name="position-datetime" placeholder="Datetime" readonly="" required="required" title="" type="text"/>
<span class="input-group-addon"><span class="glyphicon glyphicon-remove"></span></span>
<span class="input-group-addon"><span class="glyphicon glyphicon-calendar"></span></span>
</div>
<script type="text/javascript">
$("#id_position-datetime").datetimepicker({minView: 2,
autoclose: true,
language: 'en',
startView: 2,
format: 'yyyy-mm-dd'}).find('input').addClass("form-control");
</script>
</div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-link">Link</label><div class="col-md-9"><input class="form-control" id="id_position-link" maxlength="500" name="position-link" placeholder="Link" required="required" title="" type="url"/></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-title">Title</label><div class="col-md-9"><input class="form-control" id="id_position-title" maxlength="500" name="position-title" placeholder="Title" required="required" title="" type="text"/></div></div>
</div>
<div class="col-sm-6">
<div class="form-group"><label class="col-md-3 control-label" for="id_position-kind">Kind</label><div class="col-md-9"><select class="form-control" id="id_position-kind" name="position-kind" required="required" title="">
<option selected="selected" value="other">Other</option>
<option value="blog">Blog post</option>
<option value="social">Social network</option>
<option value="press">Press interview</option>
<option value="parliament">Parliament debate</option>
</select></div></div>
<div class="form-group"><label class="col-md-3 control-label" for="id_position-themes_0">Themes</label><div class="col-md-9"><div id="id_position-themes"><div class="checkbox"><label for="id_position-themes_0"><input class="" id="id_position-themes_0" name="position-themes" title="" type="checkbox" value="1"/> Etat d'urgence</label></div>
<div class="checkbox"><label for="id_position-themes_1"><input class="" id="id_position-themes_1" name="position-themes" title="" type="checkbox" value="2"/> ACTA</label></div></div></div></div>
</div>
</div>
<div class="row">
<div class="col-sm-12">
<div class="col-sm-12">
<div class="form-group"><label class="control-label" for="id_position-text">Text</label><textarea class="form-control" cols="40" id="id_position-text" name="position-text" placeholder="Text" required="required" rows="10" title=""></textarea></div>
</div>
</div>
</div>
</div>
<div class="modal-footer">
<button class="btn btn-default" data-dismiss="modal" type="button">Close</button>
<button class="btn btn-primary" type="submit">Submit public position</button>
</div>
</form>
\ No newline at end of file
src/memopol/tests/response_fixtures/RepresentativeDetailTest.test_details.content
View file @ d895d897
......@@ -58,16 +58,16 @@
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="1 Rue Sadi Carnot – Annonay 07100, France">
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="126 Rue de l'Université – Paris 75355, France">
<i class="fa fa-envelope" title=""></i>
Other address
Assemblée nationale
</span>
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="126 Rue de l'Université – Paris 75355, France">
<span class="label label-primary" data-placement="bottom" data-toggle="tooltip" title="1 Rue Sadi Carnot – Annonay 07100, France">
<i class="fa fa-envelope" title=""></i>
Assemblée nationale
Other address
</span>
......
src/memopol/tests/response_fixtures/RepresentativeListTest.test_navbar_order_options.content
View file @ d895d897
......@@ -7,9 +7,9 @@
</li>
---
<li>
<a href="?&amp;sort=score-desc">Worst score</a>
<a href="?&amp;sort=score-asc">Best score</a>
</li>
---
<li>
<a href="?&amp;sort=score-asc">Best score</a>
<a href="?&amp;sort=score-desc">Worst score</a>
</li>
\ No newline at end of file
src/memopol/tests/response_fixtures/ThemeListTest.test_navbar_order_options.content
View file @ d895d897
<li>
<a href="?&amp;sort=name-desc">Name Z-A</a>
</li>
---
<li class="disabled">
<a href="?&amp;sort=name-asc">Name A-Z</a>
</li>
---
<li>
<a href="?&amp;sort=name-desc">Name Z-A</a>
</li>
\ No newline at end of file
src/memopol/tests/test_position_form.py
View file @ d895d897
......@@ -50,7 +50,8 @@ class PositionFormTest(BaseTest):
fixture.pop('position-representative')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
def test_create_position_without_datetime(self):
......@@ -58,7 +59,8 @@ class PositionFormTest(BaseTest):
fixture.pop('position-datetime')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
def test_create_position_without_link(self):
......@@ -66,5 +68,6 @@ class PositionFormTest(BaseTest):
fixture.pop('position-link')
response = self.client.post(self.create_url, fixture)
self.assertResponseDiffEmpty(response, '#add-position-form .has-error')
self.assertResponseDiffEmpty(response,
'#add-position-form .has-error .form-control')
assert response.context['position_form'].is_valid() is False
src/memopol/views/representative_detail_base.py
View file @ d895d897
......@@ -3,7 +3,8 @@
from django.db import models
from django.views import generic
from representatives.models import Chamber, Representative, Phone, WebSite
from representatives.models import (Address, Chamber, Representative, Phone,
WebSite)
from .representative_mixin import RepresentativeViewMixin
......@@ -30,22 +31,29 @@ class RepresentativeDetailBase(RepresentativeViewMixin, PositionFormMixin,
'email_set',
models.Prefetch(
'website_set',
queryset=WebSite.objects.filter(kind__in=social),
queryset=WebSite.objects.filter(kind__in=social)
.order_by('id'),
to_attr='social_websites'
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.filter(kind__in=chambers),
queryset=WebSite.objects.filter(kind__in=chambers)
.order_by('id'),
to_attr='chamber_websites'
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.exclude(kind__in=social)
.exclude(kind__in=chambers),
.exclude(kind__in=chambers)
.order_by('id'),
to_attr='other_websites'
),
'address_set__country',
'address_set__phones',
models.Prefetch(
'address_set',
queryset=Address.objects.select_related('country')
.prefetch_related('phones')
.order_by('id')
),
models.Prefetch(
'phone_set',
queryset=Phone.objects.filter(address__isnull=True)
......
src/memopol/views/representative_list.py
View file @ d895d897
......@@ -37,7 +37,7 @@ class RepresentativeList(CSVDownloadMixin, GridListMixin, PaginationMixin,
'fields': ['-representative_score__score']
},
'score-desc': {
'order': 2,
'order': 3,
'label': 'Worst score',
'fields': ['representative_score__score']
}
......
src/memopol/views/theme_list.py
View file @ d895d897
......@@ -30,7 +30,7 @@ class ThemeList(PaginationMixin, SortMixin, PositionFormMixin,
'fields': ['name']
},
'name-desc': {
'order': 0,
'order': 1,
'label': 'Name Z-A',
'fields': ['-name']
}
......
src/representatives/api.py
View file @ d895d897
......@@ -24,10 +24,12 @@ from .models import (
Chamber,
Constituency,
Country,
Email,
Group,
Mandate,
Phone,
Representative,
WebSite,
)
......@@ -49,7 +51,7 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
"""
API endpoint that allows representatives to be viewed.
"""
queryset = Representative.objects.all()
queryset = Representative.objects.order_by('slug')
filter_backends = (
filters.DjangoFilterBackend,
filters.SearchFilter,
......@@ -68,23 +70,33 @@ class RepresentativeViewSet(viewsets.ReadOnlyModelViewSet):
'birth_date': ['exact', 'gte', 'lte'],
}
search_fields = ('first_name', 'last_name', 'slug')
ordering_fields = ('id', 'birth_date', 'last_name', 'full_name')
pagination_class = DefaultWebPagination
def get_queryset(self):
qs = super(RepresentativeViewSet, self).get_queryset()
qs = qs.prefetch_related(
'email_set',
'website_set',
models.Prefetch(
'email_set',
queryset=Email.objects.order_by('id')
),
models.Prefetch(
'website_set',
queryset=WebSite.objects.order_by('id')
),
models.Prefetch(
'address_set',
queryset=Address.objects.select_related('country')
.order_by('id')
),
models.Prefetch(
'phone_set',
queryset=Phone.objects.select_related('address__country')
.order_by('id')
),
'mandates',
models.Prefetch(
'mandates',
queryset=Mandate.objects.order_by('id')
)
)
return qs
......@@ -102,7 +114,8 @@ class MandateViewSet(viewsets.ReadOnlyModelViewSet):
API endpoint that allows mandates to be viewed.
"""
pagination_class = DefaultWebPagination
queryset = Mandate.objects.select_related('representative')
queryset = Mandate.objects.select_related('representative') \
.order_by('representative_id', 'id')
serializer_class = MandateSerializer
filter_backends = (
......@@ -121,7 +134,7 @@ class MandateViewSet(viewsets.ReadOnlyModelViewSet):
class ConstituencyViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Constituency.objects.all()
queryset = Constituency.objects.order_by('id')
serializer_class = ConstituencySerializer
filter_backends = (
......@@ -131,7 +144,7 @@ class ConstituencyViewSet(viewsets.ReadOnlyModelViewSet):
class GroupViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Group.objects.all()
queryset = Group.objects.order_by('id')
serializer_class = GroupSerializer
filter_backends = (
......@@ -141,7 +154,7 @@ class GroupViewSet(viewsets.ReadOnlyModelViewSet):
class ChamberViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Chamber.objects.all()
queryset = Chamber.objects.order_by('id')
serializer_class = ChamberSerializer
filter_backends = (
......@@ -151,7 +164,7 @@ class ChamberViewSet(viewsets.ReadOnlyModelViewSet):
class CountryViewSet(viewsets.ReadOnlyModelViewSet):
pagination_class = DefaultWebPagination
queryset = Country.objects.all()
queryset = Country.objects.order_by('id')
serializer_class = CountrySerializer
filter_backends = (
......
src/representatives/contrib/francedata/tests/test_francedata_import_representatives.py
View file @ d895d897
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.tests.base import TestBase
from representatives.contrib.francedata import import_representatives
@pytest.mark.django_db
def test_francedata_import_representatives():
inputjson = os.path.join(os.path.dirname(__file__),
'representatives_input.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
class FranceDataRepresentativesTest(TestBase):
def test_francedata_import_representatives(self):
inputjson = os.path.join(os.path.dirname(__file__),
'representatives_input.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(inputjson, 'r') as f:
import_representatives.main(f)
with open(inputjson, 'r') as f:
import_representatives.main(f)
missing = []
with open(expected, 'r') as f:
for obj in Deserializer(f.read()):
compare = copy.copy(obj.object.__dict__)
for field in exclude:
if field in compare:
compare.pop(field)
try:
type(obj.object).objects.get(**compare)
except:
missing.append(compare)
assert len(missing) is 0
assert Representative.objects.count() == 2
self.assertObjectsFromFixture(expected)
assert Representative.objects.count() == 2
src/representatives/contrib/parltrack/tests/test_import_representatives.py
View file @ d895d897
import pytest
import os
import copy
from django.core.serializers.json import Deserializer
from representatives.models import Representative
from representatives.tests.base import TestBase
from representatives.contrib.parltrack import import_representatives
@pytest.mark.django_db
def test_parltrack_import_representatives():
fixture = os.path.join(os.path.dirname(__file__),
'representatives_fixture.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
class ParltracRepresentativesTest(TestBase):
def test_parltrack_import_representatives(self):
fixture = os.path.join(os.path.dirname(__file__),
'representatives_fixture.json')
expected = os.path.join(os.path.dirname(__file__),
'representatives_expected.json')
# Disable django auto fields
exclude = ('id', '_state', 'created', 'updated', 'fingerprint')
with open(fixture, 'r') as f:
import_representatives.main(f)
with open(fixture, 'r') as f: