Comparer les révisions

81c7c918 · 81c7c918 · 81c7c918 · 81c7c918 · 5f31138e · 81c7c918
--- a/bin/update_all
+++ b/bin/update_all
@@ -2,15 +2,14 @@

 bin/update_representatives

-# grace time for pg
-sleep 120
+sleep 10

 bin/update_dossiers

-sleep 120
+sleep 10

-bin/update_votes
+bin/update_proposals

-sleep 120
+sleep 10

-bin/update_scores
+bin/update_votes
--- a/bin/update_dossiers
+++ b/bin/update_dossiers
 #!/bin/bash
-set -ex
+set -x

-source ${OPENSHIFT_REPO_DIR}bin/lib.sh
+source bin/lib.sh

-pipe_download_to_command ep_dossiers.json.xz parltrack_import_dossiers
+parltrack_download_pipe ep_dossiers.json.xz parltrack_import_dossiers
+
+sleep 10
+
+francedata_download_pipe dossiers.json.gz francedata_import_dossiers
--- a/bin/update_proposals
+++ b/bin/update_proposals
+#!/bin/bash
+set -x
+
+source bin/lib.sh
+
+# Nothing for parltrack - proposals imported with votes
+
+francedata_download_pipe scrutins.json.gz francedata_import_scrutins
--- a/bin/update_representatives
+++ b/bin/update_representatives
 #!/bin/bash
-set -ex
+set -x

-source ${OPENSHIFT_REPO_DIR}bin/lib.sh
+source bin/lib.sh

-pipe_download_to_command ep_meps_current.json.xz parltrack_import_representatives
+parltrack_download_pipe ep_meps_current.json.xz parltrack_import_representatives
+
+sleep 10
+
+francedata_download_pipe parlementaires.json.gz francedata_import_representatives
--- a/bin/update_scores
+++ b/bin/update_scores
-#!/bin/bash
-set -ex
-
-source ${OPENSHIFT_REPO_DIR}bin/lib.sh
-
-[ -n "$OPENSHIFT_REPO_DIR" ] && cd $OPENSHIFT_REPO_DIR
-./manage.py update_score
--- a/bin/update_votes
+++ b/bin/update_votes
 #!/bin/bash
-set -ex
+set -x

-source ${OPENSHIFT_REPO_DIR}bin/lib.sh
+source bin/lib.sh

-pipe_download_to_command ep_votes.json.xz parltrack_import_votes
+parltrack_download_pipe ep_votes.json.xz parltrack_import_votes
+
+sleep 10
+
+parltrack_download_pipe ep_com_votes.json.xz parltrack_import_votes
+
+sleep 10
+
+francedata_download_pipe votes.json.gz francedata_import_votes
+
+if [ "x$1" != "xnoscores" ]; then
+	sleep 10
+	refresh_scores
+fi
--- a/core/templates/base.html
+++ b/core/templates/base.html
-<!DOCTYPE html>
-<html class="no-js" lang="en"> <!--<![endif]-->
-  <head>
-    <meta charset="utf-8" />
-    <link rel="icon" type="image/png" href="{{ STATIC_URL }}img/favicon.ico" />
-    <!-- Set the viewport width to device width for mobile -->
-    <meta name="viewport" content="width=device-width" />
-    <title>
-      {% block title %}Home{% endblock %}
-      - The Political Memory of {{ config.ORGANIZATION_NAME }}</title>
-    {% load compress %}
-    {% load staticfiles %}
-    {% compress css %}
-    <link rel="stylesheet" href="{% static 'libs/bootstrap/dist/css/bootstrap.min.css' %}" type="text/css" />
-    <link rel="stylesheet" href="{% static 'libs/fontawesome/css/font-awesome.min.css' %}" type="text/css" />
-    <link rel="stylesheet" href="{% static 'libs/flag-icon-css/css/flag-icon.min.css' %}" type="text/css" />
-    <link rel="stylesheet" href="{% static 'less/base.less' %}" type="text/less" />
-    {% endcompress %}
-    {% compress js %}
-    <script type="text/javascript" src="{% static 'libs/jquery/dist/jquery.js' %}"></script>
-    <script type="text/javascript" src="{% static 'libs/bootstrap/dist/js/bootstrap.js' %}"></script>
-    {% endcompress %}
-    {% block head %}{% endblock %}
-  </head>
-  <body {% block bodyattrs %}{% endblock %}>
-    {% include "core/blocks/header.html" %}
-    <div class="container-fluid main-container">
-      {% block content %}
-      {% endblock %}
-    </div>
-    {% include "core/blocks/footer.html" %}
-  </body>
-</html>
--- a/core/templates/core/blocks/footer.haml
+++ b/core/templates/core/blocks/footer.haml
- load i18n
-
-#footer.container-fluid
-  - blocktrans
-    Memopol is free software
-    <a href="https://gitorious.org/memopol2-0">released</a>
-    under the terms of the
-    <a href="http://www.gnu.org/licenses/agpl.html">GNU aGPLV3+</a>
-  %a{href: "https://projets.lqdn.fr/projects/mempol"}
-    - trans "Bug tracking system"
-  %a{href: "https://projets.lqdn.fr/projects/mempol/issues/new"}
-    - trans "Report a bug"
-  %br
-  {% trans "Provided by" %} <a href="http://www.laquadrature.net/">La Quadrature du Net</a>
--- a/core/templates/core/blocks/header.haml
+++ b/core/templates/core/blocks/header.haml
-/ -load memopol_tags cache
- load i18n
- load cache
- load staticfiles
-
-#header.container-fluid
-  %a{href: "/", id: 'logo'}
-    %img{src: '{% static "images/logo.png" %}'}
-  %h1
-    %a#header_banner{href: "/"}
-      -trans "Political Memory"
-  %p.organization
-    =config.ORGANIZATION_NAME
-
-#nav.container-fluid
-  -include "core/blocks/navigation.html"
--- a/core/templates/core/blocks/navigation.haml
+++ b/core/templates/core/blocks/navigation.haml
-%ul.nav
-  %li
-    %a{href: "{% url 'legislature:representative-index' %}"}
-      Representatives
-  %li
-    %a{href: "{% url 'legislature:group-index' kind='country' %}"}
-      Countries
-  %li
-    %a{href: "{% url 'legislature:group-index' kind='group' %}"}
-      Parties
-  %li
-    %a{href: "{% url 'legislature:group-index' kind='delegation' %}"}
-      Delegations
-  %li
-    %a{href: "{% url 'legislature:group-index' kind='committee' %}"}
-      Committees
-
-%ul.nav
-  %li
-    %a{href: "{% url 'votes:dossier-index' %}"}
-      Votes
--- a/core/templates/core/blocks/pagination.haml
+++ b/core/templates/core/blocks/pagination.haml
-.pagination-block
-
-  %nav
-    %ul.pagination.pagination-sm
-      - if page.has_previous
-        %li
-          %a{'href': '?={queries.urlencode}&page=={page.previous_page_number}',
-             'aria-label': 'Previous'}
-            <i aria-hidden="true" class="fa fa-chevron-left"></i>
-
-      - for p in page.pages
-        - if p
-          - if p == page.number
-            %li.active
-              %a{'href': ''}
-                {{ p }}
-          - else
-            %li
-              %a{'href': '?={queries.urlencode}&page=={p}'}
-                {{ p }}
-        - else
-          %li.disabled
-            %a{'href': ''}
-              …
-
-      - if page.has_next
-        %li
-          %a{'href': '?={queries.urlencode}&page=={page.next_page_number}',
-             'aria-label': 'Next'}
-            <i aria-hidden="true" class="fa fa-chevron-right"></i>
-
-  %div.count
-    Number of results : {{ paginator.count }}
-    %br
-    Number of displayed results :
-    {{ paginator.per_page }}
-    (
-    - for limit in pagination_limits
-      %a{'href': '?limit={{ limit }}'}
-        {{ limit }}
-      - if not forloop.last
-        \/
-    )
-    - include 'core/blocks/grid-list.html'
--- a/core/templates/core/home.haml
+++ b/core/templates/core/home.haml
- extends "base.html"
- block content
-  .row
-    .col-md-8
-      %p
-        Memopol is reachable only in <b>reduced functionality mode</b>.
-        By the way, you could access to
-        <a href="{% url 'legislature:representative-index' %}">the list of MEPs</a>.
-      %p
-        You can help on building the new Memopol by <a href="https://wiki.laquadrature.net/Projects/Memopol/Roadmap">coding, translating, de    signing, funding, etc...</a>.
-    .col-md-4
-      .panel.panel-default
-        .panel-body
-          %p
-            <a href="http://memopol.org">Memopol Blog</a> is available as well as the new
-            <a href="http://git.laquadrature.net/memopol/memopol/issues">
-            bugtracking system</a>
-          %h3
-            What is memopol?
-          %p
-            Political  Memory is a tool designed by La Quadrature du Net to help
-            European citizens to reach members of European Parliament (MEPs) and
-            track their voting records on issues related to fundamental
-            freedoms online. <em><a href="">More...</a></em>
--- a/core/utils.py
+++ b/core/utils.py
-from __future__ import absolute_import
-
-from pure_pagination import EmptyPage
-from pure_pagination import Paginator
-from django.shortcuts import render
-
-
-def create_child_instance_from_parent(child_cls, parent_instance):
-    """
-    Create a child model instance from a parent instance
-    """
-    parent_cls = parent_instance.__class__
-    field = child_cls._meta.get_ancestor_link(parent_cls).column
-
-    child_instance = child_cls(**{
-        field: parent_instance.pk
-    })
-
-    child_instance.__dict__.update(parent_instance.__dict__)
-    child_instance.save()
-    return child_instance
-
-
-def render_paginate_list(request, object_list, template_name):
-    """
-    Render a paginated list of representatives
-    """
-    pagination_limits = (10, 20, 50, 100)
-    num_by_page = request.GET.get('limit', '30')
-    num_by_page = int(num_by_page) if num_by_page.isdigit() else 1
-    paginator = Paginator(object_list, num_by_page)
-    number = request.GET.get('page', '1')
-    number = int(number) if number.isdigit() else 1
-
-    try:
-        page = paginator.page(number)
-    except EmptyPage:
-        page = paginator.page(paginator.num_pages)
-
-    context = {}
-    context['paginator'] = paginator
-    context['page'] = page
-    context['object_list'] = context['page'].object_list
-    context['pagination_limits'] = pagination_limits
-
-    return render(
-        request,
-        template_name,
-        context
-    )
--- a/core/views.py
+++ b/core/views.py
-# coding: utf-8
-
-# This file is part of memopol.
-#
-# memopol is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of
-# the License, or any later version.
-#
-# memopol is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU General Affero Public
-# License along with django-representatives.
-# If not, see <http://www.gnu.org/licenses/>.
-#
-# Copyright (C) 2015 Arnaud Fabre <af@laquadrature.net>
-
-from django.views.generic.base import TemplateView
-
-
-class HomeView(TemplateView):
-    template_name = "core/home.html"
--- a/db.sqlite
+++ b/db.sqlite
--- a/docs/administration.rst
+++ b/docs/administration.rst
@@ -10,7 +10,7 @@ Authentication in the admin backend
 If you haven't already, create a super-administrator
 account with command::

-    ./manage.py createsuperuser
+    memopol createsuperuser

 Then use this account to authenticate in the
 administration backend located in ``/admin``.

--- a/docs/api.rst
+++ b/docs/api.rst
+API Documentation
+~~~~~~~~~~~~~~~~~
+
+Memopol publishes its data with a read-only REST API using `Django REST
+Framework <http://www.django-rest-framework.org/>`_.  The API is browseable and
+partially self-documented, from the ``/api`` URL.
+
+The goal of this documentation is to plug the holes in the
+automatically-generated documentation.
+
+Overview
+========
+
+The API homepage resides at the ``/api/`` URL. Each model is accessible from two
+URLs:
+
+* ``/api/<model>/`` for a (filterable) list of objects
+* ``/api/<model>/<pk>/`` to get a detailed view of a specific object
+
+When accessing the API from a web browser, data should be displayed in a
+user-friendly paginated interface, and relation fields are links to the
+corresponding API URLs (if that's not the case, you can force it by appending
+``format=api`` to the querystring).  To get raw JSON data instead, either pass
+an ``Accept: application/json`` request header or append ``format=json`` to the
+querystring.
+
+Object lists may be filtered using ``field=value`` in the querystring ; related
+fields may be used in filters with double underscores (for example
+``representative__name=John``).  Alternative operators may be available for
+some fields (for example ``score__gte=100``), amongst which:
+
+* ``lte``: matches values that are lower than or equal to the specified value
+* ``gte``: matches values greater than equal to or the specified value
+* ``icontains``: matches values that contain the specified value, ignoring case
+
+Note that fields that are made available for filtering depend on each model;
+the same goes for alternative operators (by default only exact match is
+available).
+
+Finally, note that this documentation does not show the data schema.  Feel free
+to browse the API yourself to find out how data is published.
+
+Representatives API
+===================
+
+This API provides views on representatives and mandates.
+
+Representatives
+---------------
+
+The ``/api/representatives/[<pk>/]`` endpoints give access to representatives.
+The following fields are available for filtering:
+
+* ``active``: True or False
+* ``slug`` (alt.: icontains)
+* ``id``
+* ``remote_id``
+* ``first_name``
+* ``last_name`` (alt.: icontains)
+* ``gender``: M or F
+* ``birth_place``
+* ``birth_date`` (alt.: gte, lte)
+* ``search``: searches in the ``first_name``, ``last_name`` and ``slug`` fields
+
+Mandates
+--------
+
+The ``/api/mandates/[<pk>/]`` endpoints give access to mandates. The following
+fields are available for filtering:
+
+* ``id``
+* ``group__name`` (alt.: icontains)
+* ``group__abbreviation``
+* ``search``: searches in the ``group__name`` and ``group_abbreviation`` fields
+
+Constituency
+------------
+
+The ``/api/constituencies/[<pk>/]`` endpoints give access to constituencies.
+
+Groups
+------
+
+The ``/api/groups/[<pk>/]`` endpoints give access to groups.
+
+Representatives-votes API
+=========================
+
+This API provides views on dossiers, proposals and votes.
+
+Dossiers
+--------
+
+The ``/api/dossiers/[<pk>/]`` endpoints give access to dossiers. The following
+fields are available for filtering:
+
+* ``fingerprint``
+* ``title`` (alt.: icontains)
+* ``reference`` (alt.: icontains)
+* ``search``: searches in the ``fingerprint``, ``title``, ``reference``,
+  ``text`` and ``proposals__title`` fields
+
+Proposals
+---------
+
+The ``/api/proposals/[<pk>/]`` endpoints give access to proposals. The following
+fields are available for filtering:
+
+* ``fingerprint``
+* ``dossier__fingerprint``
+* ``title`` (alt.: icontains)
+* ``description`` (alt.: icontains)
+* ``reference`` (alt.: icontains)
+* ``datetime`` (alt.: gte, lte)
+* ``kind``
+* ``search``: searches in the ``fingerprint``, ``title``,
+  ``dossier__fingerprint``, ``dossier__title`` and ``dossier__reference`` fields
+
+Votes
+-----
+
+The ``/api/votes/[<pk>/]`` endpoints give access to votes. The following fields
+are available for filtering:
+
+* ``proposal__fingerprint``
+* ``position``
+* ``representative_name`` (alt.: icontains)
+* ``representative``
+
+
+Memopol API
+===========
+
+This API provides views on recommendations and representative scores.
+
+Recommendations
+---------------
+
+The ``/api/recommendations/[<pk>/]`` endpoints give access to recommendations.
+The following fields are available for filtering:
+
+* ``id``
+* ``recommendation``
+* ``title`` (alt.: icontains)
+* ``description`` (alt.: icontains)
+* ``weight`` (alt.: gte, lte)
+* ``search``: searches in the ``title`` and ``description`` fields
+
+Vote Scores
+-----------
+
+The ``/api/vote_scores/[<pk>/]`` endpoints give access to scored votes; that is,
+representative votes with their contribution to the representative score. Only
+votes that match a recommendation are visible using this endpoint.  The
+following fields are available for filtering:
+
+* ``representative``
+* ``proposal``
+* ``proposal__dossier``
+
+Dossier Scores
+--------------
+
+The ``/api/dossier_scores/[<pk>/]`` endpoints give access to dossier scores;
+that is, the contribution of each dossier on a representative score. Only
+dossiers with recommendations are visible using this endpoint.  The following
+fields are available for filtering:
+
+* ``dossier``
+* ``representative``
+* ``score`` (alt.: gte, lte)
+
+Representative scores
+---------------------
+
+The ``/api/scores/[<pk>/]`` endpoints give access to total scores for each
+representative. The following fields are available for filtering:
+
+* ``representative``
+* ``score`` (alt.: gte, lte)
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -113,7 +113,7 @@ html_theme = 'alabaster'
 #html_theme_options = {}

 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+html_theme_path = ['../memopol_env/local/lib/python2.7/site-packages']

 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".

--- a/docs/deploy-custom.rst
+++ b/docs/deploy-custom.rst
+Deployment on a custom machine
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Prerequisites
+=============
+
+You will need the following:
+
+* A clone of the git repository at ``https://git.laquadrature.net/memopol/memopol.git``
+* A dedicated python virtualenv
+* a PostgreSQL>=9.2 database (not necessarily on the same machine)
+* a WSGI-capable web server
+
+Setup environment
+=================
+
+Set the following environment variables::
+
+    DJANGO_SETTINGS_MODULE=memopol.settings
+
+Customize settings
+==================
+
+Create a copy of the example local settings file::
+
+    $ cp src/memopol/local_settings.py.example src/memopol/local_settings.py
+
+Edit ``src/memopol/local_settings.py`` to set directories, database settings and
+allowed hosts.  Setup your WSGI server to serve:
+
+* Static files from the directory specified in the ``PUBLIC_DIR`` setting to the
+  ``/static`` URL
+* The ``src/memopol/wsgi.py`` WSGI application
+
+Initial memopol setup
+=====================
+
+From the repository root, install python dependencies (you may want to do that
+in a virtualenv)::
+
+    $ pip install -U pip setuptools
+    $ pip install -Ue .
+
+Install client libraries::
+
+    $ src/memopol/bin/install_client_deps.sh
+
+Setup the database schema::
+
+    $ memopol migrate --noinput
+
+Collect static files::
+
+    $ memopol collectstatic --noinput
+
+Memopol should be ready to go.
+
+Updating
+========
+
+To update simply pull the repository and run setup commands again::
+
+    $ git pull
+    $ pip install -Ue .
+    $ src/memopol/bin/install_client_deps.sh
+    $ memopol migrate --noinput
+    $ memopol collectstatic --noinput
+
+Data provisionning
+==================
+
+Set up two cron jobs:
+
+* One to update data from parliaments, that runs ``bin/update_all``.  This
+  script takes quite some time to run, so you should schedule it once every
+  night for example
+* One to refresh scores,  that runs ``memopol refresh_scores``.  This one
+  runs quite quickly (a few seconds), you may want to run it after the update
+  job has completed (but you can run it more often).
+
+Ensure that cron jobs get the same environment as the application.
+
+
+Continue to :doc:`administration`.
--- a/docs/deployment.rst
+++ b/docs/deployment.rst
-Deployment on OpenShift
-~~~~~~~~~~~~~~~~~~~~~~~
-
-OpenShift is an Open-Source Platform-as-a-Service software by Red Hat. It is
-also available in its hosted version known as "OpenShift Online" and the first
-three websites ("gears") are free.
-
-Clone the repository
-====================
-
-You should fork the project on github and use the fork's clone url. For the
-sake of the demo, we'll use the main repository URL::
-
-    $ git clone https://github.com/political-memory/political_memory.git
-    Cloning into 'political_memory'...
-    remote: Counting objects: 2516, done.
-    remote: Compressing objects: 100% (109/109), done.
-    remote: Total 2516 (delta 44), reused 0 (delta 0), pack-reused 2402
-    Receiving objects: 100% (2516/2516), 4.40 MiB | 79.00 KiB/s, done.
-    Resolving deltas: 100% (1103/1103), done.
-    Checking connectivity... done.
-
-    $ cd political_memory/
-
-Create your own branch, ie::
-
-    $ git checkout -b yourbranch origin/pr
-    Branch yourbranch set up to track remote branch pr from origin.
-    Switched to a new branch 'yourbranch'
-
-Create an app on OpenShift
-==========================
-
-To deploy the website, use a command like::
-
-    $ rhc app-create \
-        python-2.7 \
-        cron-1.4 \
-        postgresql-9.2 \
-        -a yourappname \
-        -e OPENSHIFT_PYTHON_WSGI_APPLICATION=memopol/wsgi.py \
-        --from-code https://github.com/political-memory/political_memory.git \
-        --no-git
-
-This should create an app on openshift. Other commands would deploy it at once
-but in this tutorial we're going to see how to manage it partly manually for
-development.
-
-Add the git remote created by OpenShift
-=======================================
-
-Add the git remote openshift created for you, you can see it with
-``rhc app-show``, ie.::
-
-    $ rhc app-show -a yourappname
-    [snip]
-    Git URL:         ssh://569f5cf500045f6a1839a0a4@yourappname-yourdomain.rhcloud.com/~/git/yourappname.git/
-    Initial Git URL: https://github.com/political-memory/political_memory.git
-    SSH:             569f5cf500045f6a1839a0a4@yourappname-yourdomain.rhcloud.com
-    [snip]
-
-    $ git remote add oo_yourappname ssh://569f5cf500045f6a1839a0a4@yourappname-yourdomain.rhcloud.com/~/git/yourappname.git/
-
-Activate OpenShift's git post-recieve hook
-==========================================
-
-Activate OpenShift's post-receive hook on your branch::
-
-    $ rhc app-configure -a yourappname --deployment-branch yourbranch
-
-Deploy your branch
-==================
-
-OpenShift will deploy when it receives commits on the deployment branch, to
-deploy just do::
-
-    $ git push oo_yourappname yourbranch
-
-If something goes wrong and you want to retry, use the ``rhc app-deploy``
-command, ie::
-
-    $ rhc app-deploy yourbranch -a yourappname
-
-Data provisionning
-==================
-
-To fill up the representatives database table, either wait for the cron script
-to be executed, either do it manually::
-
-    $ rhc ssh -a yourappname 'cd app-root/repo/ && bin/update_all'
-
-OpenShift is fun, login with ssh and look around if you're curious, you'll be
-able to recreate your app without much effort if you break it anyway.
-
-Continue to :doc:`administration`.
No results found