Merge branch 'master' into 'search_update'

# Conflicts:
#   setup.py

.gitlab-ci.yml

 variables:
-        PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
 
 cache:
-        paths:
-                - "$CI_PROJECT_DIR/pip-cache"
-        key: "$CI_PROJECT_ID"
-job test:
-        script:
-                - tox
-        stage: test
-        tags: [memopol]
-job install:
-        script:
-                - source /srv/memopol/memopol_env/bin/activate
-                - cd /srv/memopol/memopol_env/src/memopol/docs
-                - make html
-                - cd /srv/memopol/memopol_env/src/memopol
-                - git fetch origin
-                - git reset --hard origin/master
-                - find . -name '*.pyc' -delete
-                - pip install -Ue .
-                - src/memopol/bin/install_client_deps.sh
-                - memopol migrate --noinput
-                - memopol collectstatic --noinput
-                - touch /srv/memopol/ready
-        stage: deploy
-        tags: [memopol]
-        environment: production
-        only:
-                - master
+    paths:
+        - "$CI_PROJECT_DIR/pip-cache"
+    key: "$CI_PROJECT_ID"
+
+stages:
+    - test
+    - deploy
+
+before_script:
+    - pip install --user tox
+    - export PATH="$HOME/.local/bin:$PATH"
+
+unit tests:
+    stage: test
+    script: tox -e py27
+    tags: [memopol]
+
+qa checks:
+    stage: test
+    script: tox -e qa
+    tags: [memopol]
+
+docs build:
+    stage: test
+    script: git diff --raw HEAD^1 | grep docs || tox -e docs
+    tags: [memopol]
+
+docs deploy:
+    stage: deploy
+    script:
+        - source /srv/memopol/memopol_env/bin/activate
+        - cd /srv/memopol/memopol_env/src/memopol/docs
+        - make html
+    tags: [memopol]
+    environment: production
+    only:
+        - master
+
+django deploy:
+    stage: deploy
+    script:
+        - source /srv/memopol/memopol_env/bin/activate
+        - cd /srv/memopol/memopol_env/src/memopol
+        - git fetch origin
+        - git reset --hard origin/master
+        - find . -name '*.pyc' -delete
+        - pip install -Ue .
+        - src/memopol/bin/install_client_deps.sh
+        - memopol migrate --noinput
+        - memopol collectstatic --noinput
+        - touch /srv/memopol/ready
+    tags: [memopol]
+    environment: production
+    only:
+        - master

bin/dev.sh

@@ -11,24 +11,27 @@ REPOROOT="$( readlink -m "${BASH_SOURCE[0]}"/../..)"
 ALIASROOT=$REPOROOT"/.memopol.alias"
 echo $ALIASROOT
 echo "Create a dedicated alias file in $ALIASROOT"
-echo "alias memopol-code=\"cd $REPOROOT && source $REPOROOT/memopol_env/bin/activate && DJANGO_DEBUG=True\"" > $ALIASROOT
+echo "alias memopol-code=\"cd $REPOROOT && source $REPOROOT/memopol_env/bin/activate && export DJANGO_DEBUG=True\"" > $ALIASROOT
 echo "alias memopol-launch=\"memopol-code && memopol runserver\"" >> $ALIASROOT
-echo "alias memopol-update-all=\"memopol-code && bin/update-all\"" >> $ALIASROOT
+echo "alias memopol-update-all=\"memopol-code && bin/update_all\"" >> $ALIASROOT
 echo "alias memopol-refresh-scores=\"memopol-code && memopol refresh_scores\"" >> $ALIASROOT
 
-if [ $SHELL = "/bin/bash" ]
-then
+case $SHELL in
+*/bash)
 	echo "Bash detected"
 	echo "Update $HOME/.bashrc file"
-  RCSHELL="$HOME/.bashrc"
-elif [ $SHELL = "/bin/zsh" ]
-then
-  echo "Zsh detected"
-  echo "Update $HOME/.zshrc file"
-  RCSHELL="$HOME/.zshrc"
-else
-	echo "SHELL don't supported.  Try using BASH or ZSH, or manually."
-fi
+	RCSHELL="$HOME/.bashrc"
+	;;
+*/zsh)
+	echo "Zsh detected"
+	echo "Update $HOME/.zshrc file"
+	RCSHELL="$HOME/.zshrc"
+	;;
+*)
+	echo "SHELL not supported.  Try using BASH or ZSH, or set alias manually."
+	RCSHELL="/dev/null"
+	;;
+esac
 
 echo "source $ALIASROOT" >> $RCSHELL
 source $ALIASROOT
@@ -37,5 +40,5 @@ source $ALIASROOT
 echo -e "You can use the following aliases :\n"
 echo -e "\t memopol-code : Go into the repository and activate the virtualenv"
 echo -e "\t memopol-launch : Run the development server"
-echo -e "\t memopoll-update-all : Get all the production data"
+echo -e "\t memopol-update-all : Get all the production data"
 echo -e "\t memopol-refresh-scores : Refresh all scores"

docs/conf.py

@@ -113,7 +113,7 @@ html_theme = 'alabaster'
 #html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+html_theme_path = ['../memopol_env/local/lib/python2.7/site-packages']
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".

docs/development.rst

 Local development tutorial
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warn:: I reverse-engineered this from the source code I inherited, I might
-          not be doing the right way nor be able to defend all of technical
-          decisions.
-
 This tutorial drives through a local installation of the project for
 development on Linux. It requires git, a fairly recent version of python2,
 virtualenv and PostgreSQL.
 
-Quickstart
-==========
+Setup the database
+==================
+
+Memopol requires PostgreSQL 9.1 or higher.  It used to run with SQLite, too, but
+that is no longer the case. It is better to install and configure you're local
+PostgreSQL server before starting to install Memopol.
+
+On Debian
+---------
+
+To setup you're PostgreSQL database on a debian stable distribution, you can use
+the package manager apt::
+
+  $ apt install postgresql postgresql-server-dev-9.X
+
+Then you need to create the 'memopol'  user and the 'memopol' database::
+
+  # To have a root access on Postgres, you need to connect as user 'postgres'
+  $ su - postgres
+  $ psql -c "create user memopol with password 'memopol';"
+  $ psql -c "alter role memopol with createdb;"
+  $ psql -c "create database memopol with owner memopol;"
+  $ exit
+
+You're database is now setup for Memopol. You can now launch the 'quickstart.sh'
+script to automatically install all the components or do it manually.
+
+In General
+----------
+
+Make sure the corresponding user and database exist on your system; the user
+will need the 'createdb' permission in order to be able to run tests.  To create
+them, you may use the following commands::
+
+    $ psql -c "create user memopol with password 'memopol';" -U postgres
+    $ psql -c "alter role memopol with createdb;" -U postgres
+    $ psql -c "create database memopol with owner memopol;" -U postgres
+
+
+Automatic Install
+=================
 
 There is a quickstart script used and tested manually by some of the
 developers. Feel free to try it, but don't worry if it doesn't work for you
@@ -23,14 +58,42 @@ Here's how to try it::
     $ cd memopol
     $ source bin/quickstart.sh
 
+At this point, you should now run the development server and access to Memopol::
+
+  $ memopol runserver
+
 If you want more control or if it doesn't work for you, then follow the steps
 below or have a look at what the quickstart script does.
 
-.. note::
-  If you are using python 3 on your system, the quickstart script will install
-  the virtualenv with this version. You'll need to remove the directory
-  'memopol_env' and recreate it with the python version 2.7 in parameter
-  'virtualenv -p /usr/bin/python2.7 memopol_env'
+Development helper
+===================
+
+You can run the script 'bin/dev.sh' to automaticaly setup some aliases. It works
+only with Bash and Zsh.
+
+The script build a custom file named '.memopol.alias' at the root of the project
+containing all the aliases for memopol. All the path to the project are build
+automatically. A single line is added to your '$HOME/.bashrc' or '$HOME/.zshrc'
+to source the aliases.
+
+After execute 'bin/dev.sh' you should close the current terminal and open
+another one to have access to the aliases.
+
+There is a quick list of available aliases::
+
+  memopol-code : Go into the repository and activate virtualenv and
+  set Django in debug mode
+  memopol-launch : Run the development server echo
+  memopol-update-all : Get all the production data
+  memopol-refresh-scores : Refresh all scores
+
+.. warning:: If you are using multiple setup of Memopol, it is not recommended to
+  use this script.
+
+If you need to change the location of the project, you should remove this line
+from your .bashrc or .zshrc::
+
+  source $PATH_TO_THE_PROJECT/.memopol.alias
 
 Make a virtual environment
 ==========================
@@ -111,31 +174,11 @@ Activate ``DJANGO_DEBUG``
 =========================
 
 ``DEBUG`` is disabled by default, the development server
-won't run properly by default thnen, to enable it export
+won't run properly by default then, to enable it export
 the ``DJANGO_DEBUG`` variable in the current shell::
 
     $ export DJANGO_DEBUG=True
 
-Setup the database
-==================
-
-Memopol requires PostgreSQL 9.1 or higher.  It used to run with SQLite, too, but
-that is no longer the case.  Memopol uses the following environment variables
-for database access:
-
-* ``MEMOPOL_DB_NAME`` (defaults to 'memopol')
-* ``MEMOPOL_DB_USER`` (defaults to 'memopol')
-* ``MEMOPOL_DB_PASSWORD`` (defaults to 'memopol')
-* ``MEMOPOL_DB_HOST`` (defaults to 'localhost')
-* ``MEMOPOL_DB_PORT`` (defaults to '5432')
-
-Make sure the corresponding user and database exist on your system; the user
-will need the 'createdb' permission in order to be able to run tests.  To create
-them, you may use the following commands::
-
-    $ psql -c "create user memopol with password 'memopol';" -U postgres
-    $ psql -c "alter role memopol with createdb;" -U postgres
-    $ psql -c "create database memopol with owner memopol;" -U postgres
 
 Database migrations
 ===================

docs/index.rst

@@ -19,6 +19,7 @@ Contents:
    api
    development
    hacker
+   setup_solr
 
 Indices and tables
 ==================
@@ -26,4 +27,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/setup_solr.rst

+Setup Solr with Memopol
+=======================
+
+Solr is used to perform search in the data. Currently, it is used only for
+reprensatative search autocomplete.
+
+This howto will be based on the current stable version of Debian (jessie).
+
+Installation on Debian Stable
+-----------------------------
+
+Solr is in the official repository of debian, you can install it launching::
+
+    # apt install solr-tomcat/stable
+    Package: solr-tomcat
+    Source: lucene-solr
+    Version: 3.6.2+dfsg-5
+    Installed-Size: 65,5 kB
+    Maintainer: Debian Java Maintainers <pkg-java-maintainers@lists.alioth.debian.org>
+    Depends: solr-common (= 3.6.2+dfsg-5), tomcat7
+    Conflicts: solr-jetty, solr-tomcat6
+    Homepage: http://lucene.apache.org
+    Section: java
+    Priority: optional
+    Download-Size: 8 598 B
+    APT-Sources: http://ftp.fr.debian.org/debian/ jessie/main amd64 Packages
+    Description: Enterprise search server based on Lucene3 - Tomcat integration
+     Solr is an open source enterprise search server based on the Lucene
+     Java search library, with XML/HTTP and JSON APIs, hit highlighting,
+     faceted search, caching, replication, and a web administration
+     interface. It runs in a Java servlet container such as Tomcat.
+     .
+     This package provides the Tomcat integration files for Solr.
+
+
+By default, the solr server is listen on localhost:8080. Memopol is configured
+to use this addess by default, in production. If you install a newer version of
+Solr (6.5.1 is the latest release), you should update the settings.py::
+
+    HAYSTACK_CONNECTIONS = {
+        'default': {
+            'ENGINE': 'haystack.backends.solr_backend.SolrEngine',
+            'URL': 'http://127.0.0.1:8080/solr',
+        },
+    }
+
+Configure Solr
+---------------
+
+You can use the default Solr configuration to use it, You just need to generate
+the schema.xml file and copy it in `/etc/solr/`. To generate the schema.xml
+file::
+
+    $ memopol build_solr_schema > schmema.xml
+
+    ## Copy the schema.xml file into /etc/solr/conf
+    $ sudo cp schema.xml /etc/solr/conf/
+
+    ## Restart tomcat
+    $ sudo /etc/init.d/tomcat7 restart
+
+Last step, you need to build the index by using::
+
+    $ memopol rebuild_index
+
+The solr is now setup and production ready.
+
+Populate data in Solr
+---------------------
+
+Django-haystack plugin offers two ways to populate data. The first way is by
+using a cron job to update the index, the other way is to use `Dango signals to
+update / delete datas <https://django-haystack.readthedocs.io/en/v2.6.0/signal_processors.html>`_.
+
+For now, Haystack is not configured to be used with Django signals. It is 
+necessary to add a cron job to update the index ::
+
+    $ memopol update_index
+
+`More informations about cron with Solr <http://django-haystack.readthedocs.io/en/v2.6.0/searchindex_api.html?highlight=cron#keeping-the-index-fresh>`_

setup.py

@@ -33,6 +33,7 @@ setup(name='political-memory',
         'django-haystack==2.6.0',
         'pysolr==3.6.0',
         'Whoosh==2.7.4'
+	    'alabaster==0.7.10',
     ],
     extras_require={
         # Full version hardcode for testing dependencies so that