diff --git a/.travis.yml b/.travis.yml index 31051d3bd..ac9ad42ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,78 +1,78 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. notifications: email: false services: - mysql - redis - mongodb sudo: false language: python cache: - apt - pip env: - REQUIREMENTS=lowest REXTRAS=docs - REQUIREMENTS=release REXTRAS=docs - REQUIREMENTS=devel REXTRAS=docs python: - "2.7" addons: apt: packages: - apache2 - git - liblzma-dev - nodejs - poppler-utils before_install: - "travis_retry pip install --upgrade pip" - "travis_retry pip install mock" - "python requirements.py --extras=$REXTRAS --level=min > .travis-lowest-requirements.txt" - "python requirements.py --extras=$REXTRAS --level=pypi > .travis-release-requirements.txt" - "python requirements.py --extras=$REXTRAS --level=dev > .travis-devel-requirements.txt" install: - "travis_retry pip install unittest2" - "travis_retry pip install -r .travis-$REQUIREMENTS-requirements.txt --allow-all-external" - "travis_retry pip install -e .[$REXTRAS] --process-dependency-links" - "python setup.py compile_catalog" - "npm update" - - "npm install --silent -g bower less clean-css uglify-js requirejs" + - "npm install --silent -g bower@1.7.9 less@2.7.1 clean-css@3.4.12 uglify-js@2.6.2 requirejs@2.2.0" # All the step below this points are solely for test purposes, don't use them # to setup your invenio installation. Please do RTFM instead (INSTALL.rst). - "./scripts/setup_devmode.sh" before_script: - "inveniomanage apache create-config" - "inveniomanage database init --yes-i-know || echo ':('" - "inveniomanage database create --quiet || echo ':('" script: - "travis_retry sphinx-build -qnNW docs docs/_build/html" - "python setup.py test" diff --git a/INSTALL.rst b/INSTALL.rst index cc4197f89..65b210684 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,587 +1,587 @@ Invenio installation ==================== 1. About -------- This document specifies how to quickly install Invenio v2.0.0 for the first time. See RELEASE-NOTES if you are upgrading from a previous Invenio release. 2. Prerequisites ---------------- Here is the software you need to have around before you start installing Invenio for development. Unix-like operating system. The main development and production platforms for Invenio at CERN are GNU/Linux distributions Debian, Gentoo, Scientific Linux (RHEL-based), Ubuntu, but we also develop on Mac OS X. Basically any Unix system supporting the software listed below should do. 2.1. Debian / Ubuntu LTS ~~~~~~~~~~~~~~~~~~~~~~~~ If you are using Ubuntu 13.10 or later, then you can install Invenio by following this tutorial. **Note:** the recommended Python version is 2.7.5+ .. code-block:: console $ python --version Python 2.7.5+ $ sudo apt-get update $ sudo apt-get install build-essential git redis-server \ libmysqlclient-dev libxml2-dev libxslt-dev \ libjpeg-dev libfreetype6-dev libtiff-dev \ libffi-dev libssl-dev \ software-properties-common python-dev \ virtualenvwrapper subversion $ sudo pip install -U virtualenvwrapper pip $ source .bashrc 2.1.1. MySQL ++++++++++++ MySQL Server will ask you for a password, you will need it later and we will refer to it as ``$MYSQL_ROOT``. .. code-block:: console $ sudo apt-get install mysql-server 2.1.2. Node.js ++++++++++++++ `node.js `_ and `npm `_ can be installed using NodeSource's script. .. code-block:: console $ curl -sL https://deb.nodesource.com/setup_0.10 | sudo bash - $ sudo apt-get install nodejs 2.2. Centos / RHEL ~~~~~~~~~~~~~~~~~~ If you are using Redhat, Centos or Scientific Linux this will setup everything you need. We are assuming that sudo has been installed and configured nicely. .. code-block:: console $ python --version 2.6.6 $ sudo yum update $ sudo rpm -Uvh http://mirror.switch.ch/ftp/mirror/epel/6/i386/epel-release-6-8.noarch.rpm $ sudo yum -q -y groupinstall "Development Tools" $ sudo yum install git wget redis python-devel \ mysql-devel libxml2-devel libxslt-devel \ python-pip python-virtualenvwrapper $ sudo service redis start $ sudo pip install -U virtualenvwrapper pip $ source /usr/bin/virtualenvwrapper.sh 2.2.1. MySQL ++++++++++++ Setting up MySQL Server requires you to give some credentials for the root user. You will need the root password later on and we will refer to it as ``$MYSQL_ROOT``. If you are on CentOS 7, the mysql-server package is not available in the default repository. First we need to add the official YUM repository provided by Oracle. The YUM repository configuration can be downloaded from the `MySQL website `_. Choose the desired distribution (Red Hat Enterprise Linux 7 / Oracle Linux 7 for CentOS 7) and click Download. The download link can be retrieved without registering for an Oracle account. Locate the "No thanks, just start my download" link and pass the link URL as a parameter to rpm. .. code-block:: console # only needed with CentOS version >= 7 $ sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release... # for every CentOS version $ sudo yum install mysql-server $ sudo service mysqld status mysqld is stopped $ sudo service mysqld start $ sudo mysql_secure_installation # follow the instructions 2.2.2. Node.js ++++++++++++++ Node.js requires a bit more manual work to install it from the sources. We are following the tutorial: `digital ocean: tutorial on how to install node.js on centor `_ .. code-block:: console $ mkdir opt $ cd opt $ wget http://nodejs.org/dist/v0.10.29/node-v0.10.29.tar.gz $ tar xvf node-v0.10.29.tar.gz $ cd node-v0.10.29 $ ./configure $ make $ sudo make install $ node --version v0.10.29 $ npm --version 1.4.14 .. _OS X: 2.3. OS X ~~~~~~~~~~ The steps below can be used to install Invenio on a machine running OS X 10.9 or later. First, we need to install the `Homebrew `_ package manager. Follow the installation procedure by running following command: .. code-block:: console $ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" You need to check that ``/usr/local/bin`` occurs before the ``/usr/bin``, otherwise you can try following commands: .. code-block:: console $ echo export PATH="/usr/local/bin:$PATH" >> ~/.bash_profile $ source ~/.bash_profile (to reload the profile) Next, you should check if everything is up-to-date! .. code-block:: console $ brew update $ brew doctor $ brew upgrade Now, it is time to start installing the prerequisites. .. code-block:: console $ brew install python --framework --universal $ pip install virtualenv $ pip install virtualenvwrapper # edit the Bash profile $ $EDITOR ~/.bash_profile Add the following to the file you have opened and paste the following lines. .. code-block:: text export WORKON_HOME=~/.virtualenvs source /usr/local/bin/virtualenvwrapper.sh Save the file and reload it by typing: .. code-block:: console $ source ~/.bash_profile and continue with the installation of prerequisite packages: .. code-block:: console $ brew install redis $ brew install mongodb .. note:: See `MySQL on OS X`_ for installing ``mysql``. In order to install ``libxml2`` and ``libxslt`` packages run: .. code-block:: console $ brew install automake autoconf libtool libxml2 libxslt $ brew link --force libxml2 libxslt The following might not be necessary but is good to have for completeness. .. code-block:: console $ brew install libjpeg libtiff freetype libffi $ pip install -I pillow Install ``node`` by following `Node on OS X`_ For ``bower``, type: .. code-block:: console - $ npm install -g bower + $ npm install -g bower@1.7.9 After the configuration section install the following(required for the assets): .. code-block:: console - $ npm install -g less clean-css requirejs uglify-js + $ npm install -g less@2.7.1 clean-css@3.4.12 uglify-js@2.6.2 requirejs@2.2.0 See the following sections `Installation`_ , `Configuration`_ and `Development`_ The commands for ``OS X`` are the same as in ``Linux``. .. note:: When initializing the database, type: .. code-block:: console $ inveniomanage database init --user=root --yes-i-know (because we have no root password) .. note:: For developers, honcho is recommended and will make your life easier because it launches all the servers together as it finds the ``Procfile``. .. _MySQL on OS X: 2.3.1. MySQL ++++++++++++ We will install MySQL but without a root password. It should be easy to set the root password once you are connected in MySQL. .. code-block:: console $ brew install mysql $ unset TMPDIR $ mysql_install_db --verbose --user=`whoami` \ --basedir="$(brew --prefix mysql)" \ --datadir=/usr/local/var/mysql \ --tmpdir=/tmp You can start, stop, or restart MySQL server by typing: .. code-block:: console $ mysql.server (start | stop | restart) .. _Node on OS X: 2.3.2. Node.js ++++++++++++++ Install ``node`` by typing: .. code-block:: console $ brew install node 2.4. Extra tools ~~~~~~~~~~~~~~~~ 2.4.1. Bower ++++++++++++ Bower is used to manage the static assets such as JavaScript libraries (e.g., jQuery) and CSS stylesheets (e.g., Bootstrap). It's much easier to install them globally (``-g``) but you're free to choose your preferred way. .. code-block:: console # global installation $ sudo su -c "npm install -g bower" # user installation $ npm install bower 2.4.2 ``git-new-workdir`` (optional) ++++++++++++++++++++++++++++++++++++ For the rest of the tutorial you may want to use ``git-new-workdir``. It's a tool that will let you working on the same repository from different locations. Just like you would do with subversion branches. .. code-block:: console $ mkdir -p $HOME/bin $ which git-new-workdir || { \ wget https://raw.github.com/git/git/master/contrib/workdir/git-new-workdir \ -O $HOME/bin/git-new-workdir; chmod +x $HOME/bin/git-new-workdir; } **NOTE:** Check that ``~/bin`` is in your ``$PATH``. .. code-block:: console $ export PATH+=:$HOME/bin 3. Quick instructions for the impatient Invenio developer --------------------------------------------------------- This installation process is tailored for running the development version of Invenio, check out the :py:ref:`overlay` documentation for the production setup. .. _Installation: 3.1. Installation ~~~~~~~~~~~~~~~~~ The first step of the installation is to download the development version of Invenio and the Invenio Demosite. This development is done in the ``master`` branch. .. code-block:: console $ mkdir -p $HOME/src $ cd $HOME/src/ $ export BRANCH=master $ git clone --branch $BRANCH git://github.com/inveniosoftware/invenio.git $ git clone --branch $BRANCH git://github.com/inveniosoftware/invenio-demosite.git We recommend to work using `virtual environments `_ so packages are installed locally and it will make your life easier. ``(invenio)$`` tells your that the *invenio* environment is the active one. .. code-block:: console $ # choose an unique name for your virtual environment $ export VENAME=invenio $ mkvirtualenv $VENAME (invenio)$ # we are in the invenio environment now and (invenio)$ # can leave it using the deactivate command. (invenio)$ deactivate $ # Now join it back, recreating it would fail. $ workon invenio (invenio)$ # That's all there is to know about it. Let's put Invenio and the Invenio Demosite in the environment just created. .. code-block:: console (invenio)$ cdvirtualenv (invenio)$ mkdir src (invenio)$ cd src (invenio)$ git-new-workdir $HOME/src/invenio/ invenio $BRANCH (invenio)$ git-new-workdir $HOME/src/invenio-demosite/ invenio-demosite $BRANCH If you don't want to use the ``git-new-workdir`` way, you can either: - create a symbolic link, - or clone the repository directly into the virtualenv. Installing Invenio. .. code-block:: console (invenio)$ cdvirtualenv src/invenio (invenio)$ pip install -e .[development] Some modules may require specific dependencies listed as ``extras``. Pick the ones you need. E.g. to add `images` support, we can do as follow: .. code-block:: console (invenio)$ pip install -e .[img] If the Invenio is installed in development mode, you will need to compile the translations manually. .. code-block:: console (invenio)$ python setup.py compile_catalog .. note:: Translation catalog is compiled automatically if you install using `python setup.py install`. Installing Invenio Demosite. ``exists-action i`` stands for `ignore`, it means that it'll will skip any previous installation found. Because the Invenio Demosite depends on Invenio, it would have tried to reinstall it without this option. If you omit it, ``pip`` will ask you what action you want to take. .. code-block:: console (invenio)$ cdvirtualenv src/invenio-demosite (invenio)$ pip install -r requirements.txt --exists-action i Installing the required assets (JavaScript, CSS, etc.) via bower. The file ``.bowerrc`` is configuring where bower will download the files and ``bower.json`` what libraries to download. .. code-block:: console (invenio)$ inveniomanage bower -i bower-base.json > bower.json Generates or update bower.json for you. (invenio)$ cat .bowerrc { "directory": "invenio_demosite/base/static/vendors" } (invenio)$ bower install (invenio)$ ls invenio_demosite/base/static/vendors bootstrap ckeditor hogan jquery jquery-tokeninput jquery-ui plupload ... We recommend you to only alter ``bower-base.json`` and regenerate ``bower.json`` with it as needed. The :py:class:`invenio.ext.assets.commands.BowerCommand` is aggregating all the dependencies defined by each bundle. The last step, which is very important will be to collect all the assets, but it will be done after the configuration step. .. _Configuration: 3.2. Configuration ~~~~~~~~~~~~~~~~~~ Generate the secret key for your installation. .. code-block:: console (invenio)$ inveniomanage config create secret-key If you are planning to develop locally in multiple environments please run the following commands. .. code-block:: console (invenio)$ # sanitaze for usage as database name and user (invenio)$ export SAFE_NAME=`echo $VENAME$BRANCH | sed -e 's/[^A-Za-z0-9]//g'` (invenio)$ inveniomanage config set CFG_EMAIL_BACKEND flask_email.backends.console.Mail (invenio)$ inveniomanage config set CFG_BIBSCHED_PROCESS_USER $USER (invenio)$ inveniomanage config set CFG_DATABASE_NAME $SAFE_NAME (invenio)$ inveniomanage config set CFG_DATABASE_USER $SAFE_NAME (invenio)$ inveniomanage config set CFG_SITE_URL http://localhost:4000 (invenio)$ inveniomanage config set CFG_SITE_SECURE_URL http://localhost:4000 Assets in non-development mode may be combined and minified using various filters (see :ref:`ext_assets`). We need to set the path to the binaries if they are not in the environment ``$PATH`` already. .. code-block:: console # Local installation (using package.json) (invenio)$ cdvirtualenv src/invenio (invenio)$ npm install (invenio)$ inveniomanage config set LESS_BIN `find $PWD/node_modules -iname lessc | head -1` (invenio)$ inveniomanage config set CLEANCSS_BIN `find $PWD/node_modules -iname cleancss | head -1` (invenio)$ inveniomanage config set REQUIREJS_BIN `find $PWD/node_modules -iname r.js | head -1` (invenio)$ inveniomanage config set UGLIFYJS_BIN `find $PWD/node_modules -iname uglifyjs | head -1` All the assets that are spread among every invenio module or external libraries will be collected into the instance directory. By default, it create copies of the original files. As a developer you may want to have symbolic links instead. .. code-block:: console # Developer only (invenio)$ inveniomanage config set COLLECT_STORAGE flask_collect.storage.link (invenio)$ inveniomanage collect ... Done collecting. (invenio)$ cdvirtualenv var/invenio.base-instance/static (invenio)$ ls -l css js vendors ... .. _Development: 3.3. Development ~~~~~~~~~~~~~~~~ Once you have everything installed, you can create the database and populate it with demo records. .. code-block:: console (invenio)$ inveniomanage database init --user=root --password=$MYSQL_ROOT --yes-i-know (invenio)$ inveniomanage database create Now you should be able to run the development server. Invenio uses `Celery `_ and `Redis `_ which must be running alongside with the web server. .. code-block:: console # make sure that redis is running $ sudo service redis-server status redis-server is running # or start it with start $ sudo service redis-server start # launch celery $ workon invenio (invenio)$ celery worker -E -A invenio.celery.celery --workdir=$VIRTUAL_ENV # in a new terminal $ workon invenio (invenio)$ inveniomanage runserver * Running on http://0.0.0.0:4000/ * Restarting with reloader .. note:: On OS X, the command ``service`` might not be found when starting the redis server. To run redis, just type: .. code-block:: console $ redis-server **Troubleshooting:** As a developer, you may want to use the provided ``Procfile`` with `honcho `_. It starts all the services at once with nice colors. By default, it also runs `flower `_ which offers a web interface to monitor the *Celery* tasks. .. code-block:: console (invenio)$ pip install honcho flower (invenio)$ cdvirtualenv src/invenio (invenio)$ honcho start When all the servers are running, it is possible to upload the demo records. .. code-block:: console $ # in a new terminal $ workon invenio (invenio)$ inveniomanage demosite populate --packages=invenio_demosite.base And you may now open your favourite web browser on `http://0.0.0.0:4000/ `_ Optionally, if you are using Bash shell completion, then you may want to register python argcomplete for inveniomanage. .. code-block:: bash eval "$(register-python-argcomplete inveniomanage)" 4. Final words -------------- Happy hacking and thanks for flying Invenio. - Invenio Development Team diff --git a/docs/getting-started/overlay.rst b/docs/getting-started/overlay.rst index 843e92427..9b6176d65 100644 --- a/docs/getting-started/overlay.rst +++ b/docs/getting-started/overlay.rst @@ -1,629 +1,629 @@ .. This file is part of Invenio - Copyright (C) 2014 CERN. + Copyright (C) 2014, 2016 CERN. Invenio is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. Invenio is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Invenio; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. .. _overlay: ================================== How to create an Invenio overlay ================================== .. admonition:: TODO The following items are still missing from this document, feel free to contribute to it. - celery/redis/honcho/flower - populate the data Thanks What is an overlay ================== Invenio is a library that enable the creation of an digital library but it has to be coupled with an overlay. The overlay will contain your configuration options, your desired look and feel, the extra invenio modules you've developed and are using. Creating your first Invenio overlay =================================== If you've already setup the :ref:`developer's environement invenio ` itself, it will feel familiar. We are reproducing here the steps for Ubuntu LTS. Other distributions may be found in the previous link. The global setup ---------------- Some softwares and libraries are required to work on your overlay. It's mostly Python, MySQL, Redis as well as XML, XSLT and graphical libraries. Node.js is solely required for development purposes. .. code-block:: console $ python --version Python 2.7.5+ $ sudo apt-get update $ sudo apt-get install build-essential redis-server \ libmysqlclient-dev libxml2-dev libxslt-dev \ libjpeg-dev libfreetype6-dev libtiff-dev \ software-properties-common python-dev \ virtualenvwrapper $ sudo pip install -U virtualenvwrapper pip $ source .bashrc # Install MySQL server, and keep the root password somewhere safe. $ sudo apt-get install mysql-server # Install Node.js $ curl -sL https://deb.nodesource.com/setup_0.10 | sudo bash - $ sudo apt-get install nodejs The virtual environment ----------------------- Python development usually recommends to work within a ``virtualenv``, which creates an isolated environment where the libraries required by one will not intervene with the system ones or the ones of another system. We are using ``virtualenvwrapper`` but nothing prevents your from directly using ``virtualenv`` you are familiar with it. .. code-block:: console $ mkvirtualenv myoverlay (myoverlay)$ # we are in your overlay environment now and (myoverlay)$ # can leave it using the deactivate command. (myoverlay)$ deactivate $ # Now join it back, recreating it would fail. $ workon myoverlay (myoverlay)$ # That's all there is to know about it. The base of the overlay ----------------------- Let's dive in. .. code-block:: console $ workon myoverlay (myoverlay)$ cdvirtualenv (myoverlay)$ mkdir -p src/myoverlay (myoverlay)$ cd src/myoverlay (myoverlay)$ edit setup.py The ``setup.py`` file contains the definition of a python package. Having one means you can rely on existing tools like ``pip`` to package, install, deploy it later on. Here is its minimal content. .. code-block:: python from setuptools import setup from setuptools import setup, find_packages packages = find_packages() setup( name="My Overlay", version="0.1.dev0", url="http://invenio-software.org/", author="Invenio Software", author_email="invenio@invenio-software.org", description="My first overlay", packages=packages, install_requires=[ "Invenio>=2" ], entry_points={ "invenio.config": ["myoverlay = myoverlay.config"] } ) Now we can install it in editable mode (``-e``) meaning you don't have to reinstall it after each change .. code-block:: console (myoverlay)$ pip install -e . This way will use the latest invenio version published on PyPI, as a developer, you may want to use the development version of Invenio. To do so, create a file called ``requirements.txt``. .. code-block:: text git://github.com/inveniosoftware/invenio@pu#egg=Invenio-dev -e . The installation process changes a little bit. It still contains the ``-e .`` command we used before but specify the github version of Invenio instead of the PyPI one. .. code-block:: console (myoverlay)$ pip install -r requirements.txt Configuration ============= As you've seen above, we defined an entry_point for ``myoverlay.config``. It points to a module that will contain our configuration. So create your application. .. code-block:: text src/ │ ├ myoverlay/ │ │ │ ├ base/ │ │ │ │ │ └ __init__.py │ │ │ ├ __init__.py │ └ config.py │ ├ requirements.txt └ setup.py Put the required configuration into ``config.py``. .. code-block:: python CFG_SITE_LANGS = ["en"] CFG_SITE_NAME = "My Overlay" CFG_SITE_NAME_INTL = { "en": CFG_SITE_NAME } PACKAGES = [ "myoverlay.base", "invenio.modules.*", "invenio.base", ] try: from myoverlay.instance_config import * except ImportError: pass Sensitive configuration ----------------------- Other configuration elements like database username and password or the website url should not be put here as this file is not specific to the installation and may be put under a version control system such as Git or Subversion. The configuration can be handled via the `inveniomanage` command line interface (or by editing the `invenio.cfg` file in the instance folder and reloading the application). .. code-block:: console (myoverlay)$ inveniomanage config set create secret-key # MySQL configuration (myoverlay)$ inveniomanage config set CFG_DATABASE_NAME mysql-database (myoverlay)$ inveniomanage config set CFG_DATABASE_USER mysql-user # HOST configuration (for redirects, etc.) (myoverlay)$ inveniomanage config set CFG_SITE_URL http://invenio.example.com (myoverlay)$ inveniomanage config set CFG_SITE_SECURE_URL https://invenio.example.com (myoverlay)$ inveniomanage config set DEBUG True (myoverlay)$ inveniomanage config set ASSETS_DEBUG True (myoverlay)$ inveniomanage config set LESS_RUN_IN_DEBUG False Database setup -------------- .. code-block:: console (invenio)$ inveniomanage database init --user=root --password=$MYSQL_ROOT --yes-i-know ... >>> Database has been installed. (invenio)$ inveniomanage database create ... >>> Tables filled successfully. Assets ------ Most of the JavaScript and CSS libraries used are not bundled with invenio itself and needs to be downloaded via `bower `_. Bower is configured using two files: - `.bowerrc`: tells where the assets are downloaded - `bower.json`: lists the dependencies to be downloaded .. code-block:: json { "directory": "myoverlay/base/static/vendors" } The ``bower.json`` can be automagically generated. .. code-block:: console - $ sudo su -c "npm install -g bower less clean-css requirejs uglify-js" + $ sudo su -c "npm install -g bower@1.7.9 less@2.7.1 clean-css@3.4.12 uglify-js@2.6.2 requirejs@2.2.0" (myoverlay)$ inveniomanage bower > bower.json (myoverlay)$ bower install For invenio to see the static files from the ``myoverlay.base`` module, it needs to declare a Flask blueprint. Create the following file: ``myoverlay/base/views.py``. .. code-block:: python from flask import Blueprint blueprint = Blueprint( "myoverlay", __name__, url_prefix="/", template_folder="templates", # where your custom templates will go static_folder="static" # where the assets go ) The assets will now be collected into the instance static folder from your overlay, invenio itself and every libraries it uses. .. code-block:: console (myoverlay)$ inveniomanage collect Running ======= .. code-block:: console (myoverlay)$ inveniomanage runserver Translations ============ Invenio comes with full internationalization and localization support based on `Babel `_ library and `Flask-Babel `_. All strings you want to translate in your overlay have to be marked with ``_()``. When you have all strings properly marked, it is time to prepare catalog that contains all these strings for tranlations to desired languages. Configuration ------------- First of all, you have to get into the source folder of your overlay and create a configuration file for *Babel*. .. code-block:: ini [python: **.py] encoding = utf-8 [jinja2: **/templates/**] encoding = utf-8 extensions = jinja2.ext.autoescape.jinja2.ext.with_ Save it as ``babel.cfg`` next to your ``setup.py``. Before we run the extraction tool we need to add section to configure translation directory to ``setup.cfg``. .. code-block:: ini [compile_catalog] directory = myoverlay/base/translations/ [extract_messages] output-file = myoverlay/base/translations/myoverlay.pot [init_catalog] input-file = myoverlay/base/translations/myoverlay.pot output-dir = myoverlay/base/translations/ [update_catalog] input-file = myoverlay/base/translations/myoverlay.pot output-dir = myoverlay/base/translations/ Message Extraction ------------------ Then it’s time to run the Babel string extraction with given configuration: .. code-block:: console (myoverlay)$ python setup.py extract_messages Create Catalog for New Language ------------------------------- Once all translatable strings are extracted, one need to prepare catalogs for new languages. Following example shows how to prepare new catalog for French in PO (Portable Object) format. .. code-block:: console (myoverlay)$ python setup.py init_catalog -l fr Now edit the ``myoverlay/base/translations/fr/LC_MESSAGES/messages.po`` file as needed. Compiling Catalog ----------------- Next step is to prepare MO (Machine Object) files in the format which is defined by the GNU `gettext `_ tools and the GNU `translation project `_. To compile the translations for use, pybabel integration with distutils helps again: .. code-block:: console (myoverlay)$ python setup.py compile_catalog If you install Invenio in development mode you must compile catalog also from the Invenio directory project. .. note:: You should tell git to ignore your compliled translation by running: .. code-block:: console $ echo \*.mo >> .gitignore Updating Strings ---------------- It is pretty common that your strings in the code will change over the time. Pybabel provides support for updating the translation catalog with new strings or changing existing ones. What do you have to do? Create a new ``myoverlay.pot`` like above and then let pybabel merge the changes: .. code-block:: console $ python setup.py update_catalog Deployment ========== Deploying Invenio is almost a piece of cake using `Fabric `_. The following step are inspired by the Flask documentation: `Deploying with Fabric `_ Prerequisites ------------- First, you need a server with remote access (SSH), where you've installed all the python dependencies (e.g. ``build-essentials``, ``python-dev``, ``libmysqlclient-dev``, etc.). Install `fabric` locally, .. code-block:: console $ pip install fabric and create a boilerplate ``fabfile.py``: .. code-block:: python import json from fabric.api import * from fabric.utils import error from fabric.contrib.files import exists env.user = 'invenio' # remote username env.directory = '/home/invenio/www' # remote directory env.hosts = ['yourserver'] # list of servers Preparing the tarball --------------------- Before deploying anything, we need to locally prepare the python package to be installed. Thanks to our ``setup.py`` file, it's very simple. Beforehand, we have to generate the static assets into our static folder. By doing so, it's not required to install anything related to node.js on your server (no ``bower``, ``less``, ``uglifyjs``, etc.). .. code-block:: python @task def pack(): """Create a new source distribution as tarball.""" with open(".bowerrc") as fp: bower = json.load(fp) local("inveniomanage assets build --directory {directory}/gen" .format(**bower)) return local("python setup.py sdist --formats=gztar", capture=False) \ .succeeded Try it: .. code-block:: console $ fab pack ... Done $ ls dist/ My-Overlay-0.1.dev0.tar.gz This is the package that will be installed on your server. Creating the virtual environement --------------------------------- We love virtual environments. We recommend you to install each version into its own virtual env enabling quick rollbacks. .. code-block:: python @task def create_virtualenv(): """Create the virtualenv.""" package = local("python setup.py --fullname", capture=True).strip() venv = "{0}/{1}".format(env.directory, package) with cd(env.directory): if exists(package): return error("This version {0} is already installed." .format(package)) return run("virtualenv {0}".format(package)).succeeded Installing the package ---------------------- We can now upload the local tarball into the virtualenv, and install everything there. .. code-block:: python @task def install(): """Install package.""" package = local("python setup.py --fullname", capture=True).strip() venv = "{0}/{1}".format(env.directory, package) if not exists(venv): return error("Meh? I need a virtualenv first.") # Upload the package and put it into our virtualenv. put("dist/{0}.tar.gz".format(package), "/tmp/app.tgz") run("mkdir -p {0}/src".format(venv)) with cd("{0}/src".format(venv)): run("tar xzf /tmp/app.tgz") run("rm -rf /tmp/app.tgz") # Jump into the virtualenv and install stuff with cd("{0}/src/{1}".format(venv, package)): success = run("{0}/bin/python setup.py install".format(venv) if success: # post install run("{0}/bin/inveniomanage collect".format(venv)) return success Combining all the three steps: .. code-block:: console $ fab pack virtualenv install Configuration ------------- The setup doesn't have the ``invenio.cfg`` file that is generated via ``inveniomanage config``. You should do so manually. Running the server ------------------ uWSGI is super simple and neat, all you need is two files. In the example below, we've installed two versions of our overlay and a symbolic link is pointing to the one we want to run. .. code-block:: console $ ls www/ current -> My-Overlay-0.1 My-Overlay-0.1.dev1 My-Overlay-0.1.dev2 My-Overlay-0.1 wsgi.py uwsgi.ini Let's create the ``wsgi.py`` file. .. code-block:: python from invenio.base.factory import create_wsgi_app application = create_wsgi_app() And the µWSGI configuration: .. code-block:: python [uwsgi] http = 0.0.0.0:4000 master = true processes = 4 die-on-term = true vaccum = true chdir = %d virtualenv = %d/current/ module = wsgi:application touch-reload = %d/wsgi.py Let's run it. .. code-block:: console $ pip install uwsgi $ uwsgi --ini uwsgi.ini # or in daemon mode $ uwsgi -d uwsgi.log --ini uwsgi.ini If the new version causes troubles, going back to the old one is as fast as changing the symbolic link and restarting the WSGI server. .. code-block:: console $ rm current $ ln -s My-Overlay-0.1.dev1 current $ touch wsgi.py Dealing with versions --------------------- One good idea is to use symlink to point to your current virtualenv and run your overlay from there. Doing that via Fabric is left as an exercise to the reader. When installing a new version, copying the ``invenio.cfg`` file over is the only requirements. Restarting the WSGI server is usually done by ``touch``-ing the ``wsgi.py`` file. diff --git a/setup.py b/setup.py index 2ee019c51..8ad133545 100644 --- a/setup.py +++ b/setup.py @@ -1,357 +1,359 @@ # -*- coding: utf-8 -*- # # This file is part of Invenio. # Copyright (C) 2013, 2014, 2015, 2016 CERN. # # Invenio is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # Invenio is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Invenio; if not, write to the Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """Invenio is a framework for digital libraries and data repositories. Invenio enables you to run your own digital library or document repository on the web. Invenio covers all aspects of digital library management, from document ingestion, through classification, indexing and further processing, to curation, archiving, and dissemination. The flexibility and performance of Invenio make it a comprehensive solution for management of document repositories of moderate to large sizes (several millions of records). Links ----- * `website `_ * `documentation `_ * `development `_ """ import os import sys from distutils.command.build import build from setuptools import find_packages, setup from setuptools.command.install_lib import install_lib class _build(build): # noqa """Compile catalog before building the package.""" sub_commands = [('compile_catalog', None)] + build.sub_commands class _install_lib(install_lib): # noqa """Custom install_lib command.""" def run(self): """Compile catalog before running installation command.""" install_lib.run(self) self.run_command('compile_catalog') install_requires = [ "alembic>=0.6.6,<0.7", "Babel>=1.3,<2.0", "bagit>=1.5.1", "BeautifulSoup>=3.2.1", "BeautifulSoup4>=4.3.2", - "celery>=3.1.8", + "celery>=3.1.8,<4.0.0", # Cerberus>=0.7.1 api changes and is not yet supported "Cerberus>=0.7,<0.7.1", "chardet>=2.3.0", "click>=4.1,<7.0", "datacite>=0.1.0,<0.2.0", "dictdiffer>=0.0.3", "feedparser>=5.1", "fixture>=1.5", - "Flask>=0.10.1", + "Flask>=0.10.1,<0.11", "Flask-Admin>=1.0.9,<1.2", - "Flask-Assets>=0.10", + "Flask-Assets>=0.10,<0.11", "Flask-Babel>=0.9,<0.10", "Flask-Breadcrumbs>=0.2", "Flask-Cache>=0.12", - "Flask-Collect>=1.1.1", + "Flask-Collect>=1.1.1,<1.2.0", "Flask-Email>=1.4.4", "Flask-Gravatar>=0.4.2", "Flask-Login>=0.2.7,<0.3.0", "Flask-Menu>=0.2", "Flask-OAuthlib>=0.6.0,<0.7", # quick fix for issue #2158 "Flask-Principal>=0.4", "Flask-Registry>=0.2", "Flask-RESTful>=0.2.12,<0.3.0", "Flask-Script>=2.0.5", "Flask-SQLAlchemy>=2.0,<2.2", "Flask-WTF>=0.12,<0.13", - "fs>=0.4", + "fs>=0.4,<1.0.0", "intbitset>=2.0", "invenio-client>=0.1.0", "jellyfish>=0.3.2", "Jinja2>=2.7", "libmagic>=1.0", "lxml>=3.3,<3.5", "mechanize>=0.2.5", "mistune>=0.4.1", "msgpack-python>=0.3", "MySQL-python>=1.2.5", "numpy>=1.7", "nydus>=0.10.8", # FIXME new oauthlib release after 0.7.2 has some compatible problems with # the used Flask-Oauthlib version. "oauthlib==0.7.2", # pyparsing>=2.0.2 has a new api and is not compatible yet "pyparsing>=2.0.1,<2.0.2", "python-twitter>=2.0", "pyPDF>=1.13", "pyPDF2>=1.17", "PyLD>=0.5.2", "pyStemmer>=1.3", "python-dateutil>=1.5", "python-magic>=0.4.6", "pytz>=2014.1", "rauth>=0.7.0", "raven>=5.0.0", "rdflib>=4.1.2", "redis>=2.8.0", "reportlab>=2.7,<3.2", "requests>=2.3,<2.4", - "setuptools>=12.0", + "setuptools>=18.5", "six>=1.7.2", - "Sphinx>=1.3", + "Sphinx>=1.3,<1.4", "SQLAlchemy>=0.9.9,<1.0", "SQLAlchemy-Utils[encrypted]>=0.28.2,<0.30", "unidecode>=0.04.1", "workflow>=1.2.0,<2.0.0", + "Werkzeug>=0.9.6,<0.11.0", "WTForms>=2.0.2,<2.2", "WTForms-Alchemy>=0.13.0,<0.13.1", "WTForms-Components>=0.9.6,<0.10", "WTForms-SQLAlchemy>=0.1,<0.2" ] extras_require = { "docs": [ - "sphinx_rtd_theme>=0.1.7" + "sphinx_rtd_theme>=0.1.7,<0.1.8", + "docutils>=0.12,<0.13", ], "development": [ "Flask-DebugToolbar==0.9.0", "watchdog==0.8.3", ], "dropbox": [ "dropbox>=2.1.0" ], "elasticsearch": [ "pyelasticsearch>=0.6.1" ], "googledrive": [ "google-api-python-client>=1.2", "apiclient>=1.0.0", "oauth2client>=1.4.0", "urllib3>=1.8.3" ], "iiif": [ "Flask-IIIF>=0.2.0", ], "img": [ "qrcode>=5.1", "Pillow>=2.7.0,<3.0.0" ], "mongo": [ "pymongo>=3.0" ], "misc": [ # was requirements-extras "gnuplot-py==1.8", "flake8>=2.0.0", # extra=kwalitee? "pychecker==0.8.19", # extra=kwalitee? "pylint>=1.4.0", # extra=kwalitee? "nosexcover>=1.0.0", # test? "python-onedrive>=15.0.0", # extra=cloud? "python-openid>=2.2.0", # extra=sso? ], "mixer": [ "mixer>=5.1.0", ], "sso": [ "Flask-SSO>=0.2" ], "postgresql": [ "psycopg2>=2.5", ], # Alternative XML parser # # For pyRXP, the version on PyPI many not be the right one. # # $ pip install # > https://www.reportlab.com/ftp/pyRXP-1.16-daily-unix.tar.gz#egg=pyRXP # "pyrxp": [ # Any other versions are not supported. "pyRXP==1.16-daily-unix" ], "rabbitmq": [ "amqp>=1.4.5", ], "github": [ "github3.py>=0.9" ], } extras_require["docs"] += extras_require["elasticsearch"] extras_require["docs"] += extras_require["img"] extras_require["docs"] += extras_require["mongo"] extras_require["docs"] += extras_require["sso"] extras_require["docs"] += extras_require["github"] # FIXME extras_require["docs"] += extras_require["dropbox"] # FIXME extras_require["docs"] += extras_require["googledrive"] tests_require = [ "httpretty>=0.8.4", "Flask-Testing>=0.4.1", "mock>=1.0.0", "nose>=1.3.0", "selenium>=2.45.0", "unittest2>=0.5", ] setup_requires = [ 'Babel>=1.3,<2.0', ] # Add `tests` dependencies to `extras_require` so that developers # could install test dependencies also with pip: extras_require["tests"] = tests_require # Compatibility with Python 2.6 if sys.version_info < (2, 7): install_requires += [ "argparse>=1.3.0", "importlib>=1.0.0" ] # Get the version string. Cannot be done with import! g = {} with open(os.path.join("invenio", "version.py"), "rt") as fp: exec(fp.read(), g) version = g["__version__"] packages = find_packages(exclude=['docs']) packages.append('invenio_docs') setup( name='invenio', version=version, url='https://github.com/inveniosoftware/invenio', license='GPLv2', author='CERN', author_email='info@invenio-software.org', description='Invenio digital library framework', long_description=__doc__, packages=packages, package_dir={'invenio_docs': 'docs'}, include_package_data=True, zip_safe=False, platforms='any', entry_points={ 'console_scripts': [ 'inveniomanage = invenio.base.manage:main', 'plotextractor = invenio.utils.scripts.plotextractor:main', # Legacy 'alertengine = invenio.legacy.webalert.scripts.alertengine:main', 'batchuploader = invenio.legacy.bibupload.scripts.batchuploader:main', 'bibcheck = invenio.legacy.bibcheck.scripts.bibcheck:main', 'bibcircd = invenio.legacy.bibcirculation.scripts.bibcircd:main', 'bibauthorid = ' ' invenio.legacy.bibauthorid.scripts.bibauthorid:main', 'bibcatalog = invenio.legacy.bibcatalog.scripts.bibcatalog:main', 'bibclassify = invenio.modules.classifier.scripts.classifier:main', 'bibconvert = invenio.legacy.bibconvert.scripts.bibconvert:main', 'bibdocfile = invenio.legacy.bibdocfile.scripts.bibdocfile:main', 'bibedit = invenio.legacy.bibedit.scripts.bibedit:main', 'bibencode = invenio.modules.encoder.scripts.encoder:main', 'bibexport = invenio.legacy.bibexport.scripts.bibexport:main', 'bibindex = invenio.legacy.bibindex.scripts.bibindex:main', 'bibmatch = invenio.legacy.bibmatch.scripts.bibmatch:main', 'bibrank = invenio.legacy.bibrank.scripts.bibrank:main', 'bibrankgkb = invenio.legacy.bibrank.scripts.bibrankgkb:main', 'bibreformat = invenio.legacy.bibformat.scripts.bibreformat:main', 'bibsort = invenio.legacy.bibsort.scripts.bibsort:main', 'bibsched = invenio.legacy.bibsched.scripts.bibsched:main', 'bibstat = invenio.legacy.bibindex.scripts.bibstat:main', 'bibtaskex = invenio.legacy.bibsched.scripts.bibtaskex:main', 'bibtasklet = invenio.legacy.bibsched.scripts.bibtasklet:main', 'bibtex = invenio.modules.sequencegenerator.scripts.bibtex:main', 'bibupload = invenio.legacy.bibupload.scripts.bibupload:main', 'convert_journals = ' ' invenio.legacy.docextract.scripts.convert_journals:main', 'dbexec = invenio.legacy.miscutil.scripts.dbexec:main', 'dbdump = invenio.legacy.miscutil.scripts.dbdump:main', 'docextract = invenio.legacy.docextract.scripts.docextract:main', 'elmsubmit = invenio.legacy.elmsubmit.scripts.elmsubmit:main', 'gotoadmin = invenio.modules.redirector.scripts.redirector:main', 'hepdataharvest = ' ' invenio.utils.hepdata.scripts.hepdataharvest:main', 'inveniocfg = invenio.legacy.inveniocfg:main', 'inveniogc = invenio.legacy.websession.scripts.inveniogc:main', 'inveniounoconv = ' ' invenio.legacy.websubmit.scripts.inveniounoconv:main', 'oaiharvest = invenio.legacy.oaiharvest.scripts.oaiharvest:main', 'oairepositoryupdater = ' ' invenio.legacy.oairepository.scripts.oairepositoryupdater:main', 'arxiv-pdf-checker = invenio.legacy.pdfchecker:main', 'refextract = invenio.legacy.refextract.scripts.refextract:main', 'textmarc2xmlmarc = ' ' invenio.legacy.bibrecord.scripts.textmarc2xmlmarc:main', 'webaccessadmin = ' ' invenio.modules.access.scripts.webaccessadmin:main', 'webauthorprofile = ' ' invenio.legacy.webauthorprofile.scripts.webauthorprofile:main', 'webcoll = invenio.legacy.websearch.scripts.webcoll:main', 'webmessageadmin = ' ' invenio.legacy.webmessage.scripts.webmessageadmin:main', 'webstatadmin = invenio.legacy.webstat.scripts.webstatadmin:main', 'websubmitadmin = ' ' invenio.legacy.websubmit.scripts.websubmitadmin:main', 'xmlmarc2textmarc = ' ' invenio.legacy.bibrecord.scripts.xmlmarc2textmarc:main', 'xmlmarclint = invenio.legacy.bibrecord.scripts.xmlmarclint:main', ], "distutils.commands": [ "inveniomanage = invenio.base.setuptools:InvenioManageCommand", ] }, setup_requires=setup_requires, install_requires=install_requires, extras_require=extras_require, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU General Public License v2' ' or later (GPLv2+)', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', ], test_suite='invenio.testsuite.suite', tests_require=tests_require, cmdclass={ 'build': _build, 'install_lib': _install_lib, }, )