diff --git a/Vagrantfile b/Vagrantfile
index 3a935cded..b65c7285b 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -1,92 +1,96 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2015 CERN.
+# Copyright (C) 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
# This Vagrant configuration is suitable for testing detailed Invenio
# installation instructions as they are described in:
#
# http://pythonhosted.org/invenio/installation/installation-detailed.html
#
# This Vagrant configuration uses separate dedicated VMs for various services in
# order to better emulate production environment conditions. You can install an
# Invenio demo site by running:
#
# laptop> vagrant up
# laptop> vagrant ssh web -c 'source .inveniorc && /vagrant/scripts/install.sh'
# vagrant> curl http://0.0.0.0:5000/records/1
#OS = 'hfm4/centos7'
OS = 'ubuntu/trusty64'
Vagrant.configure("2") do |config|
config.vm.define "web" do |web|
web.vm.box = OS
web.vm.hostname = 'web'
web.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
web.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-web.sh", privileged: false
web.vm.network "forwarded_port", guest: 5000, host: 5000
web.vm.network "private_network", ip: ENV.fetch('INVENIO_WEB_HOST','192.168.50.10')
+ web.vm.provider :virtualbox do |vb|
+ vb.customize ["modifyvm", :id, "--memory", "1024"]
+ vb.customize ["modifyvm", :id, "--cpus", 2]
+ end
end
config.vm.define "postgresql" do |postgresql|
postgresql.vm.box = OS
postgresql.vm.hostname = 'postgresql'
postgresql.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
postgresql.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-postgresql.sh", privileged: false
postgresql.vm.network "private_network", ip: ENV.fetch('INVENIO_POSTGRESQL_HOST','192.168.50.11')
end
config.vm.define "redis" do |redis|
redis.vm.box = OS
redis.vm.hostname = 'redis'
redis.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
redis.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-redis.sh", privileged: false
redis.vm.network "private_network", ip: ENV.fetch('INVENIO_REDIS_HOST','192.168.50.12')
end
config.vm.define "elasticsearch" do |elasticsearch|
elasticsearch.vm.box = OS
elasticsearch.vm.hostname = 'elasticsearch'
elasticsearch.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
elasticsearch.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-elasticsearch.sh", privileged: false
elasticsearch.vm.network "private_network", ip: ENV.fetch('INVENIO_ELASTICSEARCH_HOST','192.168.50.13')
end
config.vm.define "rabbitmq" do |rabbitmq|
rabbitmq.vm.box = OS
rabbitmq.vm.hostname = 'rabbitmq'
rabbitmq.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
rabbitmq.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-rabbitmq.sh", privileged: false
rabbitmq.vm.network "private_network", ip: ENV.fetch('INVENIO_RABBITMQ_HOST','192.168.50.14')
end
config.vm.define "worker" do |worker|
worker.vm.box = OS
worker.vm.hostname = 'worker'
worker.vm.provision "file", source: ".inveniorc", destination: ".inveniorc"
worker.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-worker.sh", privileged: false
worker.vm.network "private_network", ip: ENV.fetch('INVENIO_WORKER_HOST','192.168.50.15')
end
end
diff --git a/docs/installation/installation-detailed.rst b/docs/installation/installation-detailed.rst
index 62d4f33b0..65028a9d9 100644
--- a/docs/installation/installation-detailed.rst
+++ b/docs/installation/installation-detailed.rst
@@ -1,519 +1,520 @@
.. This file is part of Invenio
- Copyright (C) 2014, 2015 CERN.
+ Copyright (C) 2014, 2015, 2016 CERN.
Invenio is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License as
published by the Free Software Foundation; either version 2 of the
License, or (at your option) any later version.
Invenio is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with Invenio; if not, write to the Free Software Foundation, Inc.,
59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
.. _installation_detailed:
Detailed installation guide
===========================
.. admonition:: CAVEAT LECTOR
Invenio v3.0 alpha is a bleeding-edge developer preview version that is
scheduled for public release in Q1/2016.
Introduction
------------
In this installation guide, we'll create an Invenio digital library instance
using a multi-machine setup where separate services (such as the database server
and the web server) run on separate dedicated machines. Such a multi-machine
setup emulates to what one would typically use in production. (However, it is
very well possible to follow this guide and install all the services onto the
same "localhost", if one wants to.)
We'll use six dedicated machines running the following services:
============= ============= ====================
node IP runs
============= ============= ====================
web 192.168.50.10 Invenio web application
postgresql 192.168.50.11 `PostgreSQL `_ database server
redis 192.168.50.12 `Redis `_ caching service
elasticsearch 192.168.50.13 `Elasticsearch `_ information retrieval service
rabbitmq 192.168.50.14 `RabbitMQ `_ messaging service
worker 192.168.50.15 `Celery `_ worker node
============= ============= ====================
The instructions below are tested on Ubuntu 14.04 LTS (Trusty Tahr) and CentOS 7
operating systems. For other operating systems such as Mac OS X, you may want to
check out the "kickstart" set of scripts coming with the Invenio source code
that perform the below-quoted installation steps in an unattended automated way.
Environment variables
---------------------
Let's define some useful environment variables that will describe our Invenio
instance setup:
.. glossary::
INVENIO_WEB_HOST
The IP address of the Web server node.
INVENIO_WEB_INSTANCE
The name of your Invenio instance that will be created. Usually equal to
the name of the Python virtual environment.
INVENIO_WEB_VENV
The name of the Python virtual environment where Invenio will be installed.
Usually equal to the name of the Invenio instance.
INVENIO_USER_EMAIL
The email address of a user account that will be created on the Invenio
instance.
INVENIO_USER_PASS
The password of this Invenio user.
INVENIO_POSTGRESQL_HOST
The IP address of the PostgreSQL database server.
INVENIO_POSTGRESQL_DBNAME
The database name that will hold persistent data of our Invenio instance.
INVENIO_POSTGRESQL_DBUSER
The database user name used to connect to the database server.
INVENIO_POSTGRESQL_DBPASS
The password of this database user.
INVENIO_REDIS_HOST
The IP address af the Redis server.
INVENIO_ELASTICSEARCH_HOST
The IP address of the Elasticsearch information retrieval server.
INVENIO_RABBITMQ_HOST
The IP address of the RabbitMQ messaging server.
INVENIO_WORKER_HOST
The IP address of the Celery worker node.
In our example setup, we shall use:
.. include:: ../../.inveniorc
:start-after: # sphinxdoc-kickstart-configuration-variables-begin
:end-before: # sphinxdoc-kickstart-configuration-variables-end
:literal:
Let us save this configuration in a file called ``.inveniorc`` for future use.
Web
---
The web application node (192.168.50.10) is where the main Invenio application
will be running. We need to provision it with some system dependencies in order
to be able to install various underlying Python and JavaScript libraries.
The web application node can be set up in an automated unattended way by running
the following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-web.sh
Let's see in detail what the web provisioning script does.
First, some useful system tools are installed:
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-useful-system-tools-trusty-begin
:end-before: # sphinxdoc-install-useful-system-tools-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-useful-system-tools-centos7-begin
:end-before: # sphinxdoc-install-useful-system-tools-centos7-end
:literal:
Second, an external Node.js package repository is enabled. We'll be needing to
install and run Npm on the web node later. The Node.js repository is enabled
as follows:
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-add-nodejs-external-repository-trusty-begin
:end-before: # sphinxdoc-add-nodejs-external-repository-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-add-nodejs-external-repository-centos7-begin
:end-before: # sphinxdoc-add-nodejs-external-repository-centos7-end
:literal:
Third, all the common prerequisite software libraries and packages that Invenio
needs are installed:
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-web-common-trusty-begin
:end-before: # sphinxdoc-install-web-common-trusty-end
:literal:
* on CentOS7:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-web-common-centos7-begin
:end-before: # sphinxdoc-install-web-common-centos7-end
:literal:
We want to use PostgreSQL database in this installation example, so we need to
install corresponding libraries too:
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-web-libpostgresql-trusty-begin
:end-before: # sphinxdoc-install-web-libpostgresql-trusty-end
:literal:
* on CentOS7:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-web-libpostgresql-centos7-begin
:end-before: # sphinxdoc-install-web-libpostgresql-centos7-end
:literal:
Fourth, now that Node.js is installed, we can proceed with installing Npm and
associated CSS/JS filter tools. Let's do it globally:
* on either of the operating systems:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-npm-and-css-js-filters-begin
:end-before: # sphinxdoc-install-npm-and-css-js-filters-end
:literal:
Finally, we'll install Python virtual environment wrapper tools and activate
them in the current user shell process:
* on either of the operating systems:
.. include:: ../../scripts/provision-web.sh
:start-after: # sphinxdoc-install-virtualenvwrapper-begin
:end-before: # sphinxdoc-install-virtualenvwrapper-end
:literal:
Database
--------
The database server (192.168.50.11) will hold persistent data of our Invenio
installation, such as bibliographic records or user accounts. Invenio supports
MySQL, PostgreSQL, and SQLite databases. In this tutorial, we shall use
PostgreSQL that is the recommended database platform for Invenio.
The database server node can be set up in an automated unattended way by running
the following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-postgresql.sh
Let's see in detail what the database provisioning script does.
First, we install and configure the database software:
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-postgresql.sh
:start-after: # sphinxdoc-install-postgresql-trusty-begin
:end-before: # sphinxdoc-install-postgresql-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-postgresql.sh
:start-after: # sphinxdoc-install-postgresql-centos7-begin
:end-before: # sphinxdoc-install-postgresql-centos7-end
:literal:
We can now create a new database user with the necessary access permissions on
the new database:
* on either of the operating systems:
.. include:: ../../scripts/provision-postgresql.sh
:start-after: # sphinxdoc-setup-postgresql-access-begin
:end-before: # sphinxdoc-setup-postgresql-access-end
:literal:
Redis
-----
The Redis server (192.168.50.12) is used for various caching needs.
The Redis server can be set up in an automated unattended way by running the
following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-redis.sh
Let's see in detail what the Redis provisioning script does.
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-redis.sh
:start-after: # sphinxdoc-install-redis-trusty-begin
:end-before: # sphinxdoc-install-redis-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-redis.sh
:start-after: # sphinxdoc-install-redis-centos7-begin
:end-before: # sphinxdoc-install-redis-centos7-end
:literal:
Elasticsearch
-------------
The Elasticsearch server (192.168.50.13) is used to index and search
bibliographic records, fulltext documents, and other various interesting
information managed by our Invenio digital library instance.
The Elasticsearch server can be set up in an automated unattended way by running
the following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-elasticsearch.sh
Let's see in detail what the Elasticsearch provisioning script does.
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-elasticsearch.sh
:start-after: # sphinxdoc-install-elasticsearch-trusty-begin
:end-before: # sphinxdoc-install-elasticsearch-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-elasticsearch.sh
:start-after: # sphinxdoc-install-elasticsearch-centos7-begin
:end-before: # sphinxdoc-install-elasticsearch-centos7-end
:literal:
RabbitMQ
--------
The RabbitMQ server (192.168.50.14) is used as a messaging middleware broker.
The RabbitMQ server can be set up in an automated unattended way by running the
following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-rabbitmq.sh
Let's see in detail what the RabbitMQ provisioning script does.
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-rabbitmq.sh
:start-after: # sphinxdoc-install-rabbitmq-trusty-begin
:end-before: # sphinxdoc-install-rabbitmq-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-rabbitmq.sh
:start-after: # sphinxdoc-install-rabbitmq-centos7-begin
:end-before: # sphinxdoc-install-rabbitmq-centos7-end
:literal:
Worker
------
The Celery worker node (192.168.50.15) is used to execute potentially long tasks
in asynchronous manner.
The worker node can be set up in an automated unattended way by running the
following script:
.. code-block:: shell
source .inveniorc
./scripts/provision-worker.sh
Let's see in detail what the worker provisioning script does.
* on Ubuntu 14.04 LTS (Trusty Tahr):
.. include:: ../../scripts/provision-worker.sh
:start-after: # sphinxdoc-install-worker-trusty-begin
:end-before: # sphinxdoc-install-worker-trusty-end
:literal:
* on CentOS 7:
.. include:: ../../scripts/provision-worker.sh
:start-after: # sphinxdoc-install-worker-centos7-begin
:end-before: # sphinxdoc-install-worker-centos7-end
:literal:
Invenio
-------
Now that all the prerequisites have been set up, we can proceed with the
installation of the Invenio itself. The installation is happening on the web
node (192.168.50.10).
The installation of Invenio can be done in an automated unattended way by
running the following script:
.. code-block:: shell
source .inveniorc
./scripts/install.sh
Let's see in detail what the Invenio installation script does.
We start by creating a fresh new Python virtual environment that will hold our
Invenio v3.0 instance:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-create-virtual-environment-begin
:end-before: # sphinxdoc-create-virtual-environment-end
:literal:
We continue by installing Invenio v3.0 base package and most of its available
modules (by using option ``full`` as opposed to using option ``minimal``) from
PyPI:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-install-invenio-full-begin
:end-before: # sphinxdoc-install-invenio-full-end
:literal:
Alternatively, if you want to install the very-bleeding-edge Invenio packages
from GitHub, you can run the following in the Invenio source code directory::
pip install -r requirements-devel.txt
We can now create a new Invenio instance:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-create-instance-begin
:end-before: # sphinxdoc-create-instance-end
:literal:
We can now install the newly created instance by means of provided ``setup.py``
script:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-install-instance-begin
:end-before: # sphinxdoc-install-instance-end
:literal:
Let's briefly customise our instance with respect to the location of the
database server, the Redis server, the Elasticsearch server, and all the other
dependent services in our multi-server environment:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-customise-instance-begin
:end-before: # sphinxdoc-customise-instance-end
:literal:
In the instance folder, we run Npm to install any JavaScript libraries that
Invenio needs:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-run-npm-begin
:end-before: # sphinxdoc-run-npm-end
:literal:
We can now collect and build CSS/JS assets of our Invenio instance:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-collect-and-build-assets-begin
:end-before: # sphinxdoc-collect-and-build-assets-end
:literal:
We proceed by creating a dedicated database that will hold persistent data of
our installation, such as bibliographic records or user accounts. The database
tables can be created as follows:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-create-database-begin
:end-before: # sphinxdoc-create-database-end
:literal:
We continue by creating a user account:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-create-user-account-begin
:end-before: # sphinxdoc-create-user-account-end
:literal:
Let's now start Celery worker that will execute instance tasks:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-start-celery-worker-begin
:end-before: # sphinxdoc-start-celery-worker-end
:literal:
Now that Celery is running, we can populate our Invenio instance with demo
records:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-populate-with-demo-records-begin
:end-before: # sphinxdoc-populate-with-demo-records-end
:literal:
-Let's register persistent identifiers for the uploaded demo records:
+Let's register incremental numerical persistent identifiers for the uploaded
+demo records:
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-register-pid-begin
:end-before: # sphinxdoc-register-pid-end
:literal:
Finally, let's start the web application (in debugging mode):
.. include:: ../../scripts/install.sh
:start-after: # sphinxdoc-start-application-begin
:end-before: # sphinxdoc-start-application-end
:literal:
-We should now see our demo record on the web:
+We should now see our demo records on the web:
.. code-block:: shell
firefox http://localhost:5000/records/1
-and we can access it via REST API:
+and we can access them via REST API:
.. code-block:: shell
curl -i -H "Accept: application/json" \
http://localhost:5000/api/records/1
-We are done! Our first Invenio v3.0 instance is fully up and running.
+We are done! Our first Invenio v3.0 demo instance is fully up and running.
diff --git a/requirements-devel.txt b/requirements-devel.txt
index 42440f13e..6c4e6ebbc 100644
--- a/requirements-devel.txt
+++ b/requirements-devel.txt
@@ -1,42 +1,43 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2015 CERN.
+# Copyright (C) 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
+-e git+https://github.com/inveniosoftware/dojson.git#egg=dojson
-e git+https://github.com/inveniosoftware/invenio-access.git#egg=invenio-access
-e git+https://github.com/inveniosoftware/invenio-accounts.git#egg=invenio-accounts
-e git+https://github.com/inveniosoftware/invenio-assets.git#egg=invenio-assets
-e git+https://github.com/inveniosoftware/invenio-base.git#egg=invenio-base
-e git+https://github.com/inveniosoftware/invenio-celery.git#egg=invenio-celery
-e git+https://github.com/inveniosoftware/invenio-config.git#egg=invenio-config
-e git+https://github.com/inveniosoftware/invenio-db.git#egg=invenio-db[postgresql,versioning]
-e git+https://github.com/inveniosoftware/invenio-i18n.git#egg=invenio-i18n
-e git+https://github.com/inveniosoftware/invenio-logging.git#egg=invenio-logging
-e git+https://github.com/inveniosoftware/invenio-mail.git#egg=invenio-mail
-e git+https://github.com/inveniosoftware/invenio-pidstore.git#egg=invenio-pidstore
-e git+https://github.com/inveniosoftware/invenio-query-parser.git#egg=invenio-query-parser
-e git+https://github.com/inveniosoftware/invenio-records-rest.git#egg=invenio-records-rest
-e git+https://github.com/inveniosoftware/invenio-records-ui.git#egg=invenio-records-ui
-e git+https://github.com/inveniosoftware/invenio-records.git#egg=invenio-records
-e git+https://github.com/inveniosoftware/invenio-rest.git#egg=invenio-rest
-e git+https://github.com/inveniosoftware/invenio-search.git#egg=invenio-search
-e git+https://github.com/inveniosoftware/invenio-theme.git#egg=invenio-theme
diff --git a/scripts/install.sh b/scripts/install.sh
index d29ebca44..b353c2b71 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -1,200 +1,211 @@
#!/usr/bin/env bash
#
# This file is part of Invenio.
-# Copyright (C) 2015 CERN.
+# Copyright (C) 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
# check environment variables:
if [ -v ${INVENIO_WEB_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10"
exit 1
fi
if [ -v ${INVENIO_WEB_INSTANCE} ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_INSTANCE before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_INSTANCE=invenio3"
exit 1
fi
if [ -v ${INVENIO_WEB_VENV} ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_VENV before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_VENV=invenio3"
exit 1
fi
if [ -v ${INVENIO_USER_EMAIL} ]; then
echo "[ERROR] Please set environment variable INVENIO_USER_EMAIL before runnning this script."
echo "[ERROR] Example: export INVENIO_USER_EMAIL=info@inveniosoftware.org"
exit 1
fi
if [ -v ${INVENIO_USER_PASS} ]; then
echo "[ERROR] Please set environment variable INVENIO_USER_PASS before runnning this script."
echo "[ERROR] Example: export INVENIO_USER_PASS=uspass123"
exit 1
fi
if [ -v ${INVENIO_POSTGRESQL_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_POSTGRESQL_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_POSTGRESQL_HOST=192.168.50.11"
exit 1
fi
if [ -v ${INVENIO_POSTGRESQL_DBNAME} ]; then
echo "[ERROR] Please set environment variable INVENIO_POSTGRESQL_DBNAME before runnning this script."
echo "[ERROR] Example: INVENIO_POSTGRESQL_DBNAME=invenio3"
exit 1
fi
if [ -v ${INVENIO_POSTGRESQL_DBUSER} ]; then
echo "[ERROR] Please set environment variable INVENIO_POSTGRESQL_DBUSER before runnning this script."
echo "[ERROR] Example: INVENIO_POSTGRESQL_DBUSER=invenio3"
exit 1
fi
if [ -v ${INVENIO_POSTGRESQL_DBPASS} ]; then
echo "[ERROR] Please set environment variable INVENIO_POSTGRESQL_DBPASS before runnning this script."
echo "[ERROR] Example: INVENIO_POSTGRESQL_DBPASS=dbpass123"
exit 1
fi
if [ -v ${INVENIO_REDIS_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_REDIS_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_REDIS_HOST=192.168.50.12"
exit 1
fi
if [ -v ${INVENIO_ELASTICSEARCH_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_ELASTICSEARCH_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_ELASTICSEARCH_HOST=192.168.50.13"
exit 1
fi
if [ -v ${INVENIO_RABBITMQ_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_RABBITMQ_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_RABBITMQ_HOST=192.168.50.14"
exit 1
fi
if [ -v ${INVENIO_WORKER_HOST} ]; then
echo "[ERROR] Please set environment variable INVENIO_WORKER_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_WORKER_HOST=192.168.50.15"
exit 1
fi
# load virtualenvrapper:
source $(which virtualenvwrapper.sh)
# detect pathname of this script:
scriptpathname=$(cd "$(dirname $0)" && pwd)
# sphinxdoc-create-virtual-environment-begin
mkvirtualenv ${INVENIO_WEB_VENV}
cdvirtualenv
mkdir -p src
cd src
# sphinxdoc-create-virtual-environment-end
# quit on errors and unbound symbols:
set -o errexit
set -o nounset
if [[ "$@" != *"--devel"* ]]; then
# sphinxdoc-install-invenio-full-begin
-# FIXME the next three commands are needed only for invenio<3.0.0a3
+# FIXME the next pip commands are needed only for invenio<3.0.0a3
pip install invenio-db[postgresql] --pre
pip install invenio-access[postgresql] --pre
pip install invenio-search --pre
+pip install dojson --pre
# now we can install full Invenio:
pip install invenio[full] --pre
# sphinxdoc-install-invenio-full-end
else
pip install -r $scriptpathname/../requirements-devel.txt
fi
# sphinxdoc-create-instance-begin
inveniomanage instance create ${INVENIO_WEB_INSTANCE}
# sphinxdoc-create-instance-end
# sphinxdoc-install-instance-begin
cd ${INVENIO_WEB_INSTANCE}
python setup.py install
# sphinxdoc-install-instance-end
# sphinxdoc-customise-instance-begin
mkdir -p ../../var/${INVENIO_WEB_INSTANCE}-instance/
echo "# Database" > ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://${INVENIO_POSTGRESQL_DBUSER}:${INVENIO_POSTGRESQL_DBPASS}@${INVENIO_POSTGRESQL_HOST}:5432/${INVENIO_POSTGRESQL_DBNAME}'" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "# Redis" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "CACHE_REDIS_HOST='${INVENIO_REDIS_HOST}'" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "# Celery" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "BROKER_URL='redis://${INVENIO_REDIS_HOST}:6379/0'" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "CELERY_RESULT_BACKEND='redis://${INVENIO_REDIS_HOST}:6379/1'" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "CELERY_ALWAYS_EAGER=True" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "CELERY_EAGER_PROPAGATES_EXCEPTIONS=True" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "# Elasticsearch" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
echo "SEARCH_ELASTIC_HOSTS='${INVENIO_ELASTICSEARCH_HOST}'" >> ../../var/${INVENIO_WEB_INSTANCE}-instance/${INVENIO_WEB_INSTANCE}.cfg
# sphinxdoc-customise-instance-end
# sphinxdoc-run-npm-begin
cd ${INVENIO_WEB_INSTANCE}
${INVENIO_WEB_INSTANCE} npm
cdvirtualenv var/${INVENIO_WEB_INSTANCE}-instance/static
CI=true npm install
cd -
# sphinxdoc-run-npm-end
# sphinxdoc-collect-and-build-assets-begin
${INVENIO_WEB_INSTANCE} collect -v
${INVENIO_WEB_INSTANCE} assets build
# sphinxdoc-collect-and-build-assets-end
# sphinxdoc-create-database-begin
${INVENIO_WEB_INSTANCE} db init
${INVENIO_WEB_INSTANCE} db create
# sphinxdoc-create-database-end
# sphinxdoc-create-user-account-begin
${INVENIO_WEB_INSTANCE} users create \
--email ${INVENIO_USER_EMAIL} \
--password ${INVENIO_USER_PASS} \
--active
# sphinxdoc-create-user-account-end
# sphinxdoc-start-celery-worker-begin
# FIXME we should run celery worker on another node
-# NOTE The celery worker command is not needed since we run with CELERY_ALWAYS_EAGER
+# NOTE The celery worker command is not needed since we run
+# with CELERY_ALWAYS_EAGER
# celery worker -A ${INVENIO_WEB_INSTANCE}.celery -l INFO &
# sphinxdoc-start-celery-worker-end
# sphinxdoc-populate-with-demo-records-begin
-echo '{"title": "Invenio 3 rocks"}' | \
- ${INVENIO_WEB_INSTANCE} records create \
- -i 4b0714f6-9fe4-43d3-a08f-38c2b309afba
+# discover the location of demo MARC21 record file:
+demomarc21pathname=$(echo "from __future__ import print_function; \
+import pkg_resources; \
+print(pkg_resources.resource_filename('invenio_records', \
+ 'data/marc21/bibliographic.xml'))" | python)
+
+# count the number of demo MARC21 records:
+demomarc21nbrecs=$(grep -c '' $demomarc21pathname)
+
+# convert demo records from MARC21 to JSON and load them
+# using randomly generated UUIDs:
+demouuids=$(dojson do -i $demomarc21pathname -l marcxml marc21 | \
+ ${INVENIO_WEB_INSTANCE} records create \
+ $(for i in $(seq 1 $demomarc21nbrecs); \
+ do echo "-i " $(uuid); done))
# sphinxdoc-populate-with-demo-records-end
# sphinxdoc-register-pid-begin
-echo "from invenio_db import db; \
-from uuid import uuid4; \
-from invenio_pidstore.models import PersistentIdentifier; \
-from invenio_pidstore.models import PIDStatus; \
-pid = PersistentIdentifier.create('recid', '1', \
- object_type='rec', \
- object_uuid='4b0714f6-9fe4-43d3-a08f-38c2b309afba', \
- status=PIDStatus.REGISTERED); \
-db.session.commit()" | ${INVENIO_WEB_INSTANCE} shell
+recid=1
+for demouuid in $demouuids; do
+ ${INVENIO_WEB_INSTANCE} pid create \
+ -t rec -i $demouuid -s REGISTERED recid $recid
+ let recid=recid+1
+done
# sphinxdoc-register-pid-end
# sphinxdoc-start-application-begin
${INVENIO_WEB_INSTANCE} --debug run -h 0.0.0.0 &
# sphinxdoc-start-application-end
diff --git a/scripts/provision-web.sh b/scripts/provision-web.sh
index 08c3b93ba..eb560bb35 100755
--- a/scripts/provision-web.sh
+++ b/scripts/provision-web.sh
@@ -1,188 +1,198 @@
#!/usr/bin/env bash
#
# This file is part of Invenio.
-# Copyright (C) 2015 CERN.
+# Copyright (C) 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
# quit on errors:
set -o errexit
# quit on unbound symbols:
set -o nounset
provision_web_common_ubuntu_trusty () {
# sphinxdoc-install-useful-system-tools-trusty-begin
# update list of available packages:
sudo DEBIAN_FRONTEND=noninteractive apt-get -y update
# install useful system tools:
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
curl \
git \
rlwrap \
screen \
vim
+
+ # install uuid generator (useful for loading demo records):
+ sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
+ uuid
# sphinxdoc-install-useful-system-tools-trusty-end
# sphinxdoc-add-nodejs-external-repository-trusty-begin
if [[ ! -f /etc/apt/sources.list.d/nodesource.list ]]; then
curl -sL https://deb.nodesource.com/setup_4.x | sudo bash -
fi
# sphinxdoc-add-nodejs-external-repository-trusty-end
# sphinxdoc-install-web-common-trusty-begin
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
libffi-dev \
libfreetype6-dev \
libjpeg-dev \
libmsgpack-dev \
libssl-dev \
libtiff-dev \
libxml2-dev \
libxslt-dev \
nodejs \
python-dev \
python-pip
# sphinxdoc-install-web-common-trusty-end
}
provision_web_libpostgresql_ubuntu_trusty () {
# sphinxdoc-install-web-libpostgresql-trusty-begin
sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \
libpq-dev
# sphinxdoc-install-web-libpostgresql-trusty-end
}
provision_web_common_centos7 () {
# sphinxdoc-install-useful-system-tools-centos7-begin
+ # install useful system tools:
sudo yum install -y \
curl \
git \
rlwrap \
screen \
vim
+
+ # install uuid generator (useful for loading demo records):
+ sudo yum install -y \
+ uuid
# sphinxdoc-install-useful-system-tools-centos7-end
# sphinxdoc-add-nodejs-external-repository-centos7-begin
# add EPEL external repository:
sudo yum install -y epel-release
# sphinxdoc-add-nodejs-external-repository-centos7-end
# sphinxdoc-install-web-common-centos7-begin
+ # install development tools:
sudo yum update -y
sudo yum groupinstall -y "Development Tools"
sudo yum install -y \
libffi-devel \
libxml2-devel \
libxslt-devel \
npm \
python-devel \
python-pip
# sphinxdoc-install-web-common-centos7-end
}
provision_web_libpostgresql_centos7 () {
# sphinxdoc-install-web-libpostgresql-centos7-begin
sudo yum install -y \
postgresql-devel
# sphinxdoc-install-web-libpostgresql-centos7-end
}
setup_npm_and_css_js_filters () {
# sphinxdoc-install-npm-and-css-js-filters-begin
sudo su -c "npm install -g npm"
sudo su -c "npm install -g node-sass clean-css requirejs uglify-js"
# sphinxdoc-install-npm-and-css-js-filters-end
}
setup_virtualenvwrapper () {
# disable quitting on errors due to virtualenvrapper:
set +o errexit
set +o nounset
# sphinxdoc-install-virtualenvwrapper-begin
sudo pip install -U virtualenvwrapper pip
if ! grep -q virtualenvwrapper ~/.bashrc; then
mkdir -p $HOME/.virtualenvs
echo "export WORKON_HOME=$HOME/.virtualenvs" >> $HOME/.bashrc
echo "source $(which virtualenvwrapper.sh)" >> $HOME/.bashrc
fi
export WORKON_HOME=$HOME/.virtualenvs
source $(which virtualenvwrapper.sh)
# sphinxdoc-install-virtualenvwrapper-end
# enable quitting on errors back:
set -o errexit
set -o nounset
}
main () {
# detect OS distribution and release version:
if hash lsb_release 2> /dev/null; then
os_distribution=$(lsb_release -i | cut -f 2)
os_release=$(lsb_release -r | cut -f 2)
elif [ -e /etc/redhat-release ]; then
os_distribution=$(cat /etc/redhat-release | cut -d ' ' -f 1)
os_release=$(cat /etc/redhat-release | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1)
else
os_distribution="UNDETECTED"
os_release="UNDETECTED"
fi
# call appropriate provisioning functions:
if [ "$os_distribution" = "Ubuntu" ]; then
if [ "$os_release" = "14.04" ]; then
provision_web_common_ubuntu_trusty
provision_web_libpostgresql_ubuntu_trusty
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
elif [ "$os_distribution" = "CentOS" ]; then
if [ "$os_release" = "7" ]; then
provision_web_common_centos7
provision_web_libpostgresql_centos7
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
else
echo "[ERROR] Sorry, unsupported distribution ${os_distribution}."
exit 1
fi
# finish with common setups:
setup_npm_and_css_js_filters
setup_virtualenvwrapper
}
main
diff --git a/setup.py b/setup.py
index 95efa87dc..6efe7bdd2 100644
--- a/setup.py
+++ b/setup.py
@@ -1,200 +1,201 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2013, 2014, 2015 CERN.
+# Copyright (C) 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
"""Invenio Digital Library Framework."""
import os
import sys
from setuptools import find_packages, setup
from setuptools.command.test import test as TestCommand
readme = open('README.rst').read()
history = open('CHANGES.rst').read()
tests_require = [
'check-manifest>=0.25',
'coverage>=4.0',
'isort>=4.2.2',
'pep257>=0.7.0',
'pytest-cache>=1.0',
'pytest-cov>=1.8.0',
'pytest-pep8>=1.0.6',
'pytest>=2.8.0',
]
invenio_db_version = '>=1.0.0a7,<1.1.0'
extras_require = {
'access': [
'invenio-access>=1.0.0a2,<1.1.0',
],
'accounts': [
'invenio-accounts>=1.0.0a2,<1.1.0',
],
'records': [
+ 'dojson>=0.4.0',
'invenio-pidstore>=1.0.0a1,<1.1.0',
'invenio-records>=1.0.0a8,<1.1.0',
'invenio-records-ui>=1.0.0a4,<1.1.0',
'invenio-records-rest>=1.0.0a2,<1.1.0',
],
'search': [
'invenio-search>=1.0.0a1,<1.1.0',
],
'theme': [
'invenio-assets>=1.0.0a1,<1.1.0',
'invenio-theme>=1.0.0a6,<1.1.0',
],
'utils': [
'invenio-mail>=1.0.0a1,<1.1.0',
'invenio-rest>=1.0.0a2,<1.1.0',
'invenio-logging>=1.0.0a1,<1.1.0',
],
'mysql': [
'invenio-db[mysql]' + invenio_db_version,
],
'postgresql': [
'invenio-db[postgresql]' + invenio_db_version,
],
'docs': [
'Sphinx>=1.3',
],
'tests': tests_require,
}
#
# Aliases allow for easy installation of a specific type of Invenio instances.
# pip install invenio[repository]
#
aliases = {
'minimal': ['accounts', 'theme', 'utils', ],
'full': ['access', 'accounts', 'records', 'search', 'theme', 'utils'],
}
for name, requires in aliases.items():
extras_require[name] = []
for r in requires:
extras_require[name].extend(extras_require[r])
# All alias to install every possible dependency.
extras_require['all'] = []
for name, reqs in extras_require.items():
if name in {'mysql', 'postgresql'}:
continue
extras_require['all'].extend(reqs)
#
# Minimal required packages for an Invenio instance (basically just the
# Flask application loading).
#
setup_requires = [
'Babel>=1.3',
]
install_requires = [
'invenio-base>=1.0.0a3,<1.1.0',
'invenio-celery>=1.0.0a2,<1.1.0',
'invenio-config>=1.0.0a1,<1.1.0',
'invenio-i18n>=1.0.0a1,<1.1.0',
]
packages = find_packages()
class PyTest(TestCommand):
"""PyTest Test."""
user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
def initialize_options(self):
"""Init pytest."""
TestCommand.initialize_options(self)
self.pytest_args = []
try:
from ConfigParser import ConfigParser
except ImportError:
from configparser import ConfigParser
config = ConfigParser()
config.read('pytest.ini')
self.pytest_args = config.get('pytest', 'addopts').split(' ')
def finalize_options(self):
"""Finalize pytest."""
TestCommand.finalize_options(self)
if hasattr(self, '_test_args'):
self.test_suite = ''
else:
self.test_args = []
self.test_suite = True
def run_tests(self):
"""Run tests."""
# import here, cause outside the eggs aren't loaded
import pytest
errno = pytest.main(self.pytest_args)
sys.exit(errno)
# Get the version string. Cannot be done with import!
g = {}
with open(os.path.join('invenio', 'version.py'), 'rt') as fp:
exec(fp.read(), g)
version = g['__version__']
setup(
name='invenio',
version=version,
description=__doc__,
long_description=readme + '\n\n' + history,
keywords='Invenio digital library framework',
license='GPLv2',
author='CERN',
author_email='info@invenio-software.org',
url='https://github.com/inveniosoftware/invenio',
packages=packages,
zip_safe=False,
include_package_data=True,
platforms='any',
entry_points={},
extras_require=extras_require,
install_requires=install_requires,
setup_requires=setup_requires,
tests_require=tests_require,
classifiers=[
'Environment :: Web Environment',
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
'Operating System :: OS Independent',
'Programming Language :: Python',
'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
'Topic :: Software Development :: Libraries :: Python Modules',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Development Status :: 3 - Alpha',
],
cmdclass={'test': PyTest},
)