diff --git a/.gitignore b/.gitignore index 243c576ef..134da317a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,96 +1,98 @@ .version Makefile Makefile.in configure config.cache config.log config.status config.nice config.guess config.sub install-sh missing compile autom4te.cache aclocal.m4 TAGS invenio-autotools.conf po/POTFILES po/POTFILES-py po/POTFILES-webdoc po/stamp-po po/*.gmo po/*.mo po/*.sed *~ *.pyc *.clisp.mem *.cmucl.core *.sbcl.core *.fas *.fasl *.sse2f *.lib *.x86f modules/webauthorprofile/bin/webauthorprofile modules/bibauthorid/bin/bibauthorid modules/bibcirculation/bin/bibcircd modules/bibclassify/bin/bibclassify modules/bibconvert/bin/bibconvert modules/bibdocfile/bin/bibdocfile modules/bibedit/bin/bibedit modules/refextract/bin/refextract modules/bibrecord/bin/textmarc2xmlmarc modules/bibrecord/bin/xmlmarc2textmarc modules/bibrecord/bin/xmlmarclint modules/bibencode/bin/bibencode modules/bibexport/bin/bibexport modules/bibformat/bin/bibreformat modules/oaiharvest/bin/oaiharvest modules/oairepository/bin/oairepositoryupdater modules/bibindex/bin/bibindex modules/bibindex/bin/bibstat modules/bibmatch/bin/bibmatch modules/bibrank/bin/bibrank modules/bibrank/bin/bibrankgkb modules/bibrank/etc/bibrankgkb.cfg modules/bibrank/etc/demo_jif.cfg modules/bibrank/etc/template_single_tag_rank_method.cfg modules/bibsched/bin/bibsched modules/bibsched/bin/bibtaskex modules/bibsched/bin/bibtasklet modules/bibsort/bin/bibsort modules/bibsword/bin/bibsword modules/bibupload/bin/batchuploader modules/bibupload/bin/bibupload modules/elmsubmit/bin/elmsubmit modules/elmsubmit/etc/elmsubmit.cfg modules/miscutil/bin/dbdump modules/miscutil/bin/dbexec modules/miscutil/bin/inveniocfg modules/miscutil/bin/plotextractor modules/miscutil/etc/bash_completion.d/inveniocfg modules/miscutil/lib/build modules/webaccess/bin/authaction modules/webaccess/bin/webaccessadmin modules/webalert/bin/alertengine modules/webmessage/bin/webmessageadmin modules/websearch/bin/webcoll modules/websession/bin/inveniogc modules/webstat/bin/webstat modules/webstat/bin/webstatadmin modules/webstyle/bin/webdoc modules/websubmit/bin/bibdocfile modules/websubmit/bin/inveniounoconv modules/websubmit/bin/websubmitadmin tags config.status.lineno configure.lineno *.kdevelop *.kdevses .project .settings .pydevproject org.eclipse.core.resources.prefs supervisord.log supervisord.pid +.vagrant +build \ No newline at end of file diff --git a/.inveniorc b/.inveniorc new file mode 100644 index 000000000..eca7bf257 --- /dev/null +++ b/.inveniorc @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# Kickstart Invenio 1 demo instance. + +export INVENIO_SRCDIR=/vagrant +export INVENIO_ADMIN_EMAIL=info@inveniosoftware.org +export INVENIO_WEB_DSTDIR=/opt/invenio +export INVENIO_WEB_HOST=192.168.50.10 +export INVENIO_WEB_USER=www-data +export INVENIO_MYSQL_HOST=192.168.50.11 +export INVENIO_MYSQL_DBNAME=invenio1 +export INVENIO_MYSQL_DBUSER=invenio1 +export INVENIO_MYSQL_DBPASS=dbpass123 + +# Now you can use kickstart scripts, for example: +# +# vagrant up --no-parallel +# vagrant ssh web -c 'source .inveniorc && /vagrant/scripts/create-instance.sh' +# vagrant ssh web -c 'source .inveniorc && /vagrant/scripts/populate-instance.sh' +# firefox http://192.168.50.10/record/1 +# vagrant ssh web -c 'source .inveniorc && sudo -u www-data /opt/invenio/bin/inveniocfg --run-unit-tests' +# vagrant ssh web -c 'source .inveniorc && sudo -u www-data /opt/invenio/bin/inveniocfg --run-regression-tests --yes-i-know' diff --git a/.travis.yml b/.travis.yml index 1aaed64e3..1bcf44e2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,33 +1,49 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2013, 2014 CERN. +## Copyright (C) 2013, 2014, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. notifications: email: false +sudo: true + python: - "2.6" +env: + global: + - INVENIO_SRCDIR=$(pwd) + - INVENIO_ADMIN_EMAIL=info@inveniosoftware.org + - INVENIO_WEB_DSTDIR=/opt/invenio + - INVENIO_WEB_HOST=127.0.0.1 + - INVENIO_WEB_USER=www-data + - INVENIO_WEB_SMTP_PORT=0 + - INVENIO_MYSQL_HOST=127.0.0.1 + - INVENIO_MYSQL_DBNAME=invenio1 + - INVENIO_MYSQL_DBUSER=invenio1 + - INVENIO_MYSQL_DBPASS=dbpass123 + install: - - sudo apt-get update - - git clone https://github.com/tiborsimko/invenio-devscripts /tmp/invenio-devscripts - - CFG_INVENIO_SRCDIR=$(pwd) /tmp/invenio-devscripts/invenio-kickstart --yes-i-know --yes-i-really-know + - ./scripts/provision-mysql.sh + - ./scripts/provision-web.sh + - ./scripts/create-instance.sh + - ./scripts/populate-instance.sh script: - sudo -u www-data nosetests /opt/invenio/lib/python/invenio/*_unit_tests.py - sudo -u www-data nosetests /opt/invenio/lib/python/invenio/*_regression_tests.py diff --git a/INSTALL b/INSTALL index adaa8650f..f85baa536 100644 --- a/INSTALL +++ b/INSTALL @@ -1,829 +1,851 @@ -Invenio INSTALLATION +-Invenio INSTALLATION ==================== About ===== This document specifies how to build, customize, and install Invenio v1.1.6 for the first time. See RELEASE-NOTES if you are upgrading from a previous Invenio release. Contents ======== -0. Prerequisites -1. Quick instructions for the impatient Invenio admin -2. Detailed instructions for the patient Invenio admin +1. Using kickstart scripts +2. Prerequisites +3. Quick instructions for the impatient Invenio admin +4. Detailed instructions for the patient Invenio admin -0. Prerequisites +1. Using kickstart scripts +========================== + + You can install Invenio on a Unix-based operating system in an unassisted + automated way by using kickstart scripts. Tested on Ubuntu 12.04, CentOS 6. + The other operating systems will work similarly. Please run: + + $ $EDITOR .inveniorc # edit your options + $ source .inveniorc + $ ./scripts/provision-web.sh + $ ./scripts/provision-mysql.sh + $ ./scripts/create-instance.sh + $ ./scripts/populate-instance.sh + $ firefox http://192.168.50.10/record/1 + + Note that you can use Vagrant to run this in a virtual machine; please see the + .inveniorc file for more details. + + You can also read the detailed installation instructions in the sections + below. They basically explain in detail what the kickstart scripts do. + +2. Prerequisites ================ Here is the software you need to have around before you start installing Invenio: a) Unix-like operating system. The main development and production platforms for Invenio at CERN are GNU/Linux distributions Debian, Gentoo, Scientific Linux (aka RHEL), Ubuntu, but we also develop on Mac OS X. Basically any Unix system supporting the software listed below should do. If you are using Debian GNU/Linux ``Lenny'' or later, then you can install most of the below-mentioned prerequisites and recommendations by running: $ sudo aptitude install python-dev apache2-mpm-prefork \ mysql-server mysql-client python-mysqldb \ python-4suite-xml python-simplejson python-xml \ python-libxml2 python-libxslt1 gnuplot poppler-utils \ gs-common clisp gettext libapache2-mod-wsgi unzip \ python-dateutil python-rdflib \ python-gnuplot python-magic pdftk html2text giflib-tools \ pstotext netpbm python-pypdf python-chardet python-lxml \ python-unidecode You may also want to install some of the following packages, if you have them available on your concrete architecture: $ sudo aptitude install sbcl cmucl pylint pychecker pyflakes \ python-profiler python-epydoc libapache2-mod-xsendfile \ openoffice.org python-utidylib python-beautifulsoup Moreover, you should install some Message Transfer Agent (MTA) such as Postfix so that Invenio can email notification alerts or registration information to the end users, contact moderators and reviewers of submitted documents, inform administrators about various runtime system information, etc: $ sudo aptitude install postfix After running the above-quoted aptitude command(s), you can proceed to configuring your MySQL server instance - (max_allowed_packet in my.cnf, see item 0b below) and then to + (max_allowed_packet in my.cnf, see item b below) and then to installing the Invenio software package in the section 1 below. If you are using another operating system, then please continue reading the rest of this prerequisites section, and please consult our wiki pages for any concrete hints for your specific operating system. b) MySQL server (may be on a remote machine), and MySQL client (must be available locally too). MySQL versions 4.1 or 5.0 are supported. Please set the variable "max_allowed_packet" in your "my.cnf" init file to at least 4M. (For sites such as INSPIRE, having 1M records with 10M citer-citee pairs in its citation map, you may need to increase max_allowed_packet to 1G.) You may perhaps also want to run your MySQL server natively in UTF-8 mode by setting "default-character-set=utf8" in various parts of your "my.cnf" file, such as in the "[mysql]" part and elsewhere; but this is not really required. Note also that you may encounter problems when MySQL is run in "strict mode"; you may want to configure your "my.cnf" in order to avoid using strict mode (such as `STRICT_ALL_TABLES`). c) Apache 2 server, with support for loading DSO modules, and optionally with SSL support for HTTPS-secure user authentication, and mod_xsendfile for off-loading file downloads away from Invenio processes to Apache. d) Python v2.4 or above: as well as the following Python modules: - (mandatory) MySQLdb (version >= 1.2.1_p2; see below) - (recommended) python-dateutil, for complex date processing: - (recommended) PyXML, for XML processing: - (recommended) PyRXP, for very fast XML MARC processing: - (recommended) lxml, for XML/XLST processing: - (recommended) libxml2-python, for XML/XLST processing: - (recommended) simplejson, for AJAX apps: Note that if you are using Python-2.6, you don't need to install simplejson, because the module is already included in the main Python distribution. - (recommended) Gnuplot.Py, for producing graphs: - (recommended) Snowball Stemmer, for stemming: - (recommended) py-editdist, for record merging: - (recommended) numpy, for citerank methods: - (recommended) magic, for full-text file handling: - (optional) chardet, for character encoding detection: - (optional) 4suite, slower alternative to PyRXP and libxml2-python: - (optional) feedparser, for web journal creation: - (optional) RDFLib, to use RDF ontologies and thesauri: - (optional) mechanize, to run regression web test suite: - (optional) python-mock, mocking library for the test suite: - (optional) hashlib, needed only for Python-2.4 and only if you would like to use AWS connectivity: - (optional) utidylib, for HTML washing: - (optional) Beautiful Soup, for HTML washing: - (optional) Python Twitter (and its dependencies) if you want to use the Twitter Fetcher bibtasklet: - (optional) unidecode, for ASCII representation of Unicode text: Note: MySQLdb version 1.2.1_p2 or higher is recommended. If you are using an older version of MySQLdb, you may get into problems with character encoding. e) mod_wsgi Apache module. Versions 3.x and above are recommended. Note: if you are using Python 2.4 or earlier, then you should also install the wsgiref Python module, available from: (As of Python 2.5 this module is included in standard Python distribution.) f) If you want to be able to extract references from PDF fulltext files, then you need to install pdftotext version 3 at least. g) If you want to be able to search for words in the fulltext files (i.e. to have fulltext indexing) or to stamp submitted files, then you need as well to install some of the following tools: - for Microsoft Office/OpenOffice.org document conversion: OpenOffice.org - for PDF file stamping: pdftk, pdf2ps - for PDF files: pdftotext or pstotext - for PostScript files: pstotext or ps2ascii - for DjVu creation, elaboration: DjVuLibre - to perform OCR: OCRopus (tested only with release 0.3.1) - to perform different image elaborations: ImageMagick - to generate PDF after OCR: netpbm, ReportLab and pyPdf or pyPdf2 h) If you have chosen to install fast XML MARC Python processors in the step d) above, then you have to install the parsers themselves: - (optional) 4suite: i) (recommended) Gnuplot, the command-line driven interactive plotting program. It is used to display download and citation history graphs on the Detailed record pages on the web interface. Note that Gnuplot must be compiled with PNG output support, that is, with the GD library. Note also that Gnuplot is not required, only recommended. j) (recommended) A Common Lisp implementation, such as CLISP, SBCL or CMUCL. It is used for the web server log analysing tool and the metadata checking program. Note that any of the three implementations CLISP, SBCL, or CMUCL will do. CMUCL produces fastest machine code, but it does not support UTF-8 yet. Pick up CLISP if you don't know what to do. Note that a Common Lisp implementation is not required, only recommended. k) GNU gettext, a set of tools that makes it possible to translate the application in multiple languages. This is available by default on many systems. l) (recommended) xlwt 0.7.2, Library to create spreadsheet files compatible with MS Excel 97/2000/XP/2003 XLS files, on any platform, with Python 2.3 to 2.6 m) (recommended) matplotlib 1.0.0 is a python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms. matplotlib can be used in python scripts, the python and ipython shell (ala MATLAB® or Mathematica®), web application servers, and six graphical user interface toolkits. It is used to generate pie graphs in the custom summary query (WebStat) n) (optional) FFmpeg, an open-source tools an libraries collection to convert video and audio files. It makes use of both internal as well as external libraries to generate videos for the web, such as Theora, WebM and H.264 out of almost any thinkable video input. FFmpeg is needed to run video related modules and submission workflows in Invenio. The minimal configuration of ffmpeg for the Invenio demo site requires a number of external libraries. It is highly recommended to remove all installed versions and packages that are comming with various Linux distributions and install the latest versions from sources. Additionally, you will need the Mediainfo Library for multimedia metadata handling. Minimum libraries for the demo site: - the ffmpeg multimedia encoder tools - a library for jpeg images needed for thumbnail extraction - a library for the ogg container format, needed for Vorbis and Theora - the OGG Vorbis audi codec library - the OGG Theora video codec library - the WebM video codec library - the mediainfo library for multimedia metadata Recommended for H.264 video (!be aware of licensing issues!): - a library for H.264 video encoding - a library for Advanced Audi Coding - a library for MP3 encoding Note that the configure script checks whether you have all the prerequisite software installed and that it won't let you continue unless everything is in order. It also warns you if it cannot find some optional but recommended software. -1. Quick instructions for the impatient Invenio admin -========================================================= +3. Quick instructions for the impatient Invenio admin +===================================================== -1a. Installation +3a. Installation ---------------- $ cd $HOME/src/ $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz.md5 $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz.sig $ md5sum -c invenio-1.1.6.tar.gz.md5 $ gpg --verify invenio-1.1.6.tar.gz.sig invenio-1.1.6.tar.gz $ tar xvfz invenio-1.1.6.tar.gz $ cd invenio-1.1.6 $ ./configure $ make $ make install $ make install-mathjax-plugin ## optional $ make install-jquery-plugins ## optional $ make install-ckeditor-plugin ## optional $ make install-pdfa-helper-files ## optional $ make install-mediaelement ## optional -1b. Configuration +3b. Configuration ----------------- $ sudo chown -R www-data.www-data /opt/invenio - $ sudo -u www-data emacs /opt/invenio/etc/invenio-local.conf + $ sudo -u www-data $EDITOR /opt/invenio/etc/invenio-local.conf $ sudo -u www-data /opt/invenio/bin/inveniocfg --update-all $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-tables $ sudo -u www-data /opt/invenio/bin/inveniocfg --load-webstat-conf $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-apache-conf $ sudo /etc/init.d/apache2 restart $ sudo -u www-data /opt/invenio/bin/inveniocfg --check-openoffice $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-demo-site $ sudo -u www-data /opt/invenio/bin/inveniocfg --load-demo-records $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-unit-tests $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-regression-tests $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-web-tests $ sudo -u www-data /opt/invenio/bin/inveniocfg --remove-demo-records $ sudo -u www-data /opt/invenio/bin/inveniocfg --drop-demo-site $ firefox http://your.site.com/help/admin/howto-run -2. Detailed instructions for the patient Invenio admin -========================================================== +4. Detailed instructions for the patient Invenio admin +====================================================== -2a. Installation +4a. Installation ---------------- The Invenio uses standard GNU autoconf method to build and install its files. This means that you proceed as follows: $ cd $HOME/src/ Change to a directory where we will build the Invenio sources. (The built files will be installed into different "target" directories later.) $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz.md5 $ wget http://invenio-software.org/download/invenio-1.1.6.tar.gz.sig Fetch Invenio source tarball from the distribution server, together with MD5 checksum and GnuPG cryptographic signature files useful for verifying the integrity of the tarball. $ md5sum -c invenio-1.1.6.tar.gz.md5 Verify MD5 checksum. $ gpg --verify invenio-1.1.6.tar.gz.sig invenio-1.1.6.tar.gz Verify GnuPG cryptographic signature. Note that you may first have to import my public key into your keyring, if you haven't done that already: $ gpg --keyserver pool.sks-keyservers.net --recv-key 0xBA5A2B67 The output of the gpg --verify command should then read: Good signature from "Tibor Simko " You can safely ignore any trusted signature certification warning that may follow after the signature has been successfully verified. $ tar xvfz invenio-1.1.6.tar.gz Untar the distribution tarball. $ cd invenio-1.1.6 Go to the source directory. $ ./configure Configure Invenio software for building on this specific platform. You can use the following optional parameters: --prefix=/opt/invenio Optionally, specify the Invenio general installation directory (default is /opt/invenio). It will contain command-line binaries and program libraries containing the core Invenio functionality, but also store web pages, runtime log and cache information, document data files, etc. Several subdirs like `bin', `etc', `lib', or `var' will be created inside the prefix directory to this effect. Note that the prefix directory should be chosen outside of the Apache htdocs tree, since only one its subdirectory (prefix/var/www) is to be accessible directly via the Web (see below). Note that Invenio won't install to any other directory but to the prefix mentioned in this configuration line. --with-python=/opt/python/bin/python2.4 Optionally, specify a path to some specific Python binary. This is useful if you have more than one Python installation on your system. If you don't set this option, then the first Python that will be found in your PATH will be chosen for running Invenio. --with-mysql=/opt/mysql/bin/mysql Optionally, specify a path to some specific MySQL client binary. This is useful if you have more than one MySQL installation on your system. If you don't set this option, then the first MySQL client executable that will be found in your PATH will be chosen for running Invenio. --with-clisp=/opt/clisp/bin/clisp Optionally, specify a path to CLISP executable. This is useful if you have more than one CLISP installation on your system. If you don't set this option, then the first executable that will be found in your PATH will be chosen for running Invenio. --with-cmucl=/opt/cmucl/bin/lisp Optionally, specify a path to CMUCL executable. This is useful if you have more than one CMUCL installation on your system. If you don't set this option, then the first executable that will be found in your PATH will be chosen for running Invenio. --with-sbcl=/opt/sbcl/bin/sbcl Optionally, specify a path to SBCL executable. This is useful if you have more than one SBCL installation on your system. If you don't set this option, then the first executable that will be found in your PATH will be chosen for running Invenio. --with-openoffice-python Optionally, specify the path to the Python interpreter embedded with OpenOffice.org. This is normally not contained in the normal path. If you don't specify this it won't be possible to use OpenOffice.org to convert from and to Microsoft Office and OpenOffice.org documents. This configuration step is mandatory. Usually, you do this step only once. (Note that if you are building Invenio not from a released tarball, but from the Git sources, then you have to generate the configure file via autotools: $ sudo aptitude install automake1.9 autoconf $ aclocal-1.9 $ automake-1.9 -a $ autoconf after which you proceed with the usual configure command.) $ make Launch the Invenio build. Since many messages are printed during the build process, you may want to run it in a fast-scrolling terminal such as rxvt or in a detached screen session. During this step all the pages and scripts will be pre-created and customized based on the config you have edited in the previous step. Note that on systems such as FreeBSD or Mac OS X you have to use GNU make ("gmake") instead of "make". $ make install Install the web pages, scripts, utilities and everything needed for Invenio runtime into respective installation directories, as specified earlier by the configure command. Note that if you are installing Invenio for the first time, you will be asked to create symbolic link(s) from Python's site-packages system-wide directory(ies) to the installation location. This is in order to instruct Python where to find Invenio's Python files. You will be hinted as to the exact command to use based on the parameters you have used in the configure command. $ make install-mathjax-plugin ## optional This will automatically download and install in the proper place MathJax, a JavaScript library to render LaTeX formulas in the client browser. Note that in order to enable the rendering you will have to set the variable CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS in invenio-local.conf to a suitable list of output format codes. For example: CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS = hd,hb $ make install-jquery-plugins ## optional This will automatically download and install in the proper place jQuery and related plugins. They are used for AJAX applications such as the record editor. Note that `unzip' is needed when installing jquery plugins. $ make install-ckeditor-plugin ## optional This will automatically download and install in the proper place CKeditor, a WYSIWYG Javascript-based editor (e.g. for the WebComment module). Note that in order to enable the editor you have to set the CFG_WEBCOMMENT_USE_RICH_EDITOR to True. $ make install-pdfa-helper-files ## optional This will automatically download and install in the proper place the helper files needed to create PDF/A files out of existing PDF files. $ make install-mediaelement ## optional This will automatically download and install the MediaElementJS HTML5 video player that is needed for videos on the DEMO site. $ make install-solrutils ## optional This will automatically download and install a Solr instance which can be used for full-text searching. See CFG_SOLR_URL variable in the invenio.conf. Note that the admin later has to take care of running init.d scripts which would start the Solr instance automatically. -2b. Configuration +4b. Configuration ----------------- Once the basic software installation is done, we proceed to configuring your Invenio system. $ sudo chown -R www-data.www-data /opt/invenio For the sake of simplicity, let us assume that your Invenio installation will run under the `www-data' user process identity. The above command changes ownership of installed files to www-data, so that we shall run everything under this user identity from now on. For production purposes, you would typically enable Apache server to read all files from the installation place but to write only to the `var' subdirectory of your installation place. You could achieve this by configuring Unix directory group permissions, for example. - $ sudo -u www-data emacs /opt/invenio/etc/invenio-local.conf + $ sudo -u www-data $EDITOR /opt/invenio/etc/invenio-local.conf Customize your Invenio installation. Please read the 'invenio.conf' file located in the same directory that contains the vanilla default configuration parameters of your Invenio installation. If you want to customize some of these parameters, you should create a file named 'invenio-local.conf' in the same directory where 'invenio.conf' lives and you should write there only the customizations that you want to be different from the vanilla defaults. Here is a realistic, minimalist, yet production-ready example of what you would typically put there: $ cat /opt/invenio/etc/invenio-local.conf [Invenio] CFG_SITE_NAME = John Doe's Document Server CFG_SITE_NAME_INTL_fr = Serveur des Documents de John Doe CFG_SITE_URL = http://your.site.com CFG_SITE_SECURE_URL = https://your.site.com CFG_SITE_ADMIN_EMAIL = john.doe@your.site.com CFG_SITE_SUPPORT_EMAIL = john.doe@your.site.com CFG_WEBALERT_ALERT_ENGINE_EMAIL = john.doe@your.site.com CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL = john.doe@your.site.com CFG_WEBCOMMENT_DEFAULT_MODERATOR = john.doe@your.site.com CFG_DATABASE_HOST = localhost CFG_DATABASE_NAME = invenio CFG_DATABASE_USER = invenio CFG_DATABASE_PASS = my123p$ss CFG_BIBDOCFILE_ENABLE_BIBDOCFSINFO_CACHE = 1 You should override at least the parameters mentioned above in order to define some very essential runtime parameters such as the name of your document server (CFG_SITE_NAME and CFG_SITE_NAME_INTL_*), the visible URL of your document server (CFG_SITE_URL and CFG_SITE_SECURE_URL), the email address of the local Invenio administrator, comment moderator, and alert engine (CFG_SITE_SUPPORT_EMAIL, CFG_SITE_ADMIN_EMAIL, etc), and last but not least your database credentials (CFG_DATABASE_*). If this is a first installation of Invenio it is recommended you set the CFG_BIBDOCFILE_ENABLE_BIBDOCFSINFO_CACHE variable to 1. If this is instead an upgrade from an existing installation don't add it until you have run: $ bibdocfile --fix-bibdocfsinfo-cache . The Invenio system will then read both the default invenio.conf file and your customized invenio-local.conf file and it will override any default options with the ones you have specifield in your local file. This cascading of configuration parameters will ease your future upgrades. If you want to have multiple Invenio instances for distributed video encoding, you need to share the same configuration amongs them and make some of the folders of the Invenio installation available for all nodes. Configure the allowed tasks for every node: CFG_BIBSCHED_NODE_TASKS = { "hostname_machine1" : ["bibindex", "bibupload", "bibreformat","webcoll", "bibtaskex", "bibrank", "oaiharvest", "oairepositoryupdater", "inveniogc", "webstatadmin", "bibclassify", "bibexport", "dbdump", "batchuploader", "bibauthorid", "bibtasklet"], "hostname_machine2" : ['bibencode',] } Share the following directories among Invenio instances: /var/tmp-shared hosts video uploads in a temporary form /var/tmp-shared/bibencode/jobs hosts new job files for the video encoding daemon /var/tmp-shared/bibencode/jobs/done hosts job files that have been processed by the daemon /var/data/files hosts fulltext and media files associated to records /var/data/submit hosts files created during submissions $ sudo -u www-data /opt/invenio/bin/inveniocfg --update-all Make the rest of the Invenio system aware of your invenio-local.conf changes. This step is mandatory each time you edit your conf files. $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-tables If you are installing Invenio for the first time, you have to create database tables. Note that this step checks for potential problems such as the database connection rights and may ask you to perform some more administrative steps in case it detects a problem. Notably, it may ask you to set up database access permissions, based on your configure values. If you are installing Invenio for the first time, you have to create a dedicated database on your MySQL server that the Invenio can use for its purposes. Please contact your MySQL administrator and ask him to execute the commands this step proposes you. At this point you should now have successfully completed the "make install" process. We continue by setting up the Apache web server. $ sudo -u www-data /opt/invenio/bin/inveniocfg --load-webstat-conf Load the configuration file of webstat module. It will create the tables in the database for register customevents, such as basket hits. $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-apache-conf Running this command will generate Apache virtual host configurations matching your installation. You will be instructed to check created files (usually they are located under /opt/invenio/etc/apache/) and edit your httpd.conf to activate Invenio virtual hosts. If you are using Debian GNU/Linux ``Lenny'' or later, then you can do the following to create your SSL certificate and to activate your Invenio vhosts: ## make SSL certificate: $ sudo aptitude install ssl-cert $ sudo mkdir /etc/apache2/ssl $ sudo /usr/sbin/make-ssl-cert /usr/share/ssl-cert/ssleay.cnf \ /etc/apache2/ssl/apache.pem ## add Invenio web sites: $ sudo ln -s /opt/invenio/etc/apache/invenio-apache-vhost.conf \ /etc/apache2/sites-available/invenio $ sudo ln -s /opt/invenio/etc/apache/invenio-apache-vhost-ssl.conf \ /etc/apache2/sites-available/invenio-ssl ## disable Debian's default web site: $ sudo /usr/sbin/a2dissite default ## enable Invenio web sites: $ sudo /usr/sbin/a2ensite invenio $ sudo /usr/sbin/a2ensite invenio-ssl ## enable SSL module: $ sudo /usr/sbin/a2enmod ssl ## if you are using xsendfile module, enable it too: $ sudo /usr/sbin/a2enmod xsendfile If you are using another operating system, you should do the equivalent, for example edit your system-wide httpd.conf and put the following include statements: Include /opt/invenio/etc/apache/invenio-apache-vhost.conf Include /opt/invenio/etc/apache/invenio-apache-vhost-ssl.conf Note that you may need to adapt generated vhost file snippets to match your concrete operating system specifics. For example, the generated configuration snippet will preload Invenio WSGI daemon application upon Apache start up for faster site response. The generated configuration assumes that you are using mod_wsgi version 3 or later. If you are using the old legacy mod_wsgi version 2, then you would need to comment out the WSGIImportScript directive from the generated snippet, or else move the WSGI daemon setup to the top level, outside of the VirtualHost section. Note also that you may want to tweak the generated Apache vhost snippet for performance reasons, especially with respect to WSGIDaemonProcess parameters. For example, you can increase the number of processes from the default value `processes=5' if you have lots of RAM and if many concurrent users may access your site in parallel. However, note that you must use `threads=1' there, because Invenio WSGI daemon processes are not fully thread safe yet. This may change in the future. $ sudo /etc/init.d/apache2 restart Please ask your webserver administrator to restart the Apache server after the above "httpd.conf" changes. $ sudo -u www-data /opt/invenio/bin/inveniocfg --check-openoffice If you plan to support MS Office or Open Document Format files in your installation, you should check whether LibreOffice or OpenOffice.org is well integrated with Invenio by running the above command. You may be asked to create a temporary directory for converting office files with special ownership (typically as user nobody) and permissions. Note that you can do this step later. $ sudo -u www-data /opt/invenio/bin/inveniocfg --create-demo-site This step is recommended to test your local Invenio installation. It should give you our "Atlantis Institute of Science" demo installation, exactly as you see it at . $ sudo -u www-data /opt/invenio/bin/inveniocfg --load-demo-records Optionally, load some demo records to be able to test indexing and searching of your local Invenio demo installation. $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-unit-tests Optionally, you can run the unit test suite to verify the unit behaviour of your local Invenio installation. Note that this command should be run only after you have installed the whole system via `make install'. $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-regression-tests Optionally, you can run the full regression test suite to verify the functional behaviour of your local Invenio installation. Note that this command requires to have created the demo site and loaded the demo records. Note also that running the regression test suite may alter the database content with junk data, so that rebuilding the demo site is strongly recommended afterwards. $ sudo -u www-data /opt/invenio/bin/inveniocfg --run-web-tests Optionally, you can run additional automated web tests running in a real browser. This requires to have Firefox with the Selenium IDE extension installed. $ sudo -u www-data /opt/invenio/bin/inveniocfg --remove-demo-records Optionally, remove the demo records loaded in the previous step, but keeping otherwise the demo collection, submission, format, and other configurations that you may reuse and modify for your own production purposes. $ sudo -u www-data /opt/invenio/bin/inveniocfg --drop-demo-site Optionally, drop also all the demo configuration so that you'll end up with a completely blank Invenio system. However, you may want to find it more practical not to drop the demo site configuration but to start customizing from there. $ firefox http://your.site.com/help/admin/howto-run In order to start using your Invenio installation, you can start indexing, formatting and other daemons as indicated in the "HOWTO Run" guide on the above URL. You can also use the Admin Area web interfaces to perform further runtime configurations such as the definition of data collections, document types, document formats, word indexes, etc. $ sudo ln -s /opt/invenio/etc/bash_completion.d/inveniocfg \ /etc/bash_completion.d/inveniocfg Optionally, if you are using Bash shell completion, then you may want to create the above symlink in order to configure completion for the inveniocfg command. Happy hacking and thanks for flying Invenio. - Invenio Development Team Email: info@invenio-software.org IRC: #invenio on irc.freenode.net Twitter: http://twitter.com/inveniosoftware Github: http://github.com/inveniosoftware URL: http://invenio-software.org diff --git a/Makefile.am b/Makefile.am index 491a29992..926ea2f53 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,445 +1,446 @@ ## This file is part of Invenio. -## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 CERN. +## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. confignicedir = $(sysconfdir)/build confignice_SCRIPTS=config.nice -SUBDIRS = po config modules +SUBDIRS = po config modules scripts EXTRA_DIST = UNINSTALL THANKS RELEASE-NOTES configure-tests.py config.nice.in \ - config.rpath CONTRIBUTING.rst Dockerfile docker-compose.yml + config.rpath CONTRIBUTING.rst Dockerfile docker-compose.yml \ + requirements.txt Vagrantfile .inveniorc # current MathJax version and packages # See also modules/miscutil/lib/htmlutils.py (get_mathjax_header) MJV = 2.1 MATHJAX = http://invenio-software.org/download/mathjax/MathJax-v$(MJV).zip # current CKeditor version CKV = 3.6.6 CKEDITOR = ckeditor_$(CKV).zip # current MediaElement.js version MEV = master MEDIAELEMENT = http://github.com/johndyer/mediaelement/zipball/$(MEV) #for solrutils INVENIO_JAVA_PATH = org/invenio_software/solr solrdirname = apache-solr-3.1.0 solrdir = $(prefix)/lib/$(solrdirname) solrutils_dir=$(CURDIR)/modules/miscutil/lib/solrutils CLASSPATH=.:${solrdir}/dist/solrj-lib/commons-io-1.4.jar:${solrdir}/dist/apache-solr-core-*jar:${solrdir}/contrib/jzlib-1.0.7.jar:${solrdir}/dist/apache-solr-solrj-3.1.0.jar:${solrdir}/dist/solrj-lib/slf4j-api-1.5.5.jar:${solrdir}/dist/*:${solrdir}/contrib/basic-lucene-libs/*:${solrdir}/contrib/analysis-extras/lucene-libs/*:${solrdir}/dist/solrj-lib/* # git-version-get stuff: BUILT_SOURCES = $(top_srcdir)/.version $(top_srcdir)/.version: echo $(VERSION) > $@-t && mv $@-t $@ dist-hook: echo $(VERSION) > $(distdir)/.tarball-version check-upgrade: $(PYTHON) $(top_srcdir)/modules/miscutil/lib/inveniocfg_upgrader.py $(top_srcdir) --upgrade-check kwalitee-check: @$(PYTHON) $(top_srcdir)/modules/miscutil/lib/kwalitee.py --stats $(top_srcdir) kwalitee-check-errors-only: @$(PYTHON) $(top_srcdir)/modules/miscutil/lib/kwalitee.py --check-errors $(top_srcdir) kwalitee-check-variables: @$(PYTHON) $(top_srcdir)/modules/miscutil/lib/kwalitee.py --check-variables $(top_srcdir) kwalitee-check-indentation: @$(PYTHON) $(top_srcdir)/modules/miscutil/lib/kwalitee.py --check-indentation $(top_srcdir) kwalitee-check-sql-queries: @$(PYTHON) $(top_srcdir)/modules/miscutil/lib/kwalitee.py --check-sql $(top_srcdir) etags: \rm -f $(top_srcdir)/TAGS (cd $(top_srcdir) && find $(top_srcdir) -name "*.py" -print | xargs etags) install-data-local: for d in / /cache /cache/RTdata /log /tmp /tmp-shared /data /run /tmp-shared/bibencode/jobs/done ; do \ mkdir -p $(localstatedir)$$d ; \ done @echo "************************************************************" @echo "** Invenio software has been successfully installed! **" @echo "** **" @echo "** You may proceed to customizing your installation now. **" @echo "************************************************************" install-mathjax-plugin: @echo "***********************************************************" @echo "** Installing MathJax plugin, please wait... **" @echo "***********************************************************" rm -rf /tmp/invenio-mathjax-plugin mkdir /tmp/invenio-mathjax-plugin rm -fr ${prefix}/var/www/MathJax mkdir -p ${prefix}/var/www/MathJax (cd /tmp/invenio-mathjax-plugin && \ wget '$(MATHJAX)' -O mathjax.zip && \ unzip -q mathjax.zip && cd mathjax-MathJax-* && cp -r * \ ${prefix}/var/www/MathJax) rm -fr /tmp/invenio-mathjax-plugin @echo "************************************************************" @echo "** The MathJax plugin was successfully installed. **" @echo "** Please do not forget to properly set the option **" @echo "** CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS in invenio.conf. **" @echo "************************************************************" uninstall-mathjax-plugin: @rm -rvf ${prefix}/var/www/MathJax @echo "***********************************************************" @echo "** The MathJax plugin was successfully uninstalled. **" @echo "***********************************************************" install-jscalendar-plugin: @echo "***********************************************************" @echo "** Installing jsCalendar plugin, please wait... **" @echo "***********************************************************" rm -rf /tmp/invenio-jscalendar-plugin mkdir /tmp/invenio-jscalendar-plugin (cd /tmp/invenio-jscalendar-plugin && \ wget 'http://www.dynarch.com/static/jscalendar-1.0.zip' && \ unzip -u jscalendar-1.0.zip && \ mkdir -p ${prefix}/var/www/jsCalendar && \ cp jscalendar-1.0/img.gif ${prefix}/var/www/jsCalendar/jsCalendar.gif && \ cp jscalendar-1.0/calendar.js ${prefix}/var/www/jsCalendar/ && \ cp jscalendar-1.0/calendar-setup.js ${prefix}/var/www/jsCalendar/ && \ cp jscalendar-1.0/lang/calendar-en.js ${prefix}/var/www/jsCalendar/ && \ cp jscalendar-1.0/calendar-blue.css ${prefix}/var/www/jsCalendar/) rm -fr /tmp/invenio-jscalendar-plugin @echo "***********************************************************" @echo "** The jsCalendar plugin was successfully installed. **" @echo "***********************************************************" uninstall-jscalendar-plugin: @rm -rvf ${prefix}/var/www/jsCalendar @echo "***********************************************************" @echo "** The jsCalendar plugin was successfully uninstalled. **" @echo "***********************************************************" install-jquery-plugins: @echo "***********************************************************" @echo "** Installing various jQuery plugins, please wait... **" @echo "***********************************************************" mkdir -p ${prefix}/var/www/js mkdir -p $(prefix)/var/www/css (cd ${prefix}/var/www/js && \ wget -O jquery.min.js http://invenio-software.org/download/jquery/jquery-1.7.1.min.js && \ wget -N http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.17/jquery-ui.min.js && \ wget -N http://invenio-software.org/download/jquery/v1.5/js/jquery.jeditable.mini.js && \ wget -N https://raw.githubusercontent.com/malsup/form/3.51/jquery.form.js --no-check-certificate && \ wget -N http://jquery-multifile-plugin.googlecode.com/svn-history/r54/trunk/jquery.MultiFile.js && \ wget -O jquery.tablesorter.zip http://invenio-software.org/download/jquery/jquery.tablesorter.20111208.zip && \ wget -O uploadify.zip http://invenio-software.org/download/jquery/uploadify-v2.1.4.zip && \ wget -N http://cdn.datatables.net/1.10.4/js/jquery.dataTables.min.js && \ wget -N http://invenio-software.org/download/jquery/jquery.bookmark.package-1.4.0.zip && \ unzip -u jquery.tablesorter.zip -d tablesorter && \ rm jquery.tablesorter.zip && \ rm -rf uploadify && \ unzip -u uploadify.zip -d uploadify && \ wget -N http://invenio-software.org/download/jquery/flot-0.6.zip && \ wget -O jquery-ui-timepicker-addon.js http://invenio-software.org/download/jquery/jquery-ui-timepicker-addon-1.0.3.js && \ unzip -u flot-0.6.zip && \ mv flot/jquery.flot.selection.min.js flot/jquery.flot.min.js flot/excanvas.min.js ./ && \ rm flot-0.6.zip && rm -r flot && \ mv uploadify/swfobject.js ./ && \ mv uploadify/cancel.png uploadify/uploadify.css uploadify/uploadify.allglyphs.swf uploadify/uploadify.fla uploadify/uploadify.swf ../img/ && \ mv uploadify/jquery.uploadify.v2.1.4.min.js ./jquery.uploadify.min.js && \ rm uploadify.zip && rm -r uploadify && \ wget -N https://github.com/douglascrockford/JSON-js/raw/master/json2.js --no-check-certificate && \ wget -O jquery.hotkeys.js http://invenio-software.org/download/jquery/jquery.hotkeys-0.8.js && \ wget -N http://invenio-software.org/download/jquery/jquery.treeview.zip && \ unzip -u jquery.treeview.zip -d jquery-treeview && \ rm jquery.treeview.zip && \ wget -N http://invenio-software.org/download/jquery/v1.5/js/jquery.ajaxPager.js && \ unzip -u jquery.bookmark.package-1.4.0.zip && \ rm -f jquery.bookmark.ext.* bookmarks-big.png bookmarkBasic.html jquery.bookmark.js jquery.bookmark.pack.js && \ mv bookmarks.png ../img/ && \ mv jquery.bookmark.css ../css/ && \ rm -f jquery.bookmark.package-1.4.0.zip && \ mkdir -p ${prefix}/var/www/img && \ cd ${prefix}/var/www/img && \ wget -r -np -nH --cut-dirs=4 -A "png,css" -P jquery-ui/themes http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/base/ && \ wget -r -np -nH --cut-dirs=4 -A "png,css" -P jquery-ui/themes http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/smoothness/ && \ wget -r -np -nH --cut-dirs=4 -A "png,css" -P jquery-ui/themes http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/redmond/ && \ wget -O datatables_jquery-ui.css https://raw.githubusercontent.com/DataTables/DataTables/1.10.0/media/css/demo_table_jui.css --no-check-certificate && \ wget -N http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/redmond/jquery-ui.css && \ wget -N http://jquery-ui.googlecode.com/svn/tags/1.8.17/demos/images/calendar.gif && \ wget -r -np -nH --cut-dirs=5 -A "png" http://jquery-ui.googlecode.com/svn/tags/1.8.17/themes/redmond/images/) @echo "***********************************************************" @echo "** The jQuery plugins were successfully installed. **" @echo "***********************************************************" uninstall-jquery-plugins: (cd ${prefix}/var/www/js && \ rm -f jquery.min.js && \ rm -f jquery.MultiFile.pack.js && \ rm -f jquery.jeditable.mini.js && \ rm -f jquery.flot.selection.min.js && \ rm -f jquery.flot.min.js && \ rm -f excanvas.min.js && \ rm -f jquery-ui-timepicker-addon.min.js && \ rm -f json2.js && \ rm -f jquery.uploadify.min.js && \ rm -rf tablesorter && \ rm -rf jquery-treeview && \ rm -f jquery.ajaxPager.js && \ rm -f jquery.form.js && \ rm -f jquery.dataTables.min.js && \ rm -f ui.core.js && \ rm -f jquery.bookmark.min.js && \ rm -f jquery.hotkeys.js && \ rm -f jquery.tablesorter.min.js && \ rm -f jquery-ui-1.7.3.custom.min.js && \ rm -f jquery.metadata.js && \ rm -f jquery-latest.js && \ rm -f jquery-ui.min.js) (cd ${prefix}/var/www/img && \ rm -f cancel.png uploadify.css uploadify.swf uploadify.allglyphs.swf uploadify.fla && \ rm -f datatables_jquery-ui.css \ rm -f bookmarks.png) && \ (cd ${prefix}/var/www/css && \ rm -f jquery.bookmark.css) @echo "***********************************************************" @echo "** The jquery plugins were successfully uninstalled. **" @echo "***********************************************************" install-ckeditor-plugin: @echo "***********************************************************" @echo "** Installing CKeditor plugin, please wait... **" @echo "***********************************************************" rm -rf ${prefix}/lib/python/invenio/ckeditor/ rm -rf /tmp/invenio-ckeditor-plugin mkdir /tmp/invenio-ckeditor-plugin (cd /tmp/invenio-ckeditor-plugin && \ wget 'http://invenio-software.org/download/ckeditor/$(CKEDITOR)' && \ unzip -u -d ${prefix}/var/www $(CKEDITOR)) && \ find ${prefix}/var/www/ckeditor/ -depth -name '_*' -exec rm -rf {} \; && \ find ${prefix}/var/www/ckeditor/ckeditor* -maxdepth 0 ! -name "ckeditor.js" -exec rm -r {} \; && \ rm -fr /tmp/invenio-ckeditor-plugin @echo "* Installing Invenio-specific CKeditor config..." (cd $(top_srcdir)/modules/webstyle/etc && make install) @echo "***********************************************************" @echo "** The CKeditor plugin was successfully installed. **" @echo "** Please do not forget to properly set the option **" @echo "** CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR in invenio.conf. **" @echo "***********************************************************" uninstall-ckeditor-plugin: @rm -rvf ${prefix}/var/www/ckeditor @rm -rvf ${prefix}/lib/python/invenio/ckeditor @echo "***********************************************************" @echo "** The CKeditor plugin was successfully uninstalled. **" @echo "***********************************************************" install-pdfa-helper-files: @echo "***********************************************************" @echo "** Installing PDF/A helper files, please wait... **" @echo "***********************************************************" wget 'http://invenio-software.org/download/invenio-demo-site-files/ISOCoatedsb.icc' -O ${prefix}/etc/websubmit/file_converter_templates/ISOCoatedsb.icc @echo "***********************************************************" @echo "** The PDF/A helper files were successfully installed. **" @echo "***********************************************************" install-mediaelement: @echo "***********************************************************" @echo "** MediaElement.js, please wait... **" @echo "***********************************************************" rm -rf /tmp/mediaelement mkdir /tmp/mediaelement wget 'http://github.com/johndyer/mediaelement/zipball/master' -O '/tmp/mediaelement/mediaelement.zip' --no-check-certificate unzip -u -d '/tmp/mediaelement' '/tmp/mediaelement/mediaelement.zip' rm -rf ${prefix}/var/www/mediaelement mkdir ${prefix}/var/www/mediaelement mv /tmp/mediaelement/johndyer-mediaelement-*/build/* ${prefix}/var/www/mediaelement rm -rf /tmp/mediaelement @echo "***********************************************************" @echo "** MediaElement.js was successfully installed. **" @echo "***********************************************************" uninstall-pdfa-helper-files: rm -f ${prefix}/etc/websubmit/file_converter_templates/ISOCoatedsb.icc @echo "***********************************************************" @echo "** The PDF/A helper files were successfully uninstalled. **" @echo "***********************************************************" #Solrutils allows automatic installation, running and searching of an external Solr index. install-solrutils: @echo "***********************************************************" @echo "** Installing Solrutils and solr, please wait... **" @echo "***********************************************************" cd $(prefix)/lib && \ if test -d apache-solr*; then echo A solr directory already exists in `pwd` . \ Please remove it manually, if you are sure it is not needed; exit 2; fi ; \ if test -f apache-solr*; then echo solr tarball already exists in `pwd` . \ Please remove it manually.; exit 2; fi ; \ wget http://archive.apache.org/dist/lucene/solr/3.1.0/apache-solr-3.1.0.tgz && \ tar -xzf apache-solr-3.1.0.tgz && \ rm apache-solr-3.1.0.tgz cd $(solrdir)/contrib/ ;\ wget http://mirrors.ibiblio.org/pub/mirrors/maven2/com/jcraft/jzlib/1.0.7/jzlib-1.0.7.jar && \ cd $(solrdir)/contrib/ ;\ jar -xf ../example/webapps/solr.war WEB-INF/lib/lucene-core-3.1.0.jar ; \ if test -d basic-lucene-libs; then rm -rf basic-lucene-libs; fi ; \ mv WEB-INF/lib/ basic-lucene-libs ; \ cp $(solrutils_dir)/schema.xml $(solrdir)/example/solr/conf/ cp $(solrutils_dir)/solrconfig.xml $(solrdir)/example/solr/conf/ cd $(solrutils_dir) && \ javac -classpath $(CLASSPATH) -d $(solrdir)/contrib @$(solrutils_dir)/java_sources.txt && \ cd $(solrdir)/contrib/ && \ jar -cf invenio-solr.jar org/invenio_software/solr/*class update-v0.99.0-tables: cat $(top_srcdir)/modules/miscutil/sql/tabcreate.sql | grep -v 'INSERT INTO upgrade' | ${prefix}/bin/dbexec echo "DROP TABLE IF EXISTS oaiREPOSITORY;" | ${prefix}/bin/dbexec echo "ALTER TABLE bibdoc ADD COLUMN more_info mediumblob NULL default NULL;" | ${prefix}/bin/dbexec echo "ALTER TABLE schTASK ADD COLUMN priority tinyint(4) NOT NULL default 0;" | ${prefix}/bin/dbexec echo "ALTER TABLE schTASK ADD KEY priority (priority);" | ${prefix}/bin/dbexec echo "ALTER TABLE rnkCITATIONDATA DROP PRIMARY KEY;" | ${prefix}/bin/dbexec echo "ALTER TABLE rnkCITATIONDATA ADD PRIMARY KEY (id);" | ${prefix}/bin/dbexec echo "ALTER TABLE rnkCITATIONDATA CHANGE id id mediumint(8) unsigned NOT NULL auto_increment;" | ${prefix}/bin/dbexec echo "ALTER TABLE rnkCITATIONDATA ADD UNIQUE KEY object_name (object_name);" | ${prefix}/bin/dbexec echo "ALTER TABLE sbmPARAMETERS CHANGE value value text NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE sbmAPPROVAL ADD note text NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE hstDOCUMENT CHANGE docsize docsize bigint(15) unsigned NOT NULL;" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtACTIONHISTORY CHANGE client_host client_host int(10) unsigned default NULL;" | ${prefix}/bin/dbexec update-v0.99.1-tables: @echo "Nothing to do; table structure did not change between v0.99.1 and v0.99.2." update-v0.99.2-tables: @echo "Nothing to do; table structure did not change between v0.99.2 and v0.99.3." update-v0.99.3-tables: @echo "Nothing to do; table structure did not change between v0.99.3 and v0.99.4." update-v0.99.4-tables: @echo "Nothing to do; table structure did not change between v0.99.4 and v0.99.5." update-v0.99.5-tables: @echo "Nothing to do; table structure did not change between v0.99.5 and v0.99.6." update-v0.99.6-tables: @echo "Nothing to do; table structure did not change between v0.99.6 and v0.99.7." update-v0.99.7-tables: @echo "Nothing to do; table structure did not change between v0.99.7 and v0.99.8." update-v0.99.8-tables: @echo "Nothing to do; table structure did not change between v0.99.8 and v0.99.9." update-v0.99.9-tables: # from v0.99.9 to v1.0.0-rc0 echo "RENAME TABLE oaiARCHIVE TO oaiREPOSITORY;" | ${prefix}/bin/dbexec cat $(top_srcdir)/modules/miscutil/sql/tabcreate.sql | grep -v 'INSERT INTO upgrade' | ${prefix}/bin/dbexec echo "INSERT INTO knwKB (id,name,description,kbtype) SELECT id,name,description,'' FROM fmtKNOWLEDGEBASES;" | ${prefix}/bin/dbexec echo "INSERT INTO knwKBRVAL (id,m_key,m_value,id_knwKB) SELECT id,m_key,m_value,id_fmtKNOWLEDGEBASES FROM fmtKNOWLEDGEBASEMAPPINGS;" | ${prefix}/bin/dbexec echo "ALTER TABLE sbmPARAMETERS CHANGE name name varchar(40) NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE bibdoc CHANGE docname docname varchar(250) COLLATE utf8_bin NOT NULL default 'file';" | ${prefix}/bin/dbexec echo "ALTER TABLE bibdoc CHANGE status status text NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE bibdoc ADD COLUMN text_extraction_date datetime NOT NULL default '0000-00-00';" | ${prefix}/bin/dbexec echo "ALTER TABLE collection DROP COLUMN restricted;" | ${prefix}/bin/dbexec echo "ALTER TABLE schTASK CHANGE host host varchar(255) NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE hstTASK CHANGE host host varchar(255) NOT NULL default '';" | ${prefix}/bin/dbexec echo "ALTER TABLE bib85x DROP INDEX kv, ADD INDEX kv (value(100));" | ${prefix}/bin/dbexec echo "UPDATE clsMETHOD SET location='http://invenio-software.org/download/invenio-demo-site-files/HEP.rdf' WHERE name='HEP' AND location='';" | ${prefix}/bin/dbexec echo "UPDATE clsMETHOD SET location='http://invenio-software.org/download/invenio-demo-site-files/NASA-subjects.rdf' WHERE name='NASA-subjects' AND location='';" | ${prefix}/bin/dbexec echo "UPDATE accACTION SET name='runoairepository', description='run oairepositoryupdater task' WHERE name='runoaiarchive';" | ${prefix}/bin/dbexec echo "UPDATE accACTION SET name='cfgoaiharvest', description='configure OAI Harvest' WHERE name='cfgbibharvest';" | ${prefix}/bin/dbexec echo "ALTER TABLE accARGUMENT CHANGE value value varchar(255);" | ${prefix}/bin/dbexec echo "UPDATE accACTION SET allowedkeywords='doctype,act,categ' WHERE name='submit';" | ${prefix}/bin/dbexec echo "INSERT INTO accARGUMENT(keyword,value) VALUES ('categ','*');" | ${prefix}/bin/dbexec echo "INSERT INTO accROLE_accACTION_accARGUMENT(id_accROLE,id_accACTION,id_accARGUMENT,argumentlistid) SELECT DISTINCT raa.id_accROLE,raa.id_accACTION,accARGUMENT.id,raa.argumentlistid FROM accROLE_accACTION_accARGUMENT as raa JOIN accACTION on id_accACTION=accACTION.id,accARGUMENT WHERE accACTION.name='submit' and accARGUMENT.keyword='categ' and accARGUMENT.value='*';" | ${prefix}/bin/dbexec echo "UPDATE accACTION SET allowedkeywords='name,with_editor_rights' WHERE name='cfgwebjournal';" | ${prefix}/bin/dbexec echo "INSERT INTO accARGUMENT(keyword,value) VALUES ('with_editor_rights','yes');" | ${prefix}/bin/dbexec echo "INSERT INTO accROLE_accACTION_accARGUMENT(id_accROLE,id_accACTION,id_accARGUMENT,argumentlistid) SELECT DISTINCT raa.id_accROLE,raa.id_accACTION,accARGUMENT.id,raa.argumentlistid FROM accROLE_accACTION_accARGUMENT as raa JOIN accACTION on id_accACTION=accACTION.id,accARGUMENT WHERE accACTION.name='cfgwebjournal' and accARGUMENT.keyword='with_editor_rights' and accARGUMENT.value='yes';" | ${prefix}/bin/dbexec echo "ALTER TABLE bskEXTREC CHANGE id id int(15) unsigned NOT NULL auto_increment;" | ${prefix}/bin/dbexec echo "ALTER TABLE bskEXTREC ADD external_id int(15) NOT NULL default '0';" | ${prefix}/bin/dbexec echo "ALTER TABLE bskEXTREC ADD collection_id int(15) unsigned NOT NULL default '0';" | ${prefix}/bin/dbexec echo "ALTER TABLE bskEXTREC ADD original_url text;" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD status char(2) NOT NULL default 'ok';" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD KEY status (status);" | ${prefix}/bin/dbexec echo "INSERT INTO sbmALLFUNCDESCR VALUES ('Move_Photos_to_Storage','Attach/edit the pictures uploaded with the \"create_photos_manager_interface()\" function');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFIELDDESC VALUES ('Upload_Photos',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'\"\"\"\r\nThis is an example of element that creates a photos upload interface.\r\nClone it, customize it and integrate it into your submission. Then add function \r\n\'Move_Photos_to_Storage\' to your submission functions list, in order for files \r\nuploaded with this interface to be attached to the record. More information in \r\nthe WebSubmit admin guide.\r\n\"\"\"\r\n\r\nfrom invenio.websubmit_functions.ParamFile import ParamFromFile\r\nfrom invenio.websubmit_functions.Move_Photos_to_Storage import read_param_file, create_photos_manager_interface, get_session_id\r\n\r\n# Retrieve session id\r\ntry:\r\n # User info is defined only in MBI/MPI actions...\r\n session_id = get_session_id(None, uid, user_info) \r\nexcept:\r\n session_id = get_session_id(req, uid, {})\r\n\r\n# Retrieve context\r\nindir = curdir.split(\'/\')[-3]\r\ndoctype = curdir.split(\'/\')[-2]\r\naccess = curdir.split(\'/\')[-1]\r\n\r\n# Get the record ID, if any\r\nsysno = ParamFromFile(\"%s/%s\" % (curdir,\'SN\')).strip()\r\n\r\n\"\"\"\r\nModify below the configuration of the photos manager interface.\r\nNote: \'can_reorder_photos\' parameter is not yet fully taken into consideration\r\n\r\nDocumentation of the function is available by running:\r\necho -e \'from invenio.websubmit_functions.Move_Photos_to_Storage import create_photos_manager_interface as f\\nprint f.__doc__\' | python\r\n\"\"\"\r\ntext += create_photos_manager_interface(sysno, session_id, uid,\r\n doctype, indir, curdir, access,\r\n can_delete_photos=True,\r\n can_reorder_photos=True,\r\n can_upload_photos=True,\r\n editor_width=700,\r\n editor_height=400,\r\n initial_slider_value=100,\r\n max_slider_value=200,\r\n min_slider_value=80)','0000-00-00','0000-00-00',NULL,NULL,0);" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Photos_to_Storage','iconsize');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFIELDDESC VALUES ('Upload_Files',NULL,'','R',NULL,NULL,NULL,NULL,NULL,'\"\"\"\r\nThis is an example of element that creates a file upload interface.\r\nClone it, customize it and integrate it into your submission. Then add function \r\n\'Move_Uploaded_Files_to_Storage\' to your submission functions list, in order for files \r\nuploaded with this interface to be attached to the record. More information in \r\nthe WebSubmit admin guide.\r\n\"\"\"\r\nfrom invenio.websubmit_managedocfiles import create_file_upload_interface\r\nfrom invenio.websubmit_functions.Shared_Functions import ParamFromFile\r\n\r\nindir = ParamFromFile(os.path.join(curdir, \'indir\'))\r\ndoctype = ParamFromFile(os.path.join(curdir, \'doctype\'))\r\naccess = ParamFromFile(os.path.join(curdir, \'access\'))\r\ntry:\r\n sysno = int(ParamFromFile(os.path.join(curdir, \'SN\')).strip())\r\nexcept:\r\n sysno = -1\r\nln = ParamFromFile(os.path.join(curdir, \'ln\'))\r\n\r\n\"\"\"\r\nRun the following to get the list of parameters of function \'create_file_upload_interface\':\r\necho -e \'from invenio.websubmit_managedocfiles import create_file_upload_interface as f\\nprint f.__doc__\' | python\r\n\"\"\"\r\ntext = create_file_upload_interface(recid=sysno,\r\n print_outside_form_tag=False,\r\n include_headers=True,\r\n ln=ln,\r\n doctypes_and_desc=[(\'main\',\'Main document\'),\r\n (\'additional\',\'Figure, schema, etc.\')],\r\n can_revise_doctypes=[\'*\'],\r\n can_describe_doctypes=[\'main\'],\r\n can_delete_doctypes=[\'additional\'],\r\n can_rename_doctypes=[\'main\'],\r\n sbm_indir=indir, sbm_doctype=doctype, sbm_access=access)[1]\r\n','0000-00-00','0000-00-00',NULL,NULL,0);" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Uploaded_Files_to_Storage','forceFileRevision');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmALLFUNCDESCR VALUES ('Create_Upload_Files_Interface','Display generic interface to add/revise/delete files. To be used before function \"Move_Uploaded_Files_to_Storage\"');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmALLFUNCDESCR VALUES ('Move_Uploaded_Files_to_Storage','Attach files uploaded with \"Create_Upload_Files_Interface\"')" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Revised_Files_to_Storage','elementNameToDoctype');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Revised_Files_to_Storage','createIconDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Revised_Files_to_Storage','createRelatedFormats');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Revised_Files_to_Storage','iconsize');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Revised_Files_to_Storage','keepPreviousVersionDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmALLFUNCDESCR VALUES ('Move_Revised_Files_to_Storage','Revise files initially uploaded with \"Move_Files_to_Storage\"')" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','maxsize');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','minsize');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','doctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','restrictions');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canDeleteDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canReviseDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canDescribeDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canCommentDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canKeepDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canAddFormatDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canRestrictDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canRenameDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','canNameNewFiles');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','createRelatedFormats');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','keepDefault');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','showLinks');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','fileLabel');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','filenameLabel');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','descriptionLabel');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','commentLabel');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','restrictionLabel');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','startDoc');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','endDoc');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','defaultFilenameDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Create_Upload_Files_Interface','maxFilesDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Uploaded_Files_to_Storage','iconsize');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Uploaded_Files_to_Storage','createIconDoctypes');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','nblength');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Second_Report_Number_Generation','2nd_nb_length');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Get_Recid','record_search_pattern');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmALLFUNCDESCR VALUES ('Move_FCKeditor_Files_to_Storage','Transfer files attached to the record with the FCKeditor');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_FCKeditor_Files_to_Storage','input_fields');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Stamp_Uploaded_Files','layer');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Stamp_Replace_Single_File_Approval','layer');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Stamp_Replace_Single_File_Approval','switch_file');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Stamp_Uploaded_Files','switch_file');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Files_to_Storage','paths_and_restrictions');" | ${prefix}/bin/dbexec echo "INSERT INTO sbmFUNDESC VALUES ('Move_Files_to_Storage','paths_and_doctypes');" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD round_name varchar(255) NOT NULL default ''" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD restriction varchar(50) NOT NULL default ''" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD in_reply_to_id_cmtRECORDCOMMENT int(15) unsigned NOT NULL default '0'" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD KEY in_reply_to_id_cmtRECORDCOMMENT (in_reply_to_id_cmtRECORDCOMMENT);" | ${prefix}/bin/dbexec echo "ALTER TABLE bskRECORDCOMMENT ADD in_reply_to_id_bskRECORDCOMMENT int(15) unsigned NOT NULL default '0'" | ${prefix}/bin/dbexec echo "ALTER TABLE bskRECORDCOMMENT ADD KEY in_reply_to_id_bskRECORDCOMMENT (in_reply_to_id_bskRECORDCOMMENT);" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD reply_order_cached_data blob NULL default NULL;" | ${prefix}/bin/dbexec echo "ALTER TABLE bskRECORDCOMMENT ADD reply_order_cached_data blob NULL default NULL;" | ${prefix}/bin/dbexec echo "ALTER TABLE cmtRECORDCOMMENT ADD INDEX (reply_order_cached_data(40));" | ${prefix}/bin/dbexec echo "ALTER TABLE bskRECORDCOMMENT ADD INDEX (reply_order_cached_data(40));" | ${prefix}/bin/dbexec echo -e 'from invenio.webcommentadminlib import migrate_comments_populate_threads_index;\ migrate_comments_populate_threads_index()' | $(PYTHON) echo -e 'from invenio.access_control_firerole import repair_role_definitions;\ repair_role_definitions()' | $(PYTHON) CLEANFILES = *~ *.pyc *.tmp diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..5f90d314a --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# This Vagrant configuration is suitable for Invenio demo site installation as +# governed by `.inveniorc`. It uses separate dedicated VMs for various services +# in order to better emulate production environment conditions. You can install +# an Invenio demo site by running: +# +# $ vagrant up --no-parallel +# $ vagrant ssh web -c 'source .inveniorc && /vagrant/scripts/create-instance.sh' +# $ vagrant ssh web -c 'source .inveniorc && /vagrant/scripts/populate-instance.sh' +# $ firefox http://192.168.50.10/record/1 +# $ vagrant ssh web -c 'source .inveniorc && sudo -u www-data /opt/invenio/bin/inveniocfg --run-unit-tests' +# $ vagrant ssh web -c 'source .inveniorc && sudo -u www-data /opt/invenio/bin/inveniocfg --run-regression-tests --yes-i-know' + +#OS = 'hfm4/centos6' +OS = 'ubuntu/precise64' + +Vagrant.configure("2") do |config| + + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + + config.vm.define "web" do |web| + web.vm.box = OS + web.vm.hostname = 'web' + web.vm.provision "file", source: ".inveniorc", destination: ".inveniorc" + web.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-web.sh", privileged: false + web.vm.network "forwarded_port", guest: 80, host: 80 + web.vm.network "forwarded_port", guest: 443, host: 443 + web.vm.network "private_network", ip: ENV.fetch('INVENIO_WEB_HOST','192.168.50.10') + web.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", "3072"] + vb.customize ["modifyvm", :id, "--cpus", 2] + end + end + + config.vm.define "mysql" do |mysql| + mysql.vm.box = OS + mysql.vm.hostname = 'mysql' + mysql.vm.provision "file", source: ".inveniorc", destination: ".inveniorc" + mysql.vm.provision "shell", inline: "source .inveniorc && /vagrant/scripts/provision-mysql.sh", privileged: false + mysql.vm.network "private_network", ip: ENV.fetch('INVENIO_MYSQL_HOST','192.168.50.11') + end + +end diff --git a/configure.ac b/configure.ac index 122b4f3b2..668c795c9 100644 --- a/configure.ac +++ b/configure.ac @@ -1,926 +1,927 @@ ## This file is part of Invenio. -## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2015 CERN. +## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2015, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. ## This is Invenio main configure.ac file. If you change this ## file, then please run "autoreconf" to regenerate the "configure" ## script. ## Initialize autoconf and automake: AC_INIT([invenio], m4_esyscmd([./git-version-gen .tarball-version]), [info@invenio-software.org]) AM_INIT_AUTOMAKE([tar-ustar]) ## By default we shall install into /opt/invenio. (Do not use ## AC_PREFIX_DEFAULT for this, because it would not work well with ## the localstatedir hack below.) test "${prefix}" = NONE && prefix=/opt/invenio ## Remove eventual trailing slashes from the prefix value: test "${prefix%/}" != "" && prefix=${prefix%/} ## Check for install: AC_PROG_INSTALL ## Check for gettext support: AM_GNU_GETTEXT(external) AM_GNU_GETTEXT_VERSION(0.14.4) ## Check for MySQL client: AC_MSG_CHECKING(for mysql) AC_ARG_WITH(mysql, AC_HELP_STRING([--with-mysql], [path to a specific MySQL binary (optional)]), MYSQL=${withval}) if test -n "$MYSQL"; then AC_MSG_RESULT($MYSQL) else AC_PATH_PROG(MYSQL, mysql) if test -z "$MYSQL"; then AC_MSG_ERROR([ MySQL command-line client was not found in your PATH. Please install it first. Available from .]) fi fi ## Check for Python: AC_MSG_CHECKING(for python) AC_ARG_WITH(python, AC_HELP_STRING([--with-python], [path to a specific Python binary (optional)]), PYTHON=${withval}) if test -n "$PYTHON"; then AC_MSG_RESULT($PYTHON) else AC_PATH_PROG(PYTHON, python) if test -z "$PYTHON"; then AC_MSG_ERROR([ Python was not found in your PATH. Please either install it in your PATH or specify --with-python configure option. Python is available from .]) fi fi ## Check for OpenOffice.org Python binary: AC_MSG_CHECKING(for OpenOffice.org Python binary) AC_ARG_WITH(openoffice-python, AC_HELP_STRING([--with-openoffice-python], [path to a specific OpenOffice.org Python binary (optional)]), OPENOFFICE_PYTHON=`which ${withval}`) if test -z "$OPENOFFICE_PYTHON"; then OPENOFFICE_PYTHON=`locate -l 1 -r "o.*office/program/python$"` OPENOFFICE_PYTHON="$PYTHON $OPENOFFICE_PYTHON" if test -n "$OPENOFFICE_PYTHON" && ($OPENOFFICE_PYTHON -c "import uno" 2> /dev/null); then AC_MSG_RESULT($OPENOFFICE_PYTHON) else AC_MSG_WARN([ You have not specified the path ot the OpenOffice.org Python binary. OpenOffice.org and Microsoft Office document conversion and fulltext indexing will not be available. We recommend you to install OpenOffice.org first and to rerun the configure script. OpenOffice.org is available from .]) fi elif ($OPENOFFICE_PYTHON -c "import uno" 2> /dev/null); then AC_MSG_RESULT($OPENOFFICE_PYTHON) else AC_MSG_ERROR([ The specified OpenOffice.org Python binary is not correctly configured. Please specify the correct path to the specific OpenOffice Python binary (OpenOffice.org is available from ).]) fi ## Check for Python version and modules: AC_MSG_CHECKING(for required Python modules) $PYTHON ${srcdir}/configure-tests.py if test $? -ne 0; then AC_MSG_ERROR([Please fix the above Python problem before continuing.]) fi AC_MSG_RESULT(found) ## Check for PHP: AC_PATH_PROG(PHP, php) ## Check for gzip: AC_PATH_PROG(GZIP, gzip) if test -z "$GZIP"; then AC_MSG_WARN([ Gzip was not found in your PATH. It is used in the WebSubmit module to compress the data submitted in an archive. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. Gzip is available from .]) fi ## Check for gunzip: AC_PATH_PROG(GUNZIP, gunzip) if test -z "$GUNZIP"; then AC_MSG_WARN([ Gunzip was not found in your PATH. It is used in the WebSubmit module to correctly deal with submitted compressed files. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. Gunzip is available from .]) fi ## Check for tar: AC_PATH_PROG(TAR, tar) if test -z "$TAR"; then AC_MSG_WARN([ Tar was not found in your PATH. It is used in the WebSubmit module to pack the submitted data into an archive. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. Tar is available from .]) fi ## Check for wget: AC_PATH_PROG(WGET, wget) if test -z "$WGET"; then AC_MSG_WARN([ wget was not found in your PATH. It is used for the fulltext file retrieval. You can continue without it but we recomend you to install it first and to rerun the configure script. wget is available from .]) fi ## Check for md5sum: AC_PATH_PROG(MD5SUM, md5sum) if test -z "$MD5SUM"; then AC_MSG_WARN([ md5sum was not found in your PATH. It is used for the fulltext file checksum verification. You can continue without it but we recomend you to install it first and to rerun the configure script. md5sum is available from .]) fi ## Check for ps2pdf: AC_PATH_PROG(PS2PDF, ps2pdf) if test -z "$PS2PDF"; then AC_MSG_WARN([ ps2pdf was not found in your PATH. It is used in the WebSubmit module to convert submitted PostScripts into PDF. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. ps2pdf is available from .]) fi ## Check for pdflatex: AC_PATH_PROG(PDFLATEX, pdflatex) if test -z "$PDFLATEX"; then AC_MSG_WARN([ pdflatex was not found in your PATH. It is used in the WebSubmit module to stamp PDF files. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script.]) fi ## Check for tiff2pdf: AC_PATH_PROG(TIFF2PDF, tiff2pdf) if test -z "$TIFF2PDF"; then AC_MSG_WARN([ tiff2pdf was not found in your PATH. It is used in the WebSubmit module to convert submitted TIFF file into PDF. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. tiff2pdf is available from .]) fi ## Check for gs: AC_PATH_PROG(GS, gs) if test -z "$GS"; then AC_MSG_WARN([ gs was not found in your PATH. It is used in the WebSubmit module to convert submitted PostScripts into PDF. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. gs is available from .]) fi ## Check for pdftotext: AC_PATH_PROG(PDFTOTEXT, pdftotext) if test -z "$PDFTOTEXT"; then AC_MSG_WARN([ pdftotext was not found in your PATH. It is used for the fulltext indexation of PDF files. You can continue without it but you may miss fulltext searching capability of Invenio. We recomend you to install it first and to rerun the configure script. pdftotext is available from . ]) fi ## Check for pdftotext: AC_PATH_PROG(PDFINFO, pdfinfo) if test -z "$PDFINFO"; then AC_MSG_WARN([ pdfinfo was not found in your PATH. It is used for gathering information on PDF files. You can continue without it but you may miss this feature of Invenio. We recomend you to install it first and to rerun the configure script. pdftotext is available from . ]) fi ## Check for pdftk: AC_PATH_PROG(PDFTK, pdftk) if test -z "$PDFTK"; then AC_MSG_WARN([ pdftk was not found in your PATH. It is used for the fulltext file stamping. You can continue without it but you may miss this feature of Invenio. We recomend you to install it first and to rerun the configure script. pdftk is available from . ]) fi ## Check for pdf2ps: AC_PATH_PROG(PDF2PS, pdf2ps) if test -z "$PDF2PS"; then AC_MSG_WARN([ pdf2ps was not found in your PATH. It is used in the WebSubmit module to convert submitted PDFs into PostScript. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. pdf2ps is available from .]) fi ## Check for pdftops: AC_PATH_PROG(PDFTOPS, pdftops) if test -z "$PDFTOPS"; then AC_MSG_WARN([ pdftops was not found in your PATH. It is used in the WebSubmit module to convert submitted PDFs into PostScript. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. pdftops is available from .]) fi ## Check for pdfopt: AC_PATH_PROG(PDFOPT, pdfopt) if test -z "$PDFOPT"; then AC_MSG_WARN([ pdfopt was not found in your PATH. It is used in the WebSubmit module to linearized submitted PDFs. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. pdfopt is available from .]) fi ## Check for pdfimages: AC_PATH_PROG(PDFTOPPM, pdftoppm) if test -z "$PDFTOPPM"; then AC_MSG_WARN([ pdftoppm was not found in your PATH. It is used in the WebSubmit module to extract images from PDFs for OCR. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. pdftoppm is available from .]) fi ## Check for pdfimages: AC_PATH_PROG(PAMFILE, pdftoppm) if test -z "$PAMFILE"; then AC_MSG_WARN([ pamfile was not found in your PATH. It is used in the WebSubmit module to retrieve the size of images extracted from PDFs for OCR. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. pamfile is available as part of the netpbm utilities from: .]) fi ## Check for ocroscript: AC_PATH_PROG(OCROSCRIPT, ocroscript) if test -z "$OCROSCRIPT"; then AC_MSG_WARN([ If you plan to run OCR on your PDFs, then please install ocroscript now. Otherwise you can safely continue. You have also an option to install ocroscript later and edit invenio-local.conf to let Invenio know the path to ocroscript. ocroscript is available as part of OCROpus from . NOTE: Since OCROpus is being actively developed and its api is continuosly changing, please install relase 0.3.1]) fi ## Check for pstotext: AC_PATH_PROG(PSTOTEXT, pstotext) if test -z "$PSTOTEXT"; then AC_MSG_WARN([ pstotext was not found in your PATH. It is used for the fulltext indexation of PDF and PostScript files. Please install pstotext. Otherwise you can safely continue. You have also an option to install pstotext later and edit invenio-local.conf to let Invenio know the path to pstotext. pstotext is available from . ]) fi ## Check for ps2ascii: AC_PATH_PROG(PSTOASCII, ps2ascii) if test -z "$PSTOASCII"; then AC_MSG_WARN([ ps2ascii was not found in your PATH. It is used for the fulltext indexation of PostScript files. Please install ps2ascii. Otherwise you can safely continue. You have also an option to install ps2ascii later and edit invenio-local.conf to let Invenio know the path to ps2ascii. ps2ascii is available from . ]) fi ## Check for any2djvu: AC_PATH_PROG(ANY2DJVU, any2djvu) if test -z "$ANY2DJVU"; then AC_MSG_WARN([ any2djvu was not found in your PATH. It is used in the WebSubmit module to convert documents to DJVU. Please install any2djvu. Otherwise you can safely continue. You have also an option to install any2djvu later and edit invenio-local.conf to let Invenio know the path to any2djvu. any2djvu is available from .]) fi ## Check for DJVUPS: AC_PATH_PROG(DJVUPS, djvups) if test -z "$DJVUPS"; then AC_MSG_WARN([ djvups was not found in your PATH. It is used in the WebSubmit module to convert documents from DJVU. Please install djvups. Otherwise you can safely continue. You have also an option to install djvups later and edit invenio-local.conf to let Invenio know the path to djvups. djvups is available from .]) fi ## Check for DJVUTXT: AC_PATH_PROG(DJVUTXT, djvutxt) if test -z "$DJVUTXT"; then AC_MSG_WARN([ djvutxt was not found in your PATH. It is used in the WebSubmit module to extract text from DJVU documents. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. djvutxt is available from .]) fi ## Check for file: AC_PATH_PROG(FILE, file) if test -z "$FILE"; then AC_MSG_WARN([ File was not found in your PATH. It is used in the WebSubmit module to check the validity of the submitted files. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. File is available from .]) fi ## Check for convert: AC_PATH_PROG(CONVERT, convert) if test -z "$CONVERT"; then AC_MSG_WARN([ Convert was not found in your PATH. It is used in the WebSubmit module to create an icon from a submitted picture. You can continue without it but you will miss some Invenio functionality. We recommend you to install it first and to rerun the configure script. Convert is available from .]) fi ## Check for CLISP: AC_MSG_CHECKING(for clisp) AC_ARG_WITH(clisp, AC_HELP_STRING([--with-clisp], [path to a specific CLISP binary (optional)]), CLISP=${withval}) if test -n "$CLISP"; then AC_MSG_RESULT($CLISP) else AC_PATH_PROG(CLISP, clisp) if test -z "$CLISP"; then AC_MSG_WARN([ GNU CLISP was not found in your PATH. It is used by the WebStat module to produce statistics about Invenio usage. (Alternatively, SBCL or CMUCL can be used instead of CLISP.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CMUCL nor SBCL) and to rerun the configure script. GNU CLISP is available from .]) fi fi ## Check for CMUCL: AC_MSG_CHECKING(for cmucl) AC_ARG_WITH(cmucl, AC_HELP_STRING([--with-cmucl], [path to a specific CMUCL binary (optional)]), CMUCL=${withval}) if test -n "$CMUCL"; then AC_MSG_RESULT($CMUCL) else AC_PATH_PROG(CMUCL, cmucl) if test -z "$CMUCL"; then AC_MSG_CHECKING(for lisp) # CMUCL can also be installed under `lisp' exec name AC_PATH_PROG(CMUCL, lisp) fi if test -z "$CMUCL"; then AC_MSG_WARN([ CMUCL was not found in your PATH. It is used by the WebStat module to produce statistics about Invenio usage. (Alternatively, CLISP or SBCL can be used instead of CMUCL.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP nor SBCL) and to rerun the configure script. CMUCL is available from .]) fi fi ## Check for SBCL: AC_MSG_CHECKING(for sbcl) AC_ARG_WITH(sbcl, AC_HELP_STRING([--with-sbcl], [path to a specific SBCL binary (optional)]), SBCL=${withval}) if test -n "$SBCL"; then AC_MSG_RESULT($SBCL) else AC_PATH_PROG(SBCL, sbcl) if test -z "$SBCL"; then AC_MSG_WARN([ SBCL was not found in your PATH. It is used by the WebStat module to produce statistics about Invenio usage. (Alternatively, CLISP or CMUCL can be used instead of SBCL.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP nor CMUCL) and to rerun the configure script. SBCL is available from .]) fi fi ## Check for gnuplot: AC_PATH_PROG(GNUPLOT, gnuplot) if test -z "$GNUPLOT"; then AC_MSG_WARN([ Gnuplot was not found in your PATH. It is used by the BibRank module to produce graphs about download and citation history. You can continue without it but you will miss these graphs. We recommend you to install it first and to rerun the configure script. Gnuplot is available from .]) fi ## Check for ffmpeg: AC_PATH_PROG(FFMPEG, ffmpeg) AC_PATH_PROG(FFPROBE, ffprobe) if test -z "$FFMPEG"; then AC_MSG_WARN([ FFmpeg was not found in your PATH. It is used by the BibEncode module to for video encoding. You can continue without but you will not be able to use BibEncode and no video submission workflows are thereby possible. We recommend you to install it first if you would like to support video submissions and to rerun the configure script. FFmpeg is available from .]) fi ## Check for mediainfo: AC_PATH_PROG(MEDIAINFO, mediainfo) if test -z "$MEDIAINFO"; then AC_MSG_WARN([ Mediainfo was not found in your PATH. It is used by the BibEncode module to for video encoding and media metadata handling. You can continue without but you will not be able to use BibEncode and no video submission workflows are thereby possible. We recommend you to install it first if you would like to support video submissions and to rerun the configure script. Mediainfo is available from .]) fi ## Check for ffmpeg ## Substitute variables: AC_SUBST(VERSION) AC_SUBST(OPENOFFICE_PYTHON) AC_SUBST(MYSQL) AC_SUBST(PYTHON) AC_SUBST(GZIP) AC_SUBST(GUNZIP) AC_SUBST(TAR) AC_SUBST(WGET) AC_SUBST(MD5SUM) AC_SUBST(PS2PDF) AC_SUBST(GS) AC_SUBST(PDFTOTEXT) AC_SUBST(PDFTK) AC_SUBST(PDF2PS) AC_SUBST(PDFTOPS) AC_SUBST(PDFOPT) AC_SUBST(PDFTOPPM) AC_SUBST(OCROSCRIPT) AC_SUBST(PSTOTEXT) AC_SUBST(PSTOASCII) AC_SUBST(ANY2DJVU) AC_SUBST(DJVUPS) AC_SUBST(DJVUTXT) AC_SUBST(FILE) AC_SUBST(CONVERT) AC_SUBST(GNUPLOT) AC_SUBST(CLISP) AC_SUBST(CMUCL) AC_SUBST(SBCL) AC_SUBST(CACHEDIR) AC_SUBST(FFMPEG) AC_SUBST(MEDIAINFO) AC_SUBST(FFPROBE) AC_SUBST(localstatedir, `eval echo "${localstatedir}"`) ## Define output files: AC_CONFIG_FILES([config.nice \ Makefile \ po/Makefile.in \ config/Makefile \ config/invenio-autotools.conf \ modules/Makefile \ modules/webauthorprofile/Makefile \ modules/webauthorprofile/lib/Makefile \ modules/webauthorprofile/bin/Makefile \ modules/webauthorprofile/bin/webauthorprofile \ modules/bibauthorid/Makefile \ modules/bibauthorid/bin/Makefile \ modules/bibauthorid/bin/bibauthorid \ modules/bibauthorid/doc/Makefile \ modules/bibauthorid/doc/admin/Makefile \ modules/bibauthorid/doc/hacking/Makefile \ modules/bibauthorid/lib/Makefile \ modules/bibauthorid/etc/Makefile \ modules/bibauthorid/etc/name_authority_files/Makefile \ modules/bibauthorid/web/Makefile \ modules/bibcatalog/Makefile \ modules/bibcatalog/doc/Makefile \ modules/bibcatalog/doc/admin/Makefile \ modules/bibcatalog/doc/hacking/Makefile modules/bibcatalog/lib/Makefile \ modules/bibcheck/Makefile \ modules/bibcheck/doc/Makefile \ modules/bibcheck/doc/admin/Makefile \ modules/bibcheck/doc/hacking/Makefile \ modules/bibcheck/etc/Makefile \ modules/bibcheck/web/Makefile \ modules/bibcheck/web/admin/Makefile \ modules/bibcirculation/Makefile \ modules/bibcirculation/bin/Makefile \ modules/bibcirculation/doc/Makefile \ modules/bibcirculation/doc/admin/Makefile \ modules/bibcirculation/doc/hacking/Makefile modules/bibcirculation/lib/Makefile \ modules/bibcirculation/web/Makefile \ modules/bibcirculation/web/admin/Makefile \ modules/bibclassify/Makefile \ modules/bibclassify/bin/Makefile \ modules/bibclassify/bin/bibclassify \ modules/bibclassify/doc/Makefile \ modules/bibclassify/doc/admin/Makefile \ modules/bibclassify/doc/hacking/Makefile \ modules/bibclassify/etc/Makefile \ modules/bibclassify/lib/Makefile \ modules/bibconvert/Makefile \ modules/bibconvert/bin/Makefile \ modules/bibconvert/bin/bibconvert \ modules/bibconvert/doc/Makefile \ modules/bibconvert/doc/admin/Makefile \ modules/bibconvert/doc/hacking/Makefile \ modules/bibconvert/etc/Makefile \ modules/bibconvert/lib/Makefile \ modules/bibdocfile/Makefile \ modules/bibdocfile/bin/bibdocfile \ modules/bibdocfile/bin/Makefile \ modules/bibdocfile/doc/Makefile \ modules/bibdocfile/doc/hacking/Makefile \ modules/bibdocfile/lib/Makefile \ modules/bibrecord/Makefile \ modules/bibrecord/bin/Makefile \ modules/bibrecord/bin/xmlmarc2textmarc \ modules/bibrecord/bin/textmarc2xmlmarc \ modules/bibrecord/bin/xmlmarclint \ modules/bibrecord/doc/Makefile \ modules/bibrecord/doc/admin/Makefile \ modules/bibrecord/doc/hacking/Makefile \ modules/bibrecord/etc/Makefile \ modules/bibrecord/lib/Makefile \ modules/refextract/Makefile \ modules/refextract/bin/Makefile \ modules/refextract/bin/refextract \ modules/refextract/doc/Makefile \ modules/refextract/doc/admin/Makefile \ modules/refextract/doc/hacking/Makefile \ modules/refextract/etc/Makefile \ modules/refextract/lib/Makefile \ modules/bibedit/Makefile \ modules/bibedit/bin/Makefile \ modules/bibedit/bin/bibedit \ modules/bibedit/doc/Makefile \ modules/bibedit/doc/admin/Makefile \ modules/bibedit/doc/hacking/Makefile \ modules/bibedit/etc/Makefile \ modules/bibedit/lib/Makefile \ modules/bibedit/web/Makefile \ modules/bibencode/Makefile \ modules/bibencode/bin/Makefile \ modules/bibencode/bin/bibencode \ modules/bibencode/lib/Makefile \ modules/bibencode/etc/Makefile \ modules/bibencode/www/Makefile \ modules/bibexport/Makefile \ modules/bibexport/bin/Makefile \ modules/bibexport/bin/bibexport \ modules/bibexport/doc/Makefile \ modules/bibexport/doc/admin/Makefile \ modules/bibexport/doc/hacking/Makefile modules/bibexport/etc/Makefile \ modules/bibexport/lib/Makefile \ modules/bibexport/web/Makefile \ modules/bibexport/web/admin/Makefile \ modules/bibformat/Makefile \ modules/bibformat/bin/Makefile \ modules/bibformat/bin/bibreformat \ modules/bibformat/doc/Makefile \ modules/bibformat/doc/admin/Makefile \ modules/bibformat/doc/hacking/Makefile \ modules/bibformat/etc/Makefile \ modules/bibformat/etc/format_templates/Makefile \ modules/bibformat/etc/output_formats/Makefile \ modules/bibformat/lib/Makefile \ modules/bibformat/lib/elements/Makefile \ modules/bibformat/web/Makefile \ modules/bibformat/web/admin/Makefile \ modules/oaiharvest/Makefile \ modules/oaiharvest/bin/Makefile \ modules/oaiharvest/bin/oaiharvest \ modules/oaiharvest/doc/Makefile \ modules/oaiharvest/doc/admin/Makefile \ modules/oaiharvest/doc/hacking/Makefile \ modules/oaiharvest/lib/Makefile \ modules/oaiharvest/web/Makefile \ modules/oaiharvest/web/admin/Makefile \ modules/oairepository/Makefile \ modules/oairepository/bin/Makefile \ modules/oairepository/bin/oairepositoryupdater \ modules/oairepository/doc/Makefile \ modules/oairepository/doc/admin/Makefile \ modules/oairepository/doc/hacking/Makefile \ modules/oairepository/etc/Makefile \ modules/oairepository/lib/Makefile \ modules/oairepository/web/Makefile \ modules/oairepository/web/admin/Makefile \ modules/bibindex/Makefile \ modules/bibindex/bin/Makefile \ modules/bibindex/bin/bibindex \ modules/bibindex/bin/bibstat \ modules/bibindex/doc/Makefile \ modules/bibindex/doc/admin/Makefile \ modules/bibindex/doc/hacking/Makefile \ modules/bibindex/lib/Makefile \ modules/bibindex/web/Makefile \ modules/bibindex/web/admin/Makefile \ modules/bibknowledge/Makefile \ modules/bibknowledge/lib/Makefile \ modules/bibknowledge/doc/Makefile \ modules/bibknowledge/doc/admin/Makefile \ modules/bibknowledge/doc/hacking/Makefile \ modules/bibmatch/Makefile \ modules/bibmatch/bin/Makefile \ modules/bibmatch/bin/bibmatch \ modules/bibmatch/doc/Makefile \ modules/bibmatch/doc/admin/Makefile \ modules/bibmatch/doc/hacking/Makefile \ modules/bibmatch/etc/Makefile \ modules/bibmatch/lib/Makefile \ modules/bibmerge/Makefile \ modules/bibmerge/bin/Makefile \ modules/bibmerge/doc/Makefile \ modules/bibmerge/doc/admin/Makefile \ modules/bibmerge/doc/hacking/Makefile \ modules/bibmerge/lib/Makefile \ modules/bibmerge/web/Makefile \ modules/bibmerge/web/admin/Makefile \ modules/bibrank/Makefile \ modules/bibrank/bin/Makefile \ modules/bibrank/bin/bibrank \ modules/bibrank/bin/bibrankgkb \ modules/bibrank/doc/Makefile \ modules/bibrank/doc/admin/Makefile \ modules/bibrank/doc/hacking/Makefile \ modules/bibrank/etc/Makefile \ modules/bibrank/etc/bibrankgkb.cfg \ modules/bibrank/etc/demo_jif.cfg \ modules/bibrank/etc/template_single_tag_rank_method.cfg \ modules/bibrank/lib/Makefile \ modules/bibrank/web/Makefile \ modules/bibrank/web/admin/Makefile \ modules/bibsched/Makefile \ modules/bibsched/bin/Makefile \ modules/bibsched/bin/bibsched \ modules/bibsched/bin/bibtaskex \ modules/bibsched/bin/bibtasklet \ modules/bibsched/doc/Makefile \ modules/bibsched/doc/admin/Makefile \ modules/bibsched/doc/hacking/Makefile \ modules/bibsched/lib/Makefile \ modules/bibsched/lib/tasklets/Makefile \ modules/bibupload/Makefile \ modules/bibsort/Makefile \ modules/bibsort/bin/Makefile \ modules/bibsort/bin/bibsort \ modules/bibsort/lib/Makefile \ modules/bibsort/etc/Makefile \ modules/bibsort/doc/Makefile \ modules/bibsort/doc/admin/Makefile \ modules/bibsort/doc/hacking/Makefile \ modules/bibsort/web/Makefile \ modules/bibsort/web/admin/Makefile \ modules/bibsword/Makefile \ modules/bibsword/bin/Makefile \ modules/bibsword/bin/bibsword \ modules/bibsword/doc/Makefile \ modules/bibsword/doc/admin/Makefile \ modules/bibsword/doc/hacking/Makefile \ modules/bibsword/lib/Makefile \ modules/bibsword/etc/Makefile \ modules/bibupload/bin/Makefile \ modules/bibupload/bin/bibupload \ modules/bibupload/bin/batchuploader \ modules/bibupload/doc/Makefile \ modules/bibupload/doc/admin/Makefile \ modules/bibupload/doc/hacking/Makefile \ modules/bibupload/lib/Makefile \ modules/elmsubmit/Makefile \ modules/elmsubmit/bin/Makefile \ modules/elmsubmit/bin/elmsubmit \ modules/elmsubmit/doc/Makefile \ modules/elmsubmit/doc/admin/Makefile \ modules/elmsubmit/doc/hacking/Makefile \ modules/elmsubmit/etc/Makefile \ modules/elmsubmit/etc/elmsubmit.cfg \ modules/elmsubmit/lib/Makefile \ modules/miscutil/Makefile \ modules/miscutil/bin/Makefile \ modules/miscutil/bin/dbdump \ modules/miscutil/bin/dbexec \ modules/miscutil/bin/inveniocfg \ modules/miscutil/bin/plotextractor \ modules/miscutil/demo/Makefile \ modules/miscutil/doc/Makefile \ modules/miscutil/doc/hacking/Makefile \ modules/miscutil/etc/Makefile \ modules/miscutil/etc/bash_completion.d/Makefile \ modules/miscutil/etc/bash_completion.d/inveniocfg \ modules/miscutil/etc/ckeditor_scientificchar/Makefile \ modules/miscutil/etc/ckeditor_scientificchar/dialogs/Makefile \ modules/miscutil/etc/ckeditor_scientificchar/lang/Makefile \ modules/miscutil/lib/Makefile \ modules/miscutil/lib/upgrades/Makefile \ modules/miscutil/sql/Makefile \ modules/miscutil/web/Makefile \ modules/webaccess/Makefile \ modules/webaccess/bin/Makefile \ modules/webaccess/bin/authaction \ modules/webaccess/bin/webaccessadmin \ modules/webaccess/doc/Makefile \ modules/webaccess/doc/admin/Makefile \ modules/webaccess/doc/hacking/Makefile \ modules/webaccess/lib/Makefile \ modules/webaccess/web/Makefile \ modules/webaccess/web/admin/Makefile \ modules/webalert/Makefile \ modules/webalert/bin/Makefile \ modules/webalert/bin/alertengine \ modules/webalert/doc/Makefile \ modules/webalert/doc/admin/Makefile \ modules/webalert/doc/hacking/Makefile \ modules/webalert/lib/Makefile \ modules/webalert/web/Makefile \ modules/webbasket/Makefile \ modules/webbasket/doc/Makefile \ modules/webbasket/doc/admin/Makefile \ modules/webbasket/doc/hacking/Makefile \ modules/webbasket/lib/Makefile \ modules/webbasket/web/Makefile \ modules/webcomment/Makefile \ modules/webcomment/doc/Makefile \ modules/webcomment/doc/admin/Makefile \ modules/webcomment/doc/hacking/Makefile \ modules/webcomment/lib/Makefile \ modules/webcomment/web/Makefile \ modules/webcomment/web/admin/Makefile \ modules/webhelp/Makefile \ modules/webhelp/web/Makefile \ modules/webhelp/web/admin/Makefile \ modules/webhelp/web/admin/howto/Makefile \ modules/webhelp/web/hacking/Makefile \ modules/webjournal/Makefile \ modules/webjournal/etc/Makefile \ modules/webjournal/doc/Makefile \ modules/webjournal/doc/admin/Makefile \ modules/webjournal/doc/hacking/Makefile \ modules/webjournal/lib/Makefile \ modules/webjournal/lib/elements/Makefile \ modules/webjournal/lib/widgets/Makefile \ modules/webjournal/web/Makefile \ modules/webjournal/web/admin/Makefile \ modules/webmessage/Makefile \ modules/webmessage/bin/Makefile \ modules/webmessage/bin/webmessageadmin \ modules/webmessage/doc/Makefile \ modules/webmessage/doc/admin/Makefile \ modules/webmessage/doc/hacking/Makefile \ modules/webmessage/lib/Makefile \ modules/webmessage/web/Makefile \ modules/websearch/Makefile \ modules/websearch/bin/Makefile \ modules/websearch/bin/webcoll \ modules/websearch/doc/Makefile \ modules/websearch/doc/admin/Makefile \ modules/websearch/doc/hacking/Makefile \ modules/websearch/lib/Makefile \ modules/websearch/web/Makefile \ modules/websearch/web/admin/Makefile \ modules/websession/Makefile \ modules/websession/bin/Makefile \ modules/websession/bin/inveniogc \ modules/websession/doc/Makefile \ modules/websession/doc/admin/Makefile \ modules/websession/doc/hacking/Makefile \ modules/websession/lib/Makefile \ modules/websession/web/Makefile \ modules/webstat/Makefile \ modules/webstat/bin/Makefile \ modules/webstat/bin/webstat \ modules/webstat/bin/webstatadmin \ modules/webstat/doc/Makefile \ modules/webstat/doc/admin/Makefile \ modules/webstat/doc/hacking/Makefile \ modules/webstat/etc/Makefile \ modules/webstat/lib/Makefile \ modules/webstyle/Makefile \ modules/webstyle/bin/Makefile \ modules/webstyle/bin/webdoc \ modules/webstyle/css/Makefile \ modules/webstyle/doc/Makefile \ modules/webstyle/doc/admin/Makefile \ modules/webstyle/doc/hacking/Makefile \ modules/webstyle/etc/Makefile \ modules/webstyle/img/Makefile \ modules/webstyle/lib/Makefile \ modules/websubmit/Makefile \ modules/websubmit/bin/Makefile \ modules/websubmit/bin/inveniounoconv \ modules/websubmit/bin/websubmitadmin \ modules/websubmit/doc/Makefile \ modules/websubmit/doc/admin/Makefile \ modules/websubmit/doc/hacking/Makefile \ modules/websubmit/etc/Makefile \ modules/websubmit/lib/Makefile \ modules/websubmit/lib/functions/Makefile \ modules/websubmit/web/Makefile \ modules/websubmit/web/admin/Makefile \ + scripts/Makefile \ ]) ## Finally, write output files: AC_OUTPUT ## Write help: AC_MSG_RESULT([****************************************************************************]) AC_MSG_RESULT([** Your Invenio installation is now ready for building. **]) AC_MSG_RESULT([** You have entered the following parameters: **]) AC_MSG_RESULT([** - Invenio main install directory: ${prefix}]) AC_MSG_RESULT([** - Python executable: $PYTHON]) AC_MSG_RESULT([** - MySQL client executable: $MYSQL]) AC_MSG_RESULT([** - CLISP executable: $CLISP]) AC_MSG_RESULT([** - CMUCL executable: $CMUCL]) AC_MSG_RESULT([** - SBCL executable: $SBCL]) AC_MSG_RESULT([** Here are the steps to continue the building process: **]) AC_MSG_RESULT([** 1) Type 'make' to build your Invenio system. **]) AC_MSG_RESULT([** 2) Type 'make install' to install your Invenio system. **]) AC_MSG_RESULT([** After that you can start customizing your installation as documented **]) AC_MSG_RESULT([** in the INSTALL file (i.e. edit invenio.conf, run inveniocfg, etc). **]) AC_MSG_RESULT([** Happy hacking and thanks for flying Invenio. **]) AC_MSG_RESULT([** -- Invenio Development Team **]) AC_MSG_RESULT([****************************************************************************]) ## end of file diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py index 39c818729..29479b6c3 100644 --- a/modules/miscutil/lib/inveniocfg.py +++ b/modules/miscutil/lib/inveniocfg.py @@ -1,1661 +1,1663 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013, 2015 CERN. +## Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013, 2015, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Invenio configuration and administration CLI tool. Usage: inveniocfg [options] General options: -h, --help print this help -V, --version print version number Options to finish your installation: --create-apache-conf create Apache configuration files --create-tables create DB tables for Invenio --load-webstat-conf load the WebStat configuration --drop-tables drop DB tables of Invenio --check-openoffice check for correctly set up of openoffice temporary directory Options to set up and test a demo site: --create-demo-site create demo site --load-demo-records load demo records --remove-demo-records remove demo records, keeping demo site --drop-demo-site drop demo site configurations too --run-unit-tests run unit test suite (needs demo site) --run-regression-tests run regression test suite (needs demo site) --run-web-tests run web tests in a browser (needs demo site, Firefox, Selenium IDE) Options to update config files in situ: --update-all perform all the update options --update-config-py update config.py file from invenio.conf file --update-dbquery-py update dbquery.py with DB credentials from invenio.conf --update-dbexec update dbexec with DB credentials from invenio.conf --update-bibconvert-tpl update bibconvert templates with CFG_SITE_URL from invenio.conf --update-web-tests update web test cases with CFG_SITE_URL from invenio.conf Options to update DB tables: --reset-all perform all the reset options --reset-sitename reset tables to take account of new CFG_SITE_NAME* --reset-siteadminemail reset tables to take account of new CFG_SITE_ADMIN_EMAIL --reset-fieldnames reset tables to take account of new I18N names from PO files --reset-recstruct-cache reset record structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE Options to upgrade your installation: --upgrade apply all pending upgrades --upgrade-check run pre-upgrade checks for all pending upgrades --upgrade-show-pending show pending upgrades ready to be applied --upgrade-show-applied show history of applied upgrades --upgrade-create-standard-recipe create a new upgrade recipe (for developers) --upgrade-create-release-recipe create a new release upgrade recipe (for developers) Options to help the work: --list print names and values of all options from conf files --get get value of a given option from conf files --conf-dir path to directory where invenio*.conf files are [optional] --detect-system-details print system details such as Apache/Python/MySQL versions """ __revision__ = "$Id$" from ConfigParser import ConfigParser from optparse import OptionParser, OptionGroup, IndentedHelpFormatter, Option, \ OptionError import os import re import shutil import socket import sys def print_usage(): """Print help.""" print __doc__ def get_version(): """ Get running version of Invenio """ from invenio.config import CFG_VERSION return CFG_VERSION def print_version(): """Print version information.""" print get_version() def convert_conf_option(option_name, option_value): """ Convert conf option into Python config.py line, converting values to ints or strings as appropriate. """ ## 1) convert option name to uppercase: option_name = option_name.upper() ## 1a) adjust renamed variables: if option_name in ['CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES', 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS', 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC', 'CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT', 'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS', 'CFG_WEBSUBMIT_DESIRED_CONVERSIONS']: new_option_name = option_name.replace('WEBSUBMIT', 'BIBDOCFILE') print >> sys.stderr, ("""WARNING: %s has been renamed to %s. Please, update your invenio-local.conf file accordingly.""" % (option_name, new_option_name)) option_name = new_option_name ## 2) convert option value to int or string: if option_name in ['CFG_BIBUPLOAD_REFERENCE_TAG', 'CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG', 'CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG', 'CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG', 'CFG_BIBUPLOAD_STRONG_TAGS', 'CFG_BIBFORMAT_HIDDEN_TAGS',]: # some options are supposed be string even when they look like # numeric option_value = '"' + option_value + '"' else: try: option_value = int(option_value) except ValueError: option_value = '"' + option_value + '"' ## 3a) special cases: chars regexps if option_name in ['CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS', 'CFG_BIBINDEX_CHARS_PUNCTUATION']: option_value = 'r"[' + option_value[1:-1] + ']"' ## 3abis) special cases: real regexps if option_name in ['CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES', 'CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS']: option_value = 'r"' + option_value[1:-1] + '"' ## 3b) special cases: True, False, None if option_value in ['"True"', '"False"', '"None"']: option_value = option_value[1:-1] ## 3c) special cases: dicts and real pythonic lists if option_name in ['CFG_WEBSEARCH_FIELDS_CONVERT', 'CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS', 'CFG_WEBSEARCH_FULLTEXT_SNIPPETS', 'CFG_WEBSEARCH_FULLTEXT_SNIPPETS_CHARS', 'CFG_SITE_EMERGENCY_EMAIL_ADDRESSES', 'CFG_BIBMATCH_FUZZY_WORDLIMITS', 'CFG_BIBMATCH_QUERY_TEMPLATES', 'CFG_WEBSEARCH_SYNONYM_KBRS', 'CFG_BIBINDEX_SYNONYM_KBRS', 'CFG_WEBCOMMENT_EMAIL_REPLIES_TO', 'CFG_WEBCOMMENT_RESTRICTION_DATAFIELD', 'CFG_WEBCOMMENT_ROUND_DATAFIELD', 'CFG_BIBUPLOAD_FFT_ALLOWED_EXTERNAL_URLS', 'CFG_BIBSCHED_NODE_TASKS', 'CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE', 'CFG_OAI_METADATA_FORMATS', 'CFG_BIBDOCFILE_DESIRED_CONVERSIONS', 'CFG_BIBDOCFILE_BEST_FORMATS_TO_EXTRACT_TEXT_FROM', 'CFG_WEB_API_KEY_ALLOWED_URL', 'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_MISC', 'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_DOCTYPES', 'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_RESTRICTIONS', 'CFG_ICON_CREATION_FORMAT_MAPPINGS']: try: option_value = option_value[1:-1] except TypeError: if option_name in ('CFG_WEBSEARCH_FULLTEXT_SNIPPETS',): print >> sys.stderr, """WARNING: CFG_WEBSEARCH_FULLTEXT_SNIPPETS has changed syntax: it can be customised to display different snippets for different document types. See the corresponding documentation in invenio.conf. You may want to customise your invenio-local.conf configuration accordingly.""" option_value = """{'': %s}""" % option_value else: print >> sys.stderr, "ERROR: type error in %s value %s." % \ (option_name, option_value) sys.exit(1) ## 3cbis) very special cases: dicts with backward compatible string if option_name in ['CFG_BIBINDEX_SPLASH_PAGES']: if option_value.startswith('"{') and option_value.endswith('}"'): option_value = option_value[1:-1] else: option_value = """{%s: ".*"}""" % option_value ## 3d) special cases: comma-separated lists if option_name in ['CFG_SITE_LANGS', 'CFG_BIBDOCFILE_ADDITIONAL_KNOWN_FILE_EXTENSIONS', 'CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS', 'CFG_BIBUPLOAD_STRONG_TAGS', 'CFG_BIBFORMAT_HIDDEN_TAGS', 'CFG_BIBSCHED_GC_TASKS_TO_REMOVE', 'CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE', 'CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS', 'CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS', 'CFG_BIBUPLOAD_DELETE_FORMATS', 'CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES', 'CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST', 'CFG_WEBSEARCH_RSS_I18N_COLLECTIONS', 'CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY', 'CFG_BIBAUTHORID_EXTERNAL_CLAIMED_RECORDS_KEY', 'CFG_PLOTEXTRACTOR_DISALLOWED_TEX', 'CFG_OAI_FRIENDS', 'CFG_WEBSTYLE_REVERSE_PROXY_IPS', 'CFG_BIBEDIT_AUTOCOMPLETE_INSTITUTIONS_FIELDS', 'CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS', 'CFG_BIBFORMAT_HIDDEN_FILE_FORMATS',]: out = "[" for elem in option_value[1:-1].split(","): if elem: elem = elem.strip() if option_name in ['CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES']: # 3d1) integer values out += "%i, " % int(elem) else: # 3d2) string values out += "'%s', " % elem out += "]" option_value = out ## 3e) special cases: multiline if option_name == 'CFG_OAI_IDENTIFY_DESCRIPTION': # make triple quotes option_value = '""' + option_value + '""' ## 3f) ignore some options: if option_name.startswith('CFG_SITE_NAME_INTL'): # treated elsewhere return ## 3g) special cases: float if option_name in ['CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY', 'CFG_BIBMATCH_LOCAL_SLEEPTIME', 'CFG_BIBMATCH_REMOTE_SLEEPTIME', 'CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT', 'CFG_BIBMATCH_FUZZY_MATCH_VALIDATION_LIMIT']: option_value = float(option_value[1:-1]) ## 3h) special cases: bibmatch validation list if option_name in ['CFG_BIBMATCH_MATCH_VALIDATION_RULESETS']: option_value = option_value[1:-1] ## 4a) dropped variables if option_name in ['CFG_BATCHUPLOADER_WEB_ROBOT_AGENT']: print >> sys.stderr, ("""ERROR: CFG_BATCHUPLOADER_WEB_ROBOT_AGENT has been dropped in favour of CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS. Please, update your invenio-local.conf file accordingly.""") option_value = option_value[1:-1] elif option_name in ['CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES', 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS', 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC', 'CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT', 'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS', 'CFG_WEBSUBMIT_DESIRED_CONVERSIONS']: new_option_name = option_name.replace('WEBSUBMIT', 'BIBDOCFILE') print >> sys.stderr, ("""ERROR: %s has been renamed to %s. Please, update your invenio-local.conf file accordingly.""" % (option_name, new_option_name)) option_name = new_option_name ## 5) finally, return output line: return '%s = %s' % (option_name, option_value) def cli_cmd_update_config_py(conf): """ Update new config.py from conf options, keeping previous config.py in a backup copy. """ print ">>> Going to update config.py..." ## location where config.py is: configpyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \ os.sep + 'invenio' + os.sep + 'config.py' ## backup current config.py file: if os.path.exists(configpyfile): shutil.copy(configpyfile, configpyfile + '.OLD') ## here we go: fdesc = open(configpyfile, 'w') ## generate preamble: fdesc.write("# -*- coding: utf-8 -*-\n") fdesc.write("# DO NOT EDIT THIS FILE! IT WAS AUTOMATICALLY GENERATED\n") fdesc.write("# FROM INVENIO.CONF BY EXECUTING:\n") fdesc.write("# " + " ".join(sys.argv) + "\n") ## special treatment for CFG_SITE_NAME_INTL options: fdesc.write("CFG_SITE_NAME_INTL = {}\n") for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","): fdesc.write("CFG_SITE_NAME_INTL['%s'] = \"%s\"\n" % (lang, conf.get("Invenio", "CFG_SITE_NAME_INTL_" + lang))) ## special treatment for CFG_SITE_SECURE_URL that may be empty, in ## which case it should be put equal to CFG_SITE_URL: if not conf.get("Invenio", "CFG_SITE_SECURE_URL"): conf.set("Invenio", "CFG_SITE_SECURE_URL", conf.get("Invenio", "CFG_SITE_URL")) ## process all the options normally: sections = conf.sections() sections.sort() for section in sections: options = conf.options(section) options.sort() for option in options: if not option.startswith('CFG_DATABASE_'): # put all options except for db credentials into config.py line_out = convert_conf_option(option, conf.get(section, option)) if line_out: fdesc.write(line_out + "\n") ## FIXME: special treatment for experimental variables ## CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES and CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE ## (not offering them in invenio.conf since they will be refactored) fdesc.write("CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE = 0\n") fdesc.write("CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES = [0, 1,]\n") ## generate postamble: fdesc.write("") fdesc.write("# END OF GENERATED FILE") ## we are done: fdesc.close() print "You may want to restart Apache now." print ">>> config.py updated successfully." def cli_cmd_update_dbquery_py(conf): """ Update lib/dbquery.py file with DB parameters read from conf file. Note: this edits dbquery.py in situ, taking a backup first. Use only when you know what you are doing. """ print ">>> Going to update dbquery.py..." ## location where dbquery.py is: dbquerypyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \ os.sep + 'invenio' + os.sep + 'dbquery.py' ## backup current dbquery.py file: if os.path.exists(dbquerypyfile): shutil.copy(dbquerypyfile, dbquerypyfile + '.OLD') ## replace db parameters: out = '' for line in open(dbquerypyfile, 'r').readlines(): match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS|SLAVE)(\s*=\s*)\'.*\'$', line) if match: dbparam = 'CFG_DATABASE_' + match.group(1) out += "%s%s'%s'\n" % (dbparam, match.group(2), conf.get('Invenio', dbparam)) else: out += line fdesc = open(dbquerypyfile, 'w') fdesc.write(out) fdesc.close() print "You may want to restart Apache now." print ">>> dbquery.py updated successfully." def cli_cmd_update_dbexec(conf): """ Update bin/dbexec file with DB parameters read from conf file. Note: this edits dbexec in situ, taking a backup first. Use only when you know what you are doing. """ print ">>> Going to update dbexec..." ## location where dbexec is: dbexecfile = conf.get("Invenio", "CFG_BINDIR") + \ os.sep + 'dbexec' ## backup current dbexec file: if os.path.exists(dbexecfile): shutil.copy(dbexecfile, dbexecfile + '.OLD') ## replace db parameters via sed: out = '' for line in open(dbexecfile, 'r').readlines(): match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS|SLAVE)(\s*=\s*)\'.*\'$', line) if match: dbparam = 'CFG_DATABASE_' + match.group(1) out += "%s%s'%s'\n" % (dbparam, match.group(2), conf.get("Invenio", dbparam)) else: out += line fdesc = open(dbexecfile, 'w') fdesc.write(out) fdesc.close() print ">>> dbexec updated successfully." def cli_cmd_update_bibconvert_tpl(conf): """ Update bibconvert/config/*.tpl files looking for 856 http://.../CFG_SITE_RECORD lines, replacing URL with CFG_SITE_URL taken from conf file. Note: this edits tpl files in situ, taking a backup first. Use only when you know what you are doing. """ print ">>> Going to update bibconvert templates..." ## location where bibconvert/config/*.tpl are: tpldir = conf.get("Invenio", 'CFG_ETCDIR') + \ os.sep + 'bibconvert' + os.sep + 'config' ## find all *.tpl files: for tplfilename in os.listdir(tpldir): if tplfilename.endswith(".tpl"): ## change tpl file: tplfile = tpldir + os.sep + tplfilename shutil.copy(tplfile, tplfile + '.OLD') out = '' for line in open(tplfile, 'r').readlines(): match = re.search(r'^(.*)http://.*?/%s/(.*)$' % conf.get("Invenio", 'CFG_SITE_RECORD'), line) if match: out += "%s%s/%s/%s\n" % (match.group(1), conf.get("Invenio", 'CFG_SITE_URL'), conf.get("Invenio", 'CFG_SITE_RECORD'), match.group(2)) else: out += line fdesc = open(tplfile, 'w') fdesc.write(out) fdesc.close() print ">>> bibconvert templates updated successfully." def cli_cmd_update_web_tests(conf): """ Update web test cases lib/webtest/test_*.html looking for http://.+?[>> Going to update web tests..." ## location where test_*.html files are: testdir = conf.get("Invenio", 'CFG_PREFIX') + os.sep + \ 'lib' + os.sep + 'webtest' + os.sep + 'invenio' ## find all test_*.html files: for testfilename in os.listdir(testdir): if testfilename.startswith("test_") and \ testfilename.endswith(".html"): ## change test file: testfile = testdir + os.sep + testfilename shutil.copy(testfile, testfile + '.OLD') out = '' for line in open(testfile, 'r').readlines(): match = re.search(r'^(.*)http://.+?([)/opt/invenio(.*)$', line) if match: out += "%s%s%s\n" % (match.group(1), conf.get("Invenio", 'CFG_PREFIX'), match.group(2)) else: out += line fdesc = open(testfile, 'w') fdesc.write(out) fdesc.close() print ">>> web tests updated successfully." def cli_cmd_reset_sitename(conf): """ Reset collection-related tables with new CFG_SITE_NAME and CFG_SITE_NAME_INTL* read from conf files. """ print ">>> Going to reset CFG_SITE_NAME and CFG_SITE_NAME_INTL..." from invenio.dbquery import run_sql, IntegrityError # reset CFG_SITE_NAME: sitename = conf.get("Invenio", "CFG_SITE_NAME") try: run_sql("""INSERT INTO collection (id, name, dbquery, reclist) VALUES (1,%s,NULL,NULL)""", (sitename,)) except IntegrityError: run_sql("""UPDATE collection SET name=%s WHERE id=1""", (sitename,)) # reset CFG_SITE_NAME_INTL: for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","): sitename_lang = conf.get("Invenio", "CFG_SITE_NAME_INTL_" + lang) try: run_sql("""INSERT INTO collectionname (id_collection, ln, type, value) VALUES (%s,%s,%s,%s)""", (1, lang, 'ln', sitename_lang)) except IntegrityError: run_sql("""UPDATE collectionname SET value=%s WHERE ln=%s AND id_collection=1 AND type='ln'""", (sitename_lang, lang)) print "You may want to restart Apache now." print ">>> CFG_SITE_NAME and CFG_SITE_NAME_INTL* reset successfully." def cli_cmd_reset_recstruct_cache(conf): """If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function will adapt the database to either store or not store the recstruct format.""" from invenio.intbitset import intbitset from invenio.dbquery import run_sql, serialize_via_marshal from invenio.search_engine import get_record from invenio.bibsched import server_pid, pidfile enable_recstruct_cache = conf.get("Invenio", "CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE") enable_recstruct_cache = enable_recstruct_cache in ('True', '1') pid = server_pid(ping_the_process=False) if pid: print >> sys.stderr, "ERROR: bibsched seems to run with pid %d, according to %s." % (pid, pidfile) print >> sys.stderr, " Please stop bibsched before running this procedure." sys.exit(1) if enable_recstruct_cache: print ">>> Searching records which need recstruct cache resetting; this may take a while..." all_recids = intbitset(run_sql("SELECT id FROM bibrec")) good_recids = intbitset(run_sql("SELECT bibrec.id FROM bibrec JOIN bibfmt ON bibrec.id = bibfmt.id_bibrec WHERE format='recstruct' AND modification_date < last_updated")) recids = all_recids - good_recids print ">>> Generating recstruct cache..." tot = len(recids) count = 0 for recid in recids: value = serialize_via_marshal(get_record(recid)) run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='recstruct'", (recid, )) run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'recstruct', NOW(), %s)", (recid, value)) count += 1 if count % 1000 == 0: print " ... done records %s/%s" % (count, tot) if count % 1000 != 0: print " ... done records %s/%s" % (count, tot) print ">>> recstruct cache generated successfully." else: print ">>> Cleaning recstruct cache..." run_sql("DELETE FROM bibfmt WHERE format='recstruct'") def cli_cmd_reset_siteadminemail(conf): """ Reset user-related tables with new CFG_SITE_ADMIN_EMAIL read from conf files. """ print ">>> Going to reset CFG_SITE_ADMIN_EMAIL..." from invenio.dbquery import run_sql siteadminemail = conf.get("Invenio", "CFG_SITE_ADMIN_EMAIL") run_sql("DELETE FROM user WHERE id=1") run_sql("""INSERT INTO user (id, email, password, note, nickname) VALUES (1, %s, AES_ENCRYPT(email, ''), 1, 'admin')""", (siteadminemail,)) print "You may want to restart Apache now." print ">>> CFG_SITE_ADMIN_EMAIL reset successfully." def cli_cmd_reset_fieldnames(conf): """ Reset I18N field names such as author, title, etc and other I18N ranking method names such as word similarity. Their translations are taken from the PO files. """ print ">>> Going to reset I18N field names..." from invenio.messages import gettext_set_language, language_list_long from invenio.dbquery import run_sql, IntegrityError ## get field id and name list: field_id_name_list = run_sql("SELECT id, name FROM field") ## get rankmethod id and name list: rankmethod_id_name_list = run_sql("SELECT id, name FROM rnkMETHOD") ## update names for every language: for lang, dummy in language_list_long(): _ = gettext_set_language(lang) ## this list is put here in order for PO system to pick names ## suitable for translation field_name_names = {"any field": _("any field"), "title": _("title"), "author": _("author"), "abstract": _("abstract"), "keyword": _("keyword"), "report number": _("report number"), "subject": _("subject"), "reference": _("reference"), "fulltext": _("fulltext"), "collection": _("collection"), "division": _("division"), "year": _("year"), "journal": _("journal"), "experiment": _("experiment"), "record ID": _("record ID")} ## update I18N names for every language: for (field_id, field_name) in field_id_name_list: if field_name_names.has_key(field_name): try: run_sql("""INSERT INTO fieldname (id_field,ln,type,value) VALUES (%s,%s,%s,%s)""", (field_id, lang, 'ln', field_name_names[field_name])) except IntegrityError: run_sql("""UPDATE fieldname SET value=%s WHERE id_field=%s AND ln=%s AND type=%s""", (field_name_names[field_name], field_id, lang, 'ln',)) ## ditto for rank methods: rankmethod_name_names = {"wrd": _("word similarity"), "demo_jif": _("journal impact factor"), "citation": _("times cited"), "citerank_citation_t": _("time-decay cite count"), "citerank_pagerank_c": _("all-time-best cite rank"), "citerank_pagerank_t": _("time-decay cite rank"),} for (rankmethod_id, rankmethod_name) in rankmethod_id_name_list: if rankmethod_name_names.has_key(rankmethod_name): try: run_sql("""INSERT INTO rnkMETHODNAME (id_rnkMETHOD,ln,type,value) VALUES (%s,%s,%s,%s)""", (rankmethod_id, lang, 'ln', rankmethod_name_names[rankmethod_name])) except IntegrityError: run_sql("""UPDATE rnkMETHODNAME SET value=%s WHERE id_rnkMETHOD=%s AND ln=%s AND type=%s""", (rankmethod_name_names[rankmethod_name], rankmethod_id, lang, 'ln',)) print ">>> I18N field names reset successfully." def cli_check_openoffice(conf): """ If OpenOffice.org integration is enabled, checks whether the system is properly configured. """ from invenio.bibtask import check_running_process_user from invenio.websubmit_file_converter import can_unoconv, get_file_converter_logger logger = get_file_converter_logger() for handler in logger.handlers: logger.removeHandler(handler) check_running_process_user() print ">>> Checking if Libre/OpenOffice.org is correctly integrated...", sys.stdout.flush() if can_unoconv(True): print "ok" else: sys.exit(1) def test_db_connection(): """ Test DB connection, and if fails, advise user how to set it up. Useful to be called during table creation. """ print "Testing DB connection...", from invenio.textutils import wrap_text_in_a_box from invenio.dbquery import run_sql, Error ## first, test connection to the DB server: try: run_sql("SHOW TABLES") except Error, err: from invenio.dbquery import CFG_DATABASE_HOST, CFG_DATABASE_PORT, \ CFG_DATABASE_NAME, CFG_DATABASE_USER, CFG_DATABASE_PASS print wrap_text_in_a_box("""\ DATABASE CONNECTIVITY ERROR %(errno)d: %(errmsg)s.\n Perhaps you need to set up database and connection rights? If yes, then please login as MySQL admin user and run the following commands now: $ mysql -h %(dbhost)s -P %(dbport)s -u root -p mysql mysql> CREATE DATABASE %(dbname)s DEFAULT CHARACTER SET utf8; mysql> GRANT ALL PRIVILEGES ON %(dbname)s.* TO %(dbuser)s@%(webhost)s IDENTIFIED BY '%(dbpass)s'; mysql> QUIT The values printed above were detected from your configuration. If they are not right, then please edit your invenio-local.conf file and rerun 'inveniocfg --update-all' first. If the problem is of different nature, then please inspect the above error message and fix the problem before continuing.""" % \ {'errno': err.args[0], 'errmsg': err.args[1], 'dbname': CFG_DATABASE_NAME, 'dbhost': CFG_DATABASE_HOST, 'dbport': CFG_DATABASE_PORT, 'dbuser': CFG_DATABASE_USER, 'dbpass': CFG_DATABASE_PASS, 'webhost': CFG_DATABASE_HOST == 'localhost' and 'localhost' or os.popen('hostname -f', 'r').read().strip(), }) sys.exit(1) print "ok" ## second, test insert/select of a Unicode string to detect ## possible Python/MySQL/MySQLdb mis-setup: print "Testing Python/MySQL/MySQLdb UTF-8 chain...", try: try: beta_in_utf8 = "β" # Greek beta in UTF-8 is 0xCEB2 run_sql("CREATE TABLE test__invenio__utf8 (x char(1), y varbinary(2)) DEFAULT CHARACTER SET utf8 ENGINE=MyISAM;") run_sql("INSERT INTO test__invenio__utf8 (x, y) VALUES (%s, %s)", (beta_in_utf8, beta_in_utf8)) res = run_sql("SELECT x,y,HEX(x),HEX(y),LENGTH(x),LENGTH(y),CHAR_LENGTH(x),CHAR_LENGTH(y) FROM test__invenio__utf8") assert res[0] == ('\xce\xb2', '\xce\xb2', 'CEB2', 'CEB2', 2L, 2L, 1L, 2L) run_sql("DROP TABLE test__invenio__utf8") except Exception, err: print wrap_text_in_a_box("""\ DATABASE RELATED ERROR %s\n A problem was detected with the UTF-8 treatment in the chain between the Python application, the MySQLdb connector, and the MySQL database. You may perhaps have installed older versions of some prerequisite packages?\n Please check the INSTALL file and please fix this problem before continuing.""" % err) sys.exit(1) finally: run_sql("DROP TABLE IF EXISTS test__invenio__utf8") print "ok" def cli_cmd_create_tables(conf): """Create and fill Invenio DB tables. Useful for the installation process.""" print ">>> Going to create and fill tables..." from invenio.config import CFG_PREFIX test_db_connection() for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabcreate.sql" % (CFG_PREFIX, CFG_PREFIX), "%s/bin/dbexec < %s/lib/sql/invenio/tabfill.sql" % (CFG_PREFIX, CFG_PREFIX)]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) cli_cmd_reset_sitename(conf) cli_cmd_reset_siteadminemail(conf) cli_cmd_reset_fieldnames(conf) for cmd in ["%s/bin/webaccessadmin -u admin -c -a" % CFG_PREFIX]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> Tables created and filled successfully." def cli_cmd_load_webstat_conf(conf): print ">>> Going to load WebStat config..." from invenio.config import CFG_PREFIX cmd = "%s/bin/webstatadmin --load-config" % CFG_PREFIX if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> WebStat config load successfully." def cli_cmd_drop_tables(conf): """Drop Invenio DB tables. Useful for the uninstallation process.""" print ">>> Going to drop tables..." from invenio.config import CFG_PREFIX from invenio.textutils import wrap_text_in_a_box, wait_for_user from invenio.webstat import destroy_customevents wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy your database tables!""")) msg = destroy_customevents() if msg: print msg cmd = "%s/bin/dbexec < %s/lib/sql/invenio/tabdrop.sql" % (CFG_PREFIX, CFG_PREFIX) if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> Tables dropped successfully." def cli_cmd_create_demo_site(conf): """Create demo site. Useful for testing purposes.""" print ">>> Going to create demo site..." from invenio.config import CFG_PREFIX from invenio.dbquery import run_sql run_sql("TRUNCATE schTASK") run_sql("TRUNCATE session") run_sql("DELETE FROM user WHERE email=''") for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/democfgdata.sql" % \ (CFG_PREFIX, CFG_PREFIX),]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) cli_cmd_reset_fieldnames(conf) # needed for I18N demo ranking method names for cmd in ["%s/bin/webaccessadmin -u admin -c -r -D" % CFG_PREFIX, "%s/bin/webcoll -u admin" % CFG_PREFIX, "%s/bin/webcoll 1" % CFG_PREFIX, "%s/bin/bibsort -u admin --load-config" % CFG_PREFIX, "%s/bin/bibsort 2" % CFG_PREFIX, ]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> Demo site created successfully." def cli_cmd_load_demo_records(conf): """Load demo records. Useful for testing purposes.""" from invenio.config import CFG_PREFIX from invenio.dbquery import run_sql print ">>> Going to load demo records..." run_sql("TRUNCATE schTASK") for cmd in ["%s/bin/bibupload -u admin -i %s/var/tmp/demobibdata.xml" % (CFG_PREFIX, CFG_PREFIX), "%s/bin/bibupload 1" % CFG_PREFIX, "%s/bin/bibdocfile --textify --with-ocr --recid 97" % CFG_PREFIX, "%s/bin/bibdocfile --textify --all" % CFG_PREFIX, "%s/bin/bibindex -u admin" % CFG_PREFIX, "%s/bin/bibindex 2" % CFG_PREFIX, "%s/bin/bibreformat -u admin -o HB" % CFG_PREFIX, "%s/bin/bibreformat 3" % CFG_PREFIX, "%s/bin/webcoll -u admin" % CFG_PREFIX, "%s/bin/webcoll 4" % CFG_PREFIX, "%s/bin/bibrank -u admin" % CFG_PREFIX, "%s/bin/bibrank 5" % CFG_PREFIX, "%s/bin/bibsort -u admin -R" % CFG_PREFIX, "%s/bin/bibsort 6" % CFG_PREFIX, "%s/bin/oairepositoryupdater -u admin" % CFG_PREFIX, "%s/bin/oairepositoryupdater 7" % CFG_PREFIX, "%s/bin/bibupload 8" % CFG_PREFIX,]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> Demo records loaded successfully." def cli_cmd_remove_demo_records(conf): """Remove demo records. Useful when you are finished testing.""" print ">>> Going to remove demo records..." from invenio.config import CFG_PREFIX from invenio.dbquery import run_sql from invenio.textutils import wrap_text_in_a_box, wait_for_user wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy your records and documents!""")) if os.path.exists(CFG_PREFIX + os.sep + 'var' + os.sep + 'data'): shutil.rmtree(CFG_PREFIX + os.sep + 'var' + os.sep + 'data') run_sql("TRUNCATE schTASK") for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabbibclean.sql" % (CFG_PREFIX, CFG_PREFIX), "%s/bin/webcoll -u admin" % CFG_PREFIX, "%s/bin/webcoll 1" % CFG_PREFIX,]: if os.system(cmd): print "ERROR: failed execution of", cmd sys.exit(1) print ">>> Demo records removed successfully." def cli_cmd_drop_demo_site(conf): """Drop demo site completely. Useful when you are finished testing.""" print ">>> Going to drop demo site..." from invenio.textutils import wrap_text_in_a_box, wait_for_user wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy your site and documents!""")) cli_cmd_drop_tables(conf) cli_cmd_create_tables(conf) cli_cmd_remove_demo_records(conf) print ">>> Demo site dropped successfully." def cli_cmd_run_unit_tests(conf): """Run unit tests, usually on the working demo site.""" from invenio.testutils import build_and_run_unit_test_suite if not build_and_run_unit_test_suite(): sys.exit(1) def cli_cmd_run_regression_tests(conf): """Run regression tests, usually on the working demo site.""" from invenio.testutils import build_and_run_regression_test_suite if not build_and_run_regression_test_suite(): sys.exit(1) def cli_cmd_run_web_tests(conf): """Run web tests in a browser. Requires Firefox with Selenium.""" from invenio.testutils import build_and_run_web_test_suite if not build_and_run_web_test_suite(): sys.exit(1) def _detect_ip_address(conf): """Detect IP address of this computer. Useful for creating Apache vhost conf snippet on RHEL like machines. However, if wanted site is 0.0.0.0, then use that, since we are running inside Docker. @return: IP address, or '*' if cannot detect @rtype: string @note: creates socket for real in order to detect real IP address, not the loopback one. """ if '0.0.0.0' in conf.get('Invenio', 'CFG_SITE_URL'): return '0.0.0.0' try: s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(('invenio-software.org', 0)) return s.getsockname()[0] except: return '*' def cli_cmd_create_apache_conf(conf): """ Create Apache conf files for this site, keeping previous files in a backup copy. """ print ">>> Going to create Apache conf files..." from invenio.textutils import wrap_text_in_a_box from invenio.access_control_config import CFG_EXTERNAL_AUTH_USING_SSO apache_conf_dir = conf.get("Invenio", 'CFG_ETCDIR') + \ os.sep + 'apache' if guess_apache_24(): directory_www_directive = """ # Uncomment the following on Apache < 2.4 # # Options FollowSymLinks MultiViews # AllowOverride None # Order allow,deny # Allow from all # # Comment the following on Apache < 2.4 Options FollowSymLinks MultiViews AllowOverride None Require all granted """ % {'webdir': conf.get('Invenio', 'CFG_WEBDIR')} directory_wsgi_directive = """ # Uncomment the following on Apache < 2.4 # # WSGIProcessGroup invenio # WSGIApplicationGroup %%{GLOBAL} # Options FollowSymLinks MultiViews # AllowOverride None # Order allow,deny # Allow from all # # Comment the following on Apache < 2.4 WSGIProcessGroup invenio WSGIApplicationGroup %%{GLOBAL} Options FollowSymLinks MultiViews AllowOverride None Require all granted """ % {'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi')} else: directory_www_directive = """ # Comment the following on Apache >= 2.4 Options FollowSymLinks MultiViews AllowOverride None Order allow,deny Allow from all # Uncomment the following on Apache >= 2.4 # # Options FollowSymLinks MultiViews # AllowOverride None # Require all granted # """ % {'webdir': conf.get('Invenio', 'CFG_WEBDIR')} directory_wsgi_directive = """ # Comment the following on Apache >= 2.4 WSGIProcessGroup invenio WSGIApplicationGroup %%{GLOBAL} Options FollowSymLinks MultiViews AllowOverride None Order allow,deny Allow from all # Uncomment the following on Apache >= 2.4 # # WSGIProcessGroup invenio # WSGIApplicationGroup %%{GLOBAL} # Options FollowSymLinks MultiViews # AllowOverride None # Require all granted # """ % {'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi')} ## Preparation of XSendFile directive xsendfile_directive_needed = int(conf.get("Invenio", 'CFG_BIBDOCFILE_USE_XSENDFILE')) != 0 if xsendfile_directive_needed: xsendfile_directive = "XSendFile On\n" else: xsendfile_directive = "#XSendFile On\n" for path in (conf.get('Invenio', 'CFG_BIBDOCFILE_FILEDIR'), # BibDocFile conf.get('Invenio', 'CFG_WEBDIR'), conf.get('Invenio', 'CFG_WEBSUBMIT_STORAGEDIR'), # WebSubmit conf.get('Invenio', 'CFG_TMPDIR'), os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'tmp', 'attachfile'), os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'comments'), os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'baskets', 'comments'), '/tmp'): # BibExport if xsendfile_directive_needed: xsendfile_directive += ' XSendFilePath %s\n' % path else: xsendfile_directive += ' #XSendFilePath %s\n' % path xsendfile_directive = xsendfile_directive.strip() ## Preparation of deflate directive deflate_directive_needed = int(conf.get("Invenio", 'CFG_WEBSTYLE_HTTP_USE_COMPRESSION')) != 0 if deflate_directive_needed: deflate_directive = r""" ## Configuration snippet taken from: ## SetOutputFilter DEFLATE # Netscape 4.x has some problems... BrowserMatch ^Mozilla/4 gzip-only-text/html # Netscape 4.06-4.08 have some more problems BrowserMatch ^Mozilla/4\.0[678] no-gzip # MSIE masquerades as Netscape, but it is fine # BrowserMatch \bMSIE !no-gzip !gzip-only-text/html # NOTE: Due to a bug in mod_setenvif up to Apache 2.0.48 # the above regex won't work. You can use the following # workaround to get the desired effect: BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html # Don't compress images SetEnvIfNoCase Request_URI \ \.(?:gif|jpe?g|png)$ no-gzip dont-vary # Make sure proxies don't deliver the wrong content Header append Vary User-Agent env=!dont-vary """ else: deflate_directive = "" if CFG_EXTERNAL_AUTH_USING_SSO: shibboleth_directive = r""" SSLRequireSSL # The modules only work using HTTPS AuthType shibboleth ShibRequireSession On ShibRequireAll On ShibExportAssertion Off require valid-user """ else: shibboleth_directive = "" ## Apache vhost conf file is distro specific, so analyze needs: # Gentoo (and generic defaults): listen_directive_needed = True ssl_pem_directive_needed = False ssl_pem_path = '/etc/apache2/ssl/apache.pem' ssl_crt_path = '/etc/apache2/ssl/server.crt' ssl_key_path = '/etc/apache2/ssl/server.key' vhost_ip_address_needed = False wsgi_socket_directive_needed = False # Debian: if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'debian_version'): listen_directive_needed = False ssl_pem_directive_needed = True # RHEL/SLC: if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'redhat-release'): listen_directive_needed = False ssl_crt_path = '/etc/pki/tls/certs/localhost.crt' ssl_key_path = '/etc/pki/tls/private/localhost.key' vhost_ip_address_needed = True + if os.popen('grep -c "CentOS.*[67]\." /etc/redhat-release').read().strip() == '1': + vhost_ip_address_needed = False wsgi_socket_directive_needed = True # maybe we are using non-standard ports? vhost_site_url = conf.get('Invenio', 'CFG_SITE_URL').replace("http://", "") if vhost_site_url.startswith("https://"): ## The installation is configured to require HTTPS for any connection vhost_site_url = vhost_site_url.replace("https://", "") vhost_site_url_port = '80' vhost_site_secure_url = conf.get('Invenio', 'CFG_SITE_SECURE_URL').replace("https://", "") vhost_site_secure_url_port = '443' if ':' in vhost_site_url: vhost_site_url, vhost_site_url_port = vhost_site_url.split(':', 1) if ':' in vhost_site_secure_url: vhost_site_secure_url, vhost_site_secure_url_port = vhost_site_secure_url.split(':', 1) if vhost_site_url_port != '80' or vhost_site_secure_url_port != '443': listen_directive_needed = True ## OK, let's create Apache vhost files: if not os.path.exists(apache_conf_dir): os.mkdir(apache_conf_dir) apache_vhost_file = apache_conf_dir + os.sep + \ 'invenio-apache-vhost.conf' apache_vhost_ssl_file = apache_conf_dir + os.sep + \ 'invenio-apache-vhost-ssl.conf' apache_vhost_body = """\ AddDefaultCharset UTF-8 ServerSignature Off ServerTokens Prod NameVirtualHost %(vhost_ip_address)s:%(vhost_site_url_port)s %(listen_directive)s %(wsgi_socket_directive)s WSGIPythonHome %(wsgi_python_home)s WSGIRestrictStdout Off deny from all deny from all ServerName %(servername)s ServerAlias %(serveralias)s ServerAdmin %(serveradmin)s DocumentRoot %(webdir)s %(directory_www_directive)s ErrorLog %(logdir)s/apache.err LogLevel warn LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing CustomLog %(logdir)s/apache.log combined_with_timing DirectoryIndex index.en.html index.html Alias /static/ %(webdir)s/static/ Alias /img/ %(webdir)s/img/ Alias /js/ %(webdir)s/js/ Alias /flash/ %(webdir)s/flash/ Alias /css/ %(webdir)s/css/ Alias /export/ %(webdir)s/export/ Alias /MathJax/ %(webdir)s/MathJax/ Alias /jsCalendar/ %(webdir)s/jsCalendar/ Alias /ckeditor/ %(webdir)s/ckeditor/ Alias /mediaelement/ %(webdir)s/mediaelement/ AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1 Alias /robots.txt %(webdir)s/robots.txt Alias /favicon.ico %(webdir)s/favicon.ico WSGIDaemonProcess invenio processes=5 threads=1 display-name=%%{GROUP} inactivity-timeout=3600 maximum-requests=10000 %(wsgiuser)s WSGIImportScript %(wsgidir)s/invenio.wsgi process-group=invenio application-group=%%{GLOBAL} WSGIScriptAlias / %(wsgidir)s/invenio.wsgi WSGIPassAuthorization On %(xsendfile_directive)s %(directory_wsgi_directive)s %(deflate_directive)s """ % {'vhost_site_url_port': vhost_site_url_port, 'servername': vhost_site_url, 'serveralias': vhost_site_url.split('.')[0], 'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'), 'webdir': conf.get('Invenio', 'CFG_WEBDIR'), 'logdir': conf.get('Invenio', 'CFG_LOGDIR'), 'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'), 'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'), 'wsgiuser': conf.get('Invenio', 'CFG_BIBSCHED_PROCESS_USER') and 'user='+conf.get('Invenio', 'CFG_BIBSCHED_PROCESS_USER'), 'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address(conf) or '*', 'listen_directive': listen_directive_needed and 'Listen ' + vhost_site_url_port or \ '#Listen ' + vhost_site_url_port, 'wsgi_python_home': sys.prefix, 'wsgi_socket_directive': (wsgi_socket_directive_needed and \ 'WSGISocketPrefix ' or '#WSGISocketPrefix ') + \ conf.get('Invenio', 'CFG_PREFIX') + os.sep + 'var' + os.sep + 'run', 'xsendfile_directive': xsendfile_directive, 'directory_www_directive': directory_www_directive, 'directory_wsgi_directive': directory_wsgi_directive, 'deflate_directive': deflate_directive, } apache_vhost_ssl_body = """\ ServerSignature Off ServerTokens Prod %(listen_directive)s NameVirtualHost %(vhost_ip_address)s:%(vhost_site_secure_url_port)s %(ssl_pem_directive)s %(ssl_crt_directive)s %(ssl_key_directive)s %(ssl_protocol_directive)s %(ssl_cipher_directive)s WSGIRestrictStdout Off deny from all deny from all ServerName %(servername)s ServerAlias %(serveralias)s ServerAdmin %(serveradmin)s SSLEngine on DocumentRoot %(webdir)s %(directory_www_directive)s ErrorLog %(logdir)s/apache-ssl.err LogLevel warn LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing CustomLog %(logdir)s/apache-ssl.log combined_with_timing DirectoryIndex index.en.html index.html Alias /static/ %(webdir)s/static/ Alias /img/ %(webdir)s/img/ Alias /js/ %(webdir)s/js/ Alias /flash/ %(webdir)s/flash/ Alias /css/ %(webdir)s/css/ Alias /export/ %(webdir)s/export/ Alias /MathJax/ %(webdir)s/MathJax/ Alias /jsCalendar/ %(webdir)s/jsCalendar/ Alias /ckeditor/ %(webdir)s/ckeditor/ Alias /mediaelement/ %(webdir)s/mediaelement/ AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1 Alias /robots.txt %(webdir)s/robots.txt Alias /favicon.ico %(webdir)s/favicon.ico RedirectMatch /sslredirect/(.*) http://$1 WSGIScriptAlias / %(wsgidir)s/invenio.wsgi WSGIPassAuthorization On %(xsendfile_directive)s %(directory_wsgi_directive)s %(deflate_directive)s %(shibboleth_directive)s """ % {'vhost_site_secure_url_port': vhost_site_secure_url_port, 'servername': vhost_site_secure_url, 'serveralias': vhost_site_secure_url.split('.')[0], 'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'), 'webdir': conf.get('Invenio', 'CFG_WEBDIR'), 'logdir': conf.get('Invenio', 'CFG_LOGDIR'), 'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'), 'wsgidir' : os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'), 'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address(conf) or '*', 'listen_directive' : listen_directive_needed and 'Listen ' + vhost_site_secure_url_port or \ '#Listen ' + vhost_site_secure_url_port, 'ssl_pem_directive': ssl_pem_directive_needed and \ 'SSLCertificateFile %s' % ssl_pem_path or \ '#SSLCertificateFile %s' % ssl_pem_path, 'ssl_crt_directive': ssl_pem_directive_needed and \ '#SSLCertificateFile %s' % ssl_crt_path or \ 'SSLCertificateFile %s' % ssl_crt_path, 'ssl_key_directive': ssl_pem_directive_needed and \ '#SSLCertificateKeyFile %s' % ssl_key_path or \ 'SSLCertificateKeyFile %s' % ssl_key_path, 'ssl_protocol_directive': ssl_pem_directive_needed and \ 'SSLProtocol all -SSLv2 -SSLv3' or \ '#SSLProtocol all -SSLv2 -SSLv3', 'ssl_cipher_directive': ssl_pem_directive_needed and \ 'SSLCipherSuite HIGH:MEDIUM:!ADH' or \ '#SSLCipherSuite HIGH:MEDIUM:!ADH', 'xsendfile_directive': xsendfile_directive, 'directory_www_directive': directory_www_directive, 'directory_wsgi_directive': directory_wsgi_directive, 'deflate_directive': deflate_directive, 'shibboleth_directive': shibboleth_directive, } # write HTTP vhost snippet: if os.path.exists(apache_vhost_file): shutil.copy(apache_vhost_file, apache_vhost_file + '.OLD') fdesc = open(apache_vhost_file, 'w') fdesc.write(apache_vhost_body) fdesc.close() print print "Created file", apache_vhost_file # write HTTPS vhost snippet: vhost_ssl_created = False if conf.get('Invenio', 'CFG_SITE_SECURE_URL').startswith("https://"): if os.path.exists(apache_vhost_ssl_file): shutil.copy(apache_vhost_ssl_file, apache_vhost_ssl_file + '.OLD') fdesc = open(apache_vhost_ssl_file, 'w') fdesc.write(apache_vhost_ssl_body) fdesc.close() vhost_ssl_created = True print "Created file", apache_vhost_ssl_file print wrap_text_in_a_box("""\ Apache virtual host configuration file(s) for your Invenio site was(were) created. Please check created file(s) and activate virtual host(s). For example, you can put the following include statements in your httpd.conf:\n Include %s %s Please see the INSTALL file for more details. """ % (apache_vhost_file, (vhost_ssl_created and 'Include ' or '#Include ') + apache_vhost_ssl_file)) print ">>> Apache conf files created." def cli_cmd_get(conf, varname): """ Return value of VARNAME read from CONF files. Useful for third-party programs to access values of conf options such as CFG_PREFIX. Return None if VARNAME is not found. """ try: if not varname: raise Exception("ERROR: Please specify a configuration variable.") varname = varname.lower() # do not pay attention to section names yet: all_options = {} for section in conf.sections(): for option in conf.options(section): all_options[option] = conf.get(section, option) varvalue = all_options.get(varname, None) if varvalue is None: raise Exception() print varvalue except Exception, e: if e.message: print e.message sys.exit(1) def cli_cmd_list(conf): """ Print a list of all conf options and values from CONF. """ sections = conf.sections() sections.sort() for section in sections: options = conf.options(section) options.sort() for option in options: print option.upper(), '=', conf.get(section, option) def _grep_version_from_executable(path_to_exec, version_regexp): """ Try to detect a program version by digging into its binary PATH_TO_EXEC and looking for VERSION_REGEXP. Return program version as a string. Return empty string if not succeeded. """ from invenio.shellutils import run_shell_command exec_version = "" if os.path.exists(path_to_exec): dummy1, cmd2_out, dummy2 = run_shell_command("strings %s | grep %s", (path_to_exec, version_regexp)) if cmd2_out: for cmd2_out_line in cmd2_out.split("\n"): if len(cmd2_out_line) > len(exec_version): # the longest the better exec_version = cmd2_out_line return exec_version _RE_APACHE_MAJOR_VERSION = re.compile(r"Apache/(\d+\.\d+)") def guess_apache_24(apache_versions=None): """ Returns True if it looks like the system is running Apache 2.4 or later. """ if apache_versions is None: apache_versions = detect_apache_version() for apache_version in apache_versions: g = _RE_APACHE_MAJOR_VERSION.search(apache_version) if g: try: version = float(g.group(1)) except ValueError: continue if version >= 2.4: return True return False def detect_apache_version(): """ Try to detect Apache version by localizing httpd or apache executables and grepping inside binaries. Return list of all found Apache versions and paths. (For a given executable, the returned format is 'apache_version [apache_path]'.) Return empty list if no success. """ from invenio.shellutils import run_shell_command out = [] dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache") for apache in cmd_out.split("\n"): apache_version = _grep_version_from_executable(apache, '^Apache\/') if apache_version: out.append("%s [%s]" % (apache_version, apache)) return out def cli_cmd_detect_system_details(conf): """ Detect and print system details such as Apache/Python/MySQL versions etc. Useful for debugging problems on various OS. """ import MySQLdb print ">>> Going to detect system details..." print "* Hostname: " + socket.gethostname() print "* Invenio version: " + conf.get("Invenio", "CFG_VERSION") print "* Python version: " + sys.version.replace("\n", " ") print "* Apache version: " + ";\n ".join(detect_apache_version()) print "* MySQLdb version: " + MySQLdb.__version__ try: from invenio.dbquery import run_sql print "* MySQL version:" for key, val in run_sql("SHOW VARIABLES LIKE 'version%'") + \ run_sql("SHOW VARIABLES LIKE 'charact%'") + \ run_sql("SHOW VARIABLES LIKE 'collat%'"): if False: print " - %s: %s" % (key, val) elif key in ['version', 'character_set_client', 'character_set_connection', 'character_set_database', 'character_set_results', 'character_set_server', 'character_set_system', 'collation_connection', 'collation_database', 'collation_server']: print " - %s: %s" % (key, val) except ImportError: print "* ERROR: cannot import dbquery" print ">>> System details detected successfully." def cli_cmd_upgrade(conf): """ Command for applying upgrades """ from invenio.inveniocfg_upgrader import cmd_upgrade cmd_upgrade(conf) def cli_cmd_upgrade_check(conf): """ Command for running pre-upgrade checks """ from invenio.inveniocfg_upgrader import cmd_upgrade_check cmd_upgrade_check(conf) def cli_cmd_upgrade_show_pending(conf): """ Command for showing upgrades ready to be applied """ from invenio.inveniocfg_upgrader import cmd_upgrade_show_pending cmd_upgrade_show_pending(conf) def cli_cmd_upgrade_show_applied(conf): """ Command for showing all upgrades already applied. """ from invenio.inveniocfg_upgrader import cmd_upgrade_show_applied cmd_upgrade_show_applied(conf) def cli_cmd_upgrade_create_release_recipe(conf, path): """ Create a new release upgrade recipe (for developers). """ from invenio.inveniocfg_upgrader import cmd_upgrade_create_release_recipe cmd_upgrade_create_release_recipe(conf, path) def cli_cmd_upgrade_create_standard_recipe(conf, path, depends_on=None, release=False): """ Create a new upgrade recipe (for developers). """ from invenio.inveniocfg_upgrader import cmd_upgrade_create_standard_recipe cmd_upgrade_create_standard_recipe(conf, path, depends_on=depends_on, release=release) def prepare_option_parser(): """Parse the command line options.""" class InvenioOption(Option): """ Option class that implements the action 'store_append_const' which will 1) append to list in options. 2) take a value and store in options. Useful for e.g. appending a const to an actions list, while also taking an option value and storing it. This ensures that we can run actions in the order they are given on the command-line. Python 2.4 compatibility note: *append_const* action is not available in Python 2.4, so it is implemented here, together with the new action *store_append_const*. """ ACTIONS = Option.ACTIONS + ("store_append_const", "append_const") STORE_ACTIONS = Option.STORE_ACTIONS + ("store_append_const", "append_const") TYPED_ACTIONS = Option.TYPED_ACTIONS + ("store_append_const", ) ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("store_append_const", ) CONST_ACTIONS = getattr(Option, 'CONST_ACTIONS', ()) + ("store_append_const", "append_const") def take_action(self, action, dest, opt, value, values, parser): if action == "store_append_const": # Combination of 'store' and 'append_const' actions values.ensure_value(dest, []).append(self.const) value_dest = self.const.replace('-', '_') setattr(values, value_dest, value) elif action == "append_const" and not hasattr(Option, 'CONST_ACTIONS'): values.ensure_value(dest, []).append(self.const) else: Option.take_action(self, action, dest, opt, value, values, parser) def _check_const(self): if self.action not in self.CONST_ACTIONS and self.const is not None: raise OptionError( "'const' must not be supplied for action %r" % self.action, self) CHECK_METHODS = [ Option._check_action, Option._check_type, Option._check_choice, Option._check_dest, _check_const, Option._check_nargs, Option._check_callback, ] parser = OptionParser(option_class=InvenioOption, description="Invenio configuration and administration CLI tool", formatter=IndentedHelpFormatter(max_help_position=31)) parser.add_option("-V", "--version", action="store_true", help="print version number") finish_options = OptionGroup(parser, "Options to finish your installation") finish_options.add_option("", "--create-apache-conf", dest='actions', const='create-apache-conf', action="append_const", help="create Apache configuration files") finish_options.add_option("", "--create-tables", dest='actions', const='create-tables', action="append_const", help="create DB tables for Invenio") finish_options.add_option("", "--load-webstat-conf", dest='actions', const='load-webstat-conf', action="append_const", help="load the WebStat configuration") finish_options.add_option("", "--drop-tables", dest='actions', const='drop-tables', action="append_const", help="drop DB tables of Invenio") finish_options.add_option("", "--check-openoffice", dest='actions', const='check-openoffice', action="append_const", help="check for correctly set up of openoffice temporary directory") parser.add_option_group(finish_options) demotest_options = OptionGroup(parser, "Options to set up and test a demo site") demotest_options.add_option("", "--create-demo-site", dest='actions', const='create-demo-site', action="append_const", help="create demo site") demotest_options.add_option("", "--load-demo-records", dest='actions', const='load-demo-records', action="append_const", help="load demo records") demotest_options.add_option("", "--remove-demo-records", dest='actions', const='remove-demo-records', action="append_const", help="remove demo records, keeping demo site") demotest_options.add_option("", "--drop-demo-site", dest='actions', const='drop-demo-site', action="append_const", help="drop demo site configurations too") demotest_options.add_option("", "--run-unit-tests", dest='actions', const='run-unit-tests', action="append_const", help="run unit test suite (needs demo site)") demotest_options.add_option("", "--run-regression-tests", dest='actions', const='run-regression-tests', action="append_const", help="run regression test suite (needs demo site)") demotest_options.add_option("", "--run-web-tests", dest='actions', const='run-web-tests', action="append_const", help="run web tests in a browser (needs demo site, Firefox, Selenium IDE)") parser.add_option_group(demotest_options) config_options = OptionGroup(parser, "Options to update config files in situ") config_options.add_option("", "--update-all", dest='actions', const='update-all', action="append_const", help="perform all the update options") config_options.add_option("", "--update-config-py", dest='actions', const='update-config-py', action="append_const", help="update config.py file from invenio.conf file") config_options.add_option("", "--update-dbquery-py", dest='actions', const='update-dbquery-py', action="append_const", help="update dbquery.py with DB credentials from invenio.conf") config_options.add_option("", "--update-dbexec", dest='actions', const='update-dbexec', action="append_const", help="update dbexec with DB credentials from invenio.conf") config_options.add_option("", "--update-bibconvert-tpl", dest='actions', const='update-bibconvert-tpl', action="append_const", help="update bibconvert templates with CFG_SITE_URL from invenio.conf") config_options.add_option("", "--update-web-tests", dest='actions', const='update-web-tests', action="append_const", help="update web test cases with CFG_SITE_URL from invenio.conf") parser.add_option_group(config_options) reset_options = OptionGroup(parser, "Options to update DB tables") reset_options.add_option("", "--reset-all", dest='actions', const='reset-all', action="append_const", help="perform all the reset options") reset_options.add_option("", "--reset-sitename", dest='actions', const='reset-sitename', action="append_const", help="reset tables to take account of new CFG_SITE_NAME*") reset_options.add_option("", "--reset-siteadminemail", dest='actions', const='reset-siteadminemail', action="append_const", help="reset tables to take account of new CFG_SITE_ADMIN_EMAIL") reset_options.add_option("", "--reset-fieldnames", dest='actions', const='reset-fieldnames', action="append_const", help="reset tables to take account of new I18N names from PO files") reset_options.add_option("", "--reset-recstruct-cache", dest='actions', const='reset-recstruct-cache', action="append_const", help="reset record structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE") parser.add_option_group(reset_options) upgrade_options = OptionGroup(parser, "Options to upgrade your installation") upgrade_options.add_option("", "--upgrade", dest='actions', const='upgrade', action="append_const", help="apply all pending upgrades") upgrade_options.add_option("", "--upgrade-check", dest='actions', const='upgrade-check', action="append_const", help="run pre-upgrade checks for pending upgrades") upgrade_options.add_option("", "--upgrade-show-pending", dest='actions', const='upgrade-show-pending', action="append_const", help="show pending upgrades") upgrade_options.add_option("", "--upgrade-show-applied", dest='actions', const='upgrade-show-applied', action="append_const", help="show history of applied upgrades") upgrade_options.add_option("", "--upgrade-create-standard-recipe", dest='actions', metavar='REPOSITORY[,DIR]', const='upgrade-create-standard-recipe', action="store_append_const", help="create a new standard upgrade recipe (for developers)") upgrade_options.add_option("", "--upgrade-create-release-recipe", dest='actions', metavar='REPOSITORY[,DIR]', const='upgrade-create-release-recipe', action="store_append_const", help="create a new release upgrade recipe (for developers)") parser.add_option_group(upgrade_options) helper_options = OptionGroup(parser, "Options to help the work") helper_options.add_option("", "--list", dest='actions', const='list', action="append_const", help="print names and values of all options from conf files") helper_options.add_option("", "--get", dest='actions', const='get', action="store_append_const", metavar="OPTION", help="get value of a given option from conf files") helper_options.add_option("", "--conf-dir", action="store", metavar="PATH", help="path to directory where invenio*.conf files are [optional]") helper_options.add_option("", "--detect-system-details", dest='actions', const='detect-system-details', action="append_const", help="print system details such as Apache/Python/MySQL versions") parser.add_option_group(helper_options) parser.add_option('--yes-i-know', action='store_true', dest='yes-i-know', help='use with care!') return parser def prepare_conf(options): """ Read configuration files """ conf = ConfigParser() confdir = getattr(options, 'conf_dir', None) if confdir is None: ## try to detect path to conf dir (relative to this bin dir): confdir = re.sub(r'/bin$', '/etc', sys.path[0]) if confdir and not os.path.exists(confdir): raise Exception("ERROR: bad --conf-dir option value - directory does not exists.") sys.exit(1) ## read conf files: for conffile in [confdir + os.sep + 'invenio.conf', confdir + os.sep + 'invenio-autotools.conf', confdir + os.sep + 'invenio-local.conf', ]: if os.path.exists(conffile): conf.read(conffile) else: if not conffile.endswith("invenio-local.conf"): # invenio-local.conf is optional, otherwise stop raise Exception("ERROR: Badly guessed conf file location %s (Please use --conf-dir option.)" % conffile) return conf def main(*cmd_args): """Main entry point.""" # Allow easier testing if not cmd_args: cmd_args = sys.argv[1:] # Parse arguments parser = prepare_option_parser() (options, dummy_args) = parser.parse_args(list(cmd_args)) if getattr(options, 'version', False): print_version() else: # Read configuration try: conf = prepare_conf(options) except Exception, e: print e sys.exit(1) ## Decide what to do actions = getattr(options, 'actions', None) if not actions: print """ERROR: Please specify a command. Please see '--help'.""" sys.exit(1) for action in actions: if action == 'get': cli_cmd_get(conf, getattr(options, 'get', None)) elif action == 'list': cli_cmd_list(conf) elif action == 'detect-system-details': cli_cmd_detect_system_details(conf) elif action == 'create-tables': cli_cmd_create_tables(conf) elif action == 'load-webstat-conf': cli_cmd_load_webstat_conf(conf) elif action == 'drop-tables': cli_cmd_drop_tables(conf) elif action == 'check-openoffice': cli_check_openoffice(conf) elif action == 'create-demo-site': cli_cmd_create_demo_site(conf) elif action == 'load-demo-records': cli_cmd_load_demo_records(conf) elif action == 'remove-demo-records': cli_cmd_remove_demo_records(conf) elif action == 'drop-demo-site': cli_cmd_drop_demo_site(conf) elif action == 'run-unit-tests': cli_cmd_run_unit_tests(conf) elif action == 'run-regression-tests': cli_cmd_run_regression_tests(conf) elif action == 'run-web-tests': cli_cmd_run_web_tests(conf) elif action == 'update-all': cli_cmd_update_config_py(conf) cli_cmd_update_dbquery_py(conf) cli_cmd_update_dbexec(conf) cli_cmd_update_bibconvert_tpl(conf) cli_cmd_update_web_tests(conf) elif action == 'update-config-py': cli_cmd_update_config_py(conf) elif action == 'update-dbquery-py': cli_cmd_update_dbquery_py(conf) elif action == 'update-dbexec': cli_cmd_update_dbexec(conf) elif action == 'update-bibconvert-tpl': cli_cmd_update_bibconvert_tpl(conf) elif action == 'update-web-tests': cli_cmd_update_web_tests(conf) elif action == 'reset-all': cli_cmd_reset_sitename(conf) cli_cmd_reset_siteadminemail(conf) cli_cmd_reset_fieldnames(conf) cli_cmd_reset_recstruct_cache(conf) elif action == 'reset-sitename': cli_cmd_reset_sitename(conf) elif action == 'reset-siteadminemail': cli_cmd_reset_siteadminemail(conf) elif action == 'reset-fieldnames': cli_cmd_reset_fieldnames(conf) elif action == 'reset-recstruct-cache': cli_cmd_reset_recstruct_cache(conf) elif action == 'create-apache-conf': cli_cmd_create_apache_conf(conf) elif action == 'upgrade': cli_cmd_upgrade(conf) elif action == 'upgrade-check': cli_cmd_upgrade_check(conf) elif action == 'upgrade-show-pending': cli_cmd_upgrade_show_pending(conf) elif action == 'upgrade-show-applied': cli_cmd_upgrade_show_applied(conf) elif action == 'upgrade-create-standard-recipe': cli_cmd_upgrade_create_standard_recipe(conf, getattr(options, 'upgrade_create_standard_recipe', None)) elif action == 'upgrade-create-release-recipe': cli_cmd_upgrade_create_release_recipe(conf, getattr(options, 'upgrade_create_release_recipe', None)) else: print "ERROR: Unknown command", action sys.exit(1) if __name__ == '__main__': main() diff --git a/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc b/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc index 61ca08d9b..66c79237a 100644 --- a/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc +++ b/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc @@ -1,1144 +1,1144 @@ ## -*- mode: html; coding: utf-8; -*- ## This file is part of Invenio. ## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

Contents

1. Introduction, using roles
2. WebAccess admin interface
3. Example pages, illustrating snapshots
4. Managing accounts / Access policy
5. Managing login methods
6. Firewall-like role definitions

1. INTRODUCTION, USING ROLES

   WebAccess is a common RBAC, role based access control, for all of
   Invenio. This means that users are connected to roles that cover
   different areas of access. I.e administrator of the photo
   collection or system librarian. Users can be active in
   different areas and of course connected to as many roles as needed.
 
   The roles are connected to actions. An action identifies a task you
   can perform in Invenio. It can be defined to take any number of
   arguments in order to more clearly describe what you are allowing
   connected users to do.
 
   For example the system librarian can be allowed to run bibindex on
   the different indexes. To allow system librarians to run the
   bibindex indexing on the field author we connect role system
   librarian with action runbibindex using the argument
   index='author'.
 
   Additionally, roles could have firewall-like role
   definitions. A definition is a formal description of which
   users are entitled to belong to the role. So you have two ways for
   connecting users to roles. Either linking explicitly a user with the
   role or describing the characteristics that makes users belong to
   the role.
 
   WebAccess is based on allowing users to perform actions. This means
   that only allowed actions are stored in the access control engine's
   database.

2. WEBACCESS ADMIN INTERFACE

 All the WebAccess Administration web pages have certain
 features/design choices in common
 
 - Divided into steps
 
   The process of adding new authorizations/information is
   stepwise. The subtitle contains information about wich step you are
   on and what you are supposed to do.
 
 - Restart from any wanted step
 
   You can always start from an earlier step by simply clicking the
   wanted button. This is not a way to undo changes! No information
   about previous database is kept, so all changes are definite.
 
 - Change or new entry must confirmed
 
   On all the pages you will be asked to confirm the change, with
   information about what kind of change you are about to perform.
 
 - Links to other relevant admin areas on the right side
 
   To make it easier to perform your administration tasks, we have
   added a menu area on the right hand side of these pages. The menu
   contain links to other relevant admin pages and change according to
   the page you are on and the information you have selected.

3. EXAMPLE PAGES

 I. Role area
 II. Example - connecting role and user
 
 
 I. Role area
 
   Administration tasks starts in one of the administration areas. The
   role area is the main area from where you can perform all your
   managing tasks. The other admin areas are just other ways of
   entering.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration

Role Administration

administration with roles as access point
Users:
add or remove users from the access to a role and its priviliges.
Authorizations/Actions:
these terms means almost the same, but an authorization is a
connection between a role and an action (possibly) containing arguments.
Roles:
see all the information attached to a role and decide if you want to
delete it.
id name description definition users authorizations / actions role
2 photoadmin Photo collection administrator None add / delete add / modify / remove modify / delete show details
1 superadmin superuser with all rights allow email /.*@cern.ch/ add / delete add / modify / remove modify / delete show details
3 webaccessadmin WebAccess administrator allow nickname 'jekyll' add / delete add / modify / remove modify / delete show details
Create new role
go here to add a new role.
Create new action
go here to add a new action.
 
 
 II. Example - connecting role and user
 
   One of the important tasks that can be handled via the WebAccess Admin Web Interface
   is the delegation of access rights to users. This is done by connecting them to the
   different roles offered.
 
   The task is divided into 5 simple and comprehensive steps. Below follows the pages from
   the different steps with comments on the ongoing procedure.
 
 - step 1 - select a role
 
   You must first select the role you want to connect users to. All the available roles are
   listed alfabetically in a select box. Just find the wanted role and select it. Then click on
   the button saying "select role".
 
   If you start from the Role Area, this step is already done, and you start directly on step 2.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration > Connect user to role

Connect user to role

step 1 - select a role
1. select role
Create new role
go here to add a new role.
 
 
 - step 2 - search for users
 
   As you can see, the subtitle of the page has now changed. The subtitle always tells you
   which step you are on and what your current task is.
 
   There can be possibly thousands of users using your online library, therefore it is important
   to make it easier to identify the user you are looking for. Give part of, or the entire search
   string and all users with partly matching e-mails will be listed on the next step.
 
   You can also see that the right hand menu has changed. This area is always updated with links
   to related admin areas.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration > Connect user to role

Connect user to role

step 2 - search for users
1. select role
2. search pattern
Create new role
go here to add a new role.
Remove users
remove users from role superadmin.
Connected users
show all users connected to role superadmin.
Add authorization
start adding new authorizations to role superadmin.
 
 
 - step 3 - select a user.
 
   The select box contains all users with partly matching e-mail addresses. Select the one
   you want to connect to the role and continue.
 
   Notice the navigation trail that tells you were on the Administrator pages you are currently
   working.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration > Connect user to role

Connect user to role

step 3 - select a user
1. select role
2. search pattern
3. select user
Create new role
go here to add a new role.
Remove users
remove users from role superadmin.
Connected users
show all users connected to role superadmin.
Add authorization
start adding new authorizations to role superadmin.
 
 
 - step 4 - confirm to add user
 
   All WebAccess Administrator web pages display the action you are about to peform, this
   means explaining what kind of addition, change or update will be done to your access control
   data.
 
   If you are happy with your decision, simply confirm it.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration > Connect user to role

Connect user to role

step 4 - confirm to add user
1. select role
2. search pattern
3. select user
add user mikael.vik@cern.ch to role superadmin?
Create new role
go here to add a new role.
Remove users
remove users from role superadmin.
Connected users
show all users connected to role superadmin.
Add authorization
start adding new authorizations to role superadmin.
 
 
 - step 5 - confirm user added.
 
   The user has now been added to this role. You can easily continue adding more users to this
   role be restarting from step 2 or 3. You can also go directly to another area and keep working
   on the same role.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration > Connect user to role

Connect user to role

step 5 - confirm user added
1. select role
2. search pattern
3. select user
add user mikael.vik@cern.ch to role superadmin?

confirm: user mikael.vik@cern.ch added to role superadmin.
Create new role
go here to add a new role.
Remove users
remove users from role superadmin.
Connected users
show all users connected to role superadmin.
Add authorization
start adding new authorizations to role superadmin.
 
 
 - we are done
 
   This example is very similar to all the other pages where you administrate WebAccess. The pages
   are an easy gateway to maintaing access control rights and share a lot of features.
   - divided into steps
   - restart from any wanted step (not undo)
   - changes must be confirmed
   - link to other relevant areas
   - prevent unwanted input
 
   As an administrator with access to these pages you are free to manage the rights any way you want.

IV. Managing accounts and access policy

   Here you can administrate the accounts and the access policy for your Invenio installation.
 
   - Access policy:
 
     To change the access policy, the general config file (or
     access_control_config.py) must be edited manually in a text
     editor. The site can there be defined as opened or closed, you can
     edit the access policy level for guest accounts, registered
     accounts and decide when to warn the owner of the account when
     something happens with it, either when it is created, deleted or
     approved.  The Apache server must be restarted after modifying
     these settings.
 
     The two levels for guest account, are:
        0 - Allow guest accounts
        1 - Do not allow guest accounts
     The five levels for normal accounts, are:
        0 - Allow user to create account, automatically activate new accounts
        1 - Allow user to create account, administrator must activate account
        2 - Only administrators can create account. User cannot edit the email address.
        3 - Users cannot register or update account information (email/password)
        4 - User cannot change default login method
     You can configure Invenio to send an email:
        1. To an admin email-address when an account is created
        2. To the owner of an account when it is created
        3. To the owner of an account when it is activated
        4. To the owner of an account when it is deleted
 
     Define how open the site is:
       0 = normal operation of the site
       1 = read-only site, all write operations temporarily closed
       2 = site fully closed
       3 = database connections disabled
       CFG_ACCESS_CONTROL_LEVEL_SITE = 0
     Access policy for guests:
       0 = Allow guests to search,
       1 = Guests cannot search (all users must login)
       CFG_ACCESS_CONTROL_LEVEL_GUESTS = 0
     Access policy for accounts:
       0 = Users can register, automatically acticate accounts
       1 = Users can register, but admin must activate the accounts
       2 = Users cannot register or change email address, only admin can register accounts.
       3 = Users cannot register or update email address or password, only admin can register accounts.
       4 = Same as 3, but user cannot change login method.
       CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS = 0
 
     Limit email addresses available to use when register a new account (example: cern.ch):
       CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN = ""
 
     Send an email when a new account is created by an user:
       CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS = 0
 
     Send an email to the user notifying when the account is created:
       CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT = 0
 
     Send an email to the user notifying when the account is activated:
       CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION = 0
 
     Send an email to the user notifying when the account is deleted/rejected:
       CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION = 0
 
   - Account overview:
     Here you find an overview of the number of guest accounts, registered accounts and accounts
     awaiting activation, with a link to the activation page.
 
   - Create account:
     For creating new accounts, the email address must be unique. If configured to do so, an email
     will be sent to the given address when an account is created.
 
   - Edit accounts:
     For activating or rejecting accounts in addition to modifying them. An activated account can be
     inactivated for a short period of time, but this will not warn the account owner. To find accounts
     enter a part of the email address of the account and then search. This may take some time. If there
     are more than the selected number of accounts per page, you can use the next/prev links to switch
     pages. The accounts to search in can also be limited to only activated or not activated accounts.
 
   - Edit account:
     When editing one account, you can change the email address, password, delete the account, or modify
     the baskets or alerts belonging to one account. Which login method should be the default for this
     account can also be selected. To modify baskets or alerts, you need to login as the user, and
     modify the desired data as a normal user. Remember to log out as the user when you are finished
     editing.
     Here you can also edit the user's REST API keys changing its status to the desired one. This status
     is specified by CFG_REST_API_KEY_STATUS (only the first three are mandatory):
       0 = OK
       1 = REMOVED
       2 = REVOKED
       3 = WARNING1
       4 = WARNING2
       5 = WARNING3

V. Managing login methods

    Invenio supports using external login systems to authenticate users.
 
    When a user wants to login, the username and password given by the user is checked against the selected
    system, if the user is authenticated by the external system, a valid email-address is returned to
    Invenio and used to recognize the user within Invenio.
 
    If a new user is trying to login without having an account, using an external login system, an account
    is automatically created in Invenio to recognize and store the users settings. The password for the
    local account is randomly generated.
 
    If you want the user to be unable to change login method and account username / password, forcing use
    of certain external systems, set CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS to 4 as mentioned in the last paragraph.
 
    If a user is changing login method from an external one to the internal, he also need to either change the
    password before logging out, or set the password via the lost password email service.
 
    If you are using CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS with a value greater than 1 note
    that, even if the first login of a user through an external authentication technically means registering
    the user into the system, this is not the semantic expected behaviour by the user. The user is already
    registered at an authority that we trust, so there's no need to prevent the user from being imported
    into the system. That's why for external authentication CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS is not
    considered apart from what said above.
 
    If a external login system is used, you may want to protect the users username / password using HTTPS.
 
    To add new system, two changes must be made (for the time being):
    - The name of the method, if it is default or not, and an instance of the class must be added to the variable
      CFG_EXTERNAL_AUTHENTICATION in access_control_config.py.
 
    - A class must be created derived from the class external_authentication inside file
      external_authentication.py. This class must include at least the
      function auth_user. This function returns a valid email-address in Invenio if the user
      is authenticated, not necessarily the same entered by the user as username. If the user
      is not authenticated, return None.
      The class could also provide five more methods: fetch_user_preferences, user_exists,
      fetch_user_groups_membership and fetch_all_users_groups_membership.
      The first should take an email and eventually a password and should return a dictionary of keys
      and value representing external preferences, infos or settings. If, for some reasons, you like
      to force some kind of hiding for some particular field you should export the related key
      prefixed by "HIDDEN_". Those fields won't be displayed in tables and pages regarding external
      settings.
      The second method should check through the external system if a particular email exists. If you
      provide such a method then a user will be able to switch from and to this authorization method.
      The third method should take an email and (if necessary) a password
      and should return a dictionary of external_groups_names toghether with their description, for which
      the user has a membership. Those groups will be merged into the groups system.
      The user will be a member of those groups and will be able to use them in any place
      where groups are useful, but won't be able to unsubscribe or to administrate them.
      The fourth method should just return a dictionary of external groups as keys and tuples containing
      a group description and a list of email of users belonging to each groups. Those memberships
      will be merged into the database in the way done by the previous method, but could
      provide batch synchronization of groups.
      The fifth method should just return the nickname as is known by the external authentication
      system, given the usual email/username and the password.
      Note: if your system has more than one external login methods then incoherence in the groups
      memberships could happen when a user switch his login method. This will be fixed some times in the
      future.
      If you add as an attribute of your class the enforce_external_nicknames and set it to True, this will enforce
      the system to import external nicknames whenever the user login with the external login method for the
      first time. Since a nickname is not changable this will stay fixed forever. If this nickname is
      already registered in the system (suppose that is linked with a local account) then it will not be
      imported. If this variable doesn't exist or is set to False then no nickname will be
      imported and the user will be free to choose a nickname in the future (and then this will again
      stay forever).
      Note: every method will receive as last parameter the mod_python request object, that could
      be used for particular purposes.
 
 
      Example template:
      from invenio.external_authentication import ExternalAuth, InvenioWebAccessExternalAuthError
 
      class ExternalAuthFoo(ExternalAuth):
          """External authentication template example."""
 
          def __init__ (self):
              """Initialize stuff here."""
              self.name = None
              self.enforce_external_nicknames = False
              pass
 
          def auth_user(self, username, password, req=None):
              """Authenticate user-supplied USERNAME and PASSWORD.
              Return None if authentication failed, or the email address of the
              person if the authentication was successful.  In order to do
              this you may perhaps have to keep a translation table between
              usernames and email addresses.
              Raise InvenioWebAccessExternalAuthError in case of external troubles.
              """
              raise NotImplementedError
              #return None
 
          def user_exists(self, email, req=None):
              """Checks against external_authentication for existance of email.
              @return True if the user exists, False otherwise
              """
              raise NotImplementedError
 
          def fetch_user_groups_membership(self, username, password=None, req=None):
              """Given a username, returns a dictionary of groups
              and their description to which the user is subscribed.
              Raise InvenioWebAccessExternalAuthError in case of troubles.
              """
              raise NotImplementedError
              #return {}
 
          def fetch_user_preferences(self, username, password=None, req=None):
              """Given a username and a password, returns a dictionary of keys and
              values, corresponding to external infos and settings.
 
              userprefs = {"telephone": "2392489",
              "address": "10th Downing Street"}
              """
              raise NotImplementedError
             #return {}
 
          def fetch_all_users_groups_membership(self, req=None):
              """Fetch all the groups with a description, and users who belong to
              each groups.
              @return {'mygroup': ('description', ['email1', 'email2', ...]), ...}
              """
              raise NotImplementedError
 
          def fetch_user_nickname(self, username, password, req=None):
              """Given a username and a password, returns the right nickname belonging
              to that user (username could be an email).
              """
              raise NotImplementedError
              #return Nickname

VI. FIREWALL-LIKE ROLE DEFINITIONS

The FireRole language description 
     In the WebAccess RBAC system, roles are built up from their names,
     description and definition.
 
     A definition is the way to formally implicitly define which users belong
     to which roles.
 
     A definition is expressed in a firewall like rules language. It's built up
     by rows which are matched from top to bottom, in order to decide if the
     current user (wethever he/she is logged in or not) may belong to a role.
 
     Any row has this syntax:
 
         ALLOW/DENY ANY/ALL
 
         ALLOW/DENY FROM/UNTIL "YYYY-MM-DD"
 
         or
 
         ALLOW/DENY [NOT] field {one or more values}
 
     The rows are parsed from top to bottom. If a row matches the user than the
     user belongs to the role if the rule is an ALLOW rule, otherwise, if the
     rule is a DENY one, the user doesn't belong to the role.
 
     A rule of the kind ALLOW|DENY ANY always matches, regardless of the user.
 
     Note, in place of ANY you can use the word ALL. The semantic is the same. The
     system support both to let the user comply with the English grammar.
 
     The second type of rule is interpreted as follows: given a date in the
     form "YYYY-MM-DD" (double-)quoted), the rule is matched if, when using FROM,
     the current date is either identical or 'bigger' than the given date, or if,
     when using UNTIL, the current date is either identical or 'smaller' than the
     given date. If the rule starts with ALLOW and is matched
     then the next row is evaluated. If it is not matched, then the whole FireRole
     will evaluate into a DENY ALL. If the rule starts with DENY and
     is matched then the whole FireRole while evaluate as a DENY ALL. If it is
     not matched then the next row is evaluated.
 
     The third type of rule is interpreted as follows: given a dictionary
     of keys:values describing a user (we will cover this below), the rule
     considers the value associated with the key named in field, and checks
     if it corresponds to at least one of the values in the "one or more values" list.
     This is a list of comma separated strings, which can be literal
     (double-)quoted strings or regexps (marked by `/' ... `/' signs). If at
     least a value matches (literally or through the regexp language), the
     whole rule is considered to match.
     If the optional NOT keyword is specified than if at least a value of the
     rule matches the rule is skipped, otherwise if all the value of the rules
     don't match the whole rule matches.
 
     A DENY ALL rule is implicitly added at the end of every definition. Note that
     this imply that, if you are using e.g. a temporal rule (FROM/UNTIL), you
     should explicitly add an additional row with value ALLOW ANY,
     if you actually want to allow users in the specified timeframe.
 
     Any field is valid, but only rules concerning fields which currently
     exist in the user describing dictionary are checked. All the rules
     with non existant fields are skipped.
 
     The user describing dictionary (user_info) is built at runtime with all the informations
     that can be gathered about the current user (and its session).
     Currently valid fields are: uid, email, nickname, remote_ip,
     remote_host, groups and all the external settings provided
     by the external authentication systems (e.g. CERN SSO provides:
     external_authmethod, external_building, external_department, external_email,
     external_external, external_firstname, external_fullname, external_homdir,
     external_homeinstitute, external_lastname, external_login, external_mobilenumber,
     external_phonenumber).
 
     Among those fields there are some special cases, which are remote_ip and
     (apache_)groups. Rules can refer to remote_ip either using a literal
     expression for specifing list of single ips, or a usual regexp (or list
     of regexps), or, also, using the common network group/mask notation
     (e.g. "127.0.0.0/24") as a literal string, which is a mix between literal
     expressions and regexps. (apache_)groups are related to group memberships.
     Since a user will probably belong to more than a group, then the rule
     matches if there's at least one group to which the user belong, that matches
     at least one of the expressions (NOT rules behave as you can imagine).
 
     The dictionary is built using the current user session. If the user is
     authenticated in some way (apache, locally, externally, SSO...) then more
     infos could be provided to the firerole system in order to decide if the
     user should belong to a role or not.
 
     The default fields that are always there are:
     
    
         
  • uid: an integer representing the user id
    • 
         
  • nickname: the nickname of the user
    • 
         
  • email: the email of the user
    • 
         
  • group/groups: local or external group to which the user belong
    • 
         
  • guest: 1 if the user is a guest (not logged), 0 otherwise
    • 
     

     plus all the external setting retrieved by an external authentication system.
 
     If the action to which the role defined is raised from the webinterface of
     Invenio, then you will have those additional fields:
     
    
         
  • remote_ip: the remote ip address of the user who is browsing
    • 
         
  • remote_host: the remote hostname of the user who is browsing
    • 
         
  • referer: the webpage from where the user is coming from
    • 
         
  • uri: the uri the user is visiting
    • 
         
  • agent: the agent string describing the user's browser
    • 
     

 
     Note that you can specify either (apache_)group or (apache_)groups (with or
     without the trailing s). They are semantically equal and are supported just
     to let people comply with the English grammar.
 
     Every rule is case-insensitive (apart values which must match literally
     and regexp values which don't explicitly specify case-insesitive matches).
 
     Every rule may contain comments preceded by the '#' character.
     Any comment is discarded.
 
     When you set a definition for a role, it is actually compiled and stored
     in a binary compressed form inside the database. If the syntax isn't correct
     this will be stated and the definition won't be set or updated.
 
     Example of role definition:
         allow not email /.*@gmail.com/,/.*@hotmail.com/
-        deny group badguys
+        deny group "badguys"
         allow remote_ip "127.0.0.0/24"
         deny all
 
     This definition would match all users whose emails don't end with @gmail.com and
     @hotmail.com, or who don't belong to the group badguys and have remote_ip
     in the 24bit mask network of 127.0.0.0. All the the other users don't belong
     to the role which is being defined.
 
     If you want to discover which keys are available on your system to build a FireRole
     rule, just login with your account in your installation and visit your account page,
     by activating verbose=9 variable. Under the tile you will se the available keys and
     values that you can use to build a FireRole rule. All but fields prefixed with
     precached_ are usuable.
diff --git a/modules/websession/lib/websession_regression_tests.py b/modules/websession/lib/websession_regression_tests.py index c44eacf9f..e8697713c 100644 --- a/modules/websession/lib/websession_regression_tests.py +++ b/modules/websession/lib/websession_regression_tests.py @@ -1,116 +1,123 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2006, 2007, 2008, 2010, 2011, 2014 CERN. +## Copyright (C) 2006, 2007, 2008, 2010, 2011, 2014, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. # pylint: disable=E1102 """WebSession Regression Test Suite.""" __revision__ = \ "$Id$" import unittest from mechanize import Browser -from invenio.config import CFG_SITE_SECURE_URL, CFG_SITE_ADMIN_EMAIL +from invenio.config import CFG_SITE_SECURE_URL, CFG_SITE_ADMIN_EMAIL, \ + CFG_MISCUTIL_SMTP_PORT from invenio.testutils import make_test_suite, run_test_suite, \ test_web_page_content, merge_error_messages from invenio.dbquery import run_sql class WebSessionWebPagesAvailabilityTest(unittest.TestCase): """Check WebSession web pages whether they are up or not.""" def test_your_account_pages_availability(self): """websession - availability of Your Account pages""" baseurl = CFG_SITE_SECURE_URL + '/youraccount/' # not testing 'change' endpoint here, since it is accepting # only POST requests _exports = ['', 'edit', 'lost', 'display', 'send_email', 'youradminactivities', 'delete', 'logout', 'login', 'register'] error_messages = [] for url in [baseurl + page for page in _exports]: error_messages.extend(test_web_page_content(url)) if error_messages: self.fail(merge_error_messages(error_messages)) return def test_your_groups_pages_availability(self): """websession - availability of Your Groups pages""" baseurl = CFG_SITE_SECURE_URL + '/yourgroups/' _exports = ['', 'display', 'create', 'join', 'leave', 'edit', 'members'] error_messages = [] for url in [baseurl + page for page in _exports]: error_messages.extend(test_web_page_content(url)) if error_messages: self.fail(merge_error_messages(error_messages)) return -class WebSessionLostYourPasswordTest(unittest.TestCase): - """Test Lost Your Passwords functionality.""" - - def test_lost_your_password_for_internal_accounts(self): - """websession - sending lost password for internal admin account""" - - try_with_account = CFG_SITE_ADMIN_EMAIL - - # click on "send lost password" for CFG_SITE_ADMIN_EMAIL internal account - browser = Browser() - browser.open(CFG_SITE_SECURE_URL + "/youraccount/lost") - browser.select_form(nr=0) - browser['p_email'] = try_with_account - try: - browser.submit() - except Exception, e: - # Restore the admin password (send_email set it to random number) - run_sql("UPDATE user SET password=AES_ENCRYPT(email, '')" - "WHERE id=1") - self.fail("Obtained %s: probably the email server is not installed " - "correctly." % e) - - - - # verify the response: - expected_response = "Okay, a password reset link has been emailed to " + \ - try_with_account - lost_password_response_body = browser.response().read() - try: - lost_password_response_body.index(expected_response) - except ValueError: +if CFG_MISCUTIL_SMTP_PORT: + class WebSessionLostYourPasswordTest(unittest.TestCase): + """Test Lost Your Passwords functionality.""" + + def test_lost_your_password_for_internal_accounts(self): + """websession - sending lost password for internal admin account""" + + try_with_account = CFG_SITE_ADMIN_EMAIL + + # click on "send lost password" for CFG_SITE_ADMIN_EMAIL internal account + browser = Browser() + browser.open(CFG_SITE_SECURE_URL + "/youraccount/lost") + browser.select_form(nr=0) + browser['p_email'] = try_with_account + try: + browser.submit() + except Exception, e: + # Restore the admin password (send_email set it to random number) + run_sql("UPDATE user SET password=AES_ENCRYPT(email, '')" + "WHERE id=1") + self.fail("Obtained %s: probably the email server is not installed " + "correctly." % e) + + + + # verify the response: + expected_response = "Okay, a password reset link has been emailed to " + \ + try_with_account + lost_password_response_body = browser.response().read() + try: + lost_password_response_body.index(expected_response) + except ValueError: + # Restore the admin password (send_email set it to random number) + run_sql("UPDATE user SET password=AES_ENCRYPT(email, '')" + "WHERE id=1") + self.fail("Expected to see %s, got %s." % \ + (expected_response, lost_password_response_body)) + + def tearDown(self): # Restore the admin password (send_email set it to random number) run_sql("UPDATE user SET password=AES_ENCRYPT(email, '')" "WHERE id=1") - self.fail("Expected to see %s, got %s." % \ - (expected_response, lost_password_response_body)) +else: + # SMTP server is not available. let's skip this test + class WebSessionLostYourPasswordTest(unittest.TestCase): + pass - def tearDown(self): - # Restore the admin password (send_email set it to random number) - run_sql("UPDATE user SET password=AES_ENCRYPT(email, '')" - "WHERE id=1") TEST_SUITE = make_test_suite(WebSessionWebPagesAvailabilityTest, WebSessionLostYourPasswordTest) if __name__ == "__main__": run_test_suite(TEST_SUITE, warn_user=True) diff --git a/modules/websession/lib/webuser_regression_tests.py b/modules/websession/lib/webuser_regression_tests.py index 13561d5f8..ea0dcd071 100644 --- a/modules/websession/lib/webuser_regression_tests.py +++ b/modules/websession/lib/webuser_regression_tests.py @@ -1,291 +1,292 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN. +## Copyright (C) 2007, 2008, 2009, 2010, 2011, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. # pylint: disable=E1102 """WebSession Regression Test Suite.""" __revision__ = \ "$Id$" import unittest from mechanize import Browser from invenio.dbquery import run_sql -from invenio.config import CFG_SITE_SECURE_URL +from invenio.config import CFG_SITE_SECURE_URL, CFG_MISCUTIL_SMTP_PORT from invenio.testutils import make_test_suite, run_test_suite from invenio import webuser class IsUserSuperAdminTests(unittest.TestCase): """Test functions related to the isUserSuperAdmin function.""" def setUp(self): self.id_admin = run_sql('SELECT id FROM user WHERE nickname="admin"')[0][0] self.id_hyde = run_sql('SELECT id FROM user WHERE nickname="hyde"')[0][0] def test_isUserSuperAdmin_admin(self): """webuser - isUserSuperAdmin with admin""" self.failUnless(webuser.isUserSuperAdmin(webuser.collect_user_info(self.id_admin))) def test_isUserSuperAdmin_hyde(self): """webuser - isUserSuperAdmin with hyde""" self.failIf(webuser.isUserSuperAdmin(webuser.collect_user_info(self.id_hyde))) class WebSessionYourSettingsTests(unittest.TestCase): """Check WebSession web pages whether they are up or not.""" def tearDown(self): run_sql('DELETE FROM user WHERE email="foo@cds.cern.ch"') run_sql('DELETE FROM user WHERE email="FOO@cds.cern.ch"') def test_password_setting(self): """webuser - check password settings""" browser = Browser() browser.open(CFG_SITE_SECURE_URL + "/youraccount/login") browser.select_form(nr=0) browser['p_un'] = 'admin' browser['p_pw'] = '' browser.submit() expected_response = "You are logged in as admin" login_response_body = browser.response().read() try: login_response_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, login_response_body)) # Going to set new password from "" to "123" browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_password") browser['old_password'] = "" browser['password'] = "123" browser['password2'] = "123" browser.submit() expected_response = "Password successfully edited" change_password_body = browser.response().read() try: change_password_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, change_password_body)) # Going to set a wrong old password browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_password") browser['old_password'] = "321" browser['password'] = "123" browser['password2'] = "123" browser.submit() expected_response = "Wrong old password inserted" change_password_body = browser.response().read() try: change_password_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, change_password_body)) # Going to put different new passwords browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_password") browser['old_password'] = "123" browser['password'] = "123" browser['password2'] = "321" browser.submit() expected_response = "Both passwords must match" change_password_body = browser.response().read() try: change_password_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, change_password_body)) # Reset the situation browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_password") browser['old_password'] = "123" browser['password'] = "" browser['password2'] = "" browser.submit() expected_response = "Password successfully edited" change_password_body = browser.response().read() try: change_password_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, change_password_body)) - def test_email_caseless(self): - """webuser - check email caseless""" - browser = Browser() - browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") - browser.select_form(nr=0) - browser['p_email'] = 'foo@cds.cern.ch' - browser['p_nickname'] = 'foobar' - browser['p_pw'] = '' - browser['p_pw2'] = '' - browser.submit() - - expected_response = "Account created" - login_response_body = browser.response().read() - try: - login_response_body.index(expected_response) - except ValueError: - self.fail("Expected to see %s, got %s." % \ - (expected_response, login_response_body)) - - - browser = Browser() - browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") - browser.select_form(nr=0) - browser['p_email'] = 'foo@cds.cern.ch' - browser['p_nickname'] = 'foobar2' - browser['p_pw'] = '' - browser['p_pw2'] = '' - browser.submit() - - expected_response = "Registration failure" - login_response_body = browser.response().read() - try: - login_response_body.index(expected_response) - except ValueError: - self.fail("Expected to see %s, got %s." % \ - (expected_response, login_response_body)) - - browser = Browser() - browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") - browser.select_form(nr=0) - browser['p_email'] = 'FOO@cds.cern.ch' - browser['p_nickname'] = 'foobar2' - browser['p_pw'] = '' - browser['p_pw2'] = '' - browser.submit() - - expected_response = "Registration failure" - login_response_body = browser.response().read() - try: - login_response_body.index(expected_response) - except ValueError: - self.fail("Expected to see %s, got %s." % \ - (expected_response, login_response_body)) + if CFG_MISCUTIL_SMTP_PORT: + def test_email_caseless(self): + """webuser - check email caseless""" + browser = Browser() + browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") + browser.select_form(nr=0) + browser['p_email'] = 'foo@cds.cern.ch' + browser['p_nickname'] = 'foobar' + browser['p_pw'] = '' + browser['p_pw2'] = '' + browser.submit() + + expected_response = "Account created" + login_response_body = browser.response().read() + try: + login_response_body.index(expected_response) + except ValueError: + self.fail("Expected to see %s, got %s." % \ + (expected_response, login_response_body)) + + + browser = Browser() + browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") + browser.select_form(nr=0) + browser['p_email'] = 'foo@cds.cern.ch' + browser['p_nickname'] = 'foobar2' + browser['p_pw'] = '' + browser['p_pw2'] = '' + browser.submit() + + expected_response = "Registration failure" + login_response_body = browser.response().read() + try: + login_response_body.index(expected_response) + except ValueError: + self.fail("Expected to see %s, got %s." % \ + (expected_response, login_response_body)) + + browser = Browser() + browser.open(CFG_SITE_SECURE_URL + "/youraccount/register") + browser.select_form(nr=0) + browser['p_email'] = 'FOO@cds.cern.ch' + browser['p_nickname'] = 'foobar2' + browser['p_pw'] = '' + browser['p_pw2'] = '' + browser.submit() + + expected_response = "Registration failure" + login_response_body = browser.response().read() + try: + login_response_body.index(expected_response) + except ValueError: + self.fail("Expected to see %s, got %s." % \ + (expected_response, login_response_body)) def test_select_records_per_group(self): """webuser - test of user preferences setting""" # logging in as admin browser = Browser() browser.open(CFG_SITE_SECURE_URL + "/youraccount/login") browser.select_form(nr=0) browser['p_un'] = 'admin' browser['p_pw'] = '' browser.submit() expected_response = "You are logged in as admin" login_response_body = browser.response().read() try: login_response_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, login_response_body)) # Going to edit page and setting records per group to 20 browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_websearch_settings") browser['group_records'] = ["25"] browser.submit() expected_response = "User settings saved correctly" changed_settings_body = browser.response().read() try: changed_settings_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, changed_settings_body)) # Going to the search page, making an empty search browser.open(CFG_SITE_SECURE_URL) browser.select_form(nr=0) browser.submit() expected_response = "1 - 25" records_found_body = browser.response().read() try: records_found_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, records_found_body)) # Going again to edit and setting records per group back to 10 browser.open(CFG_SITE_SECURE_URL + "/youraccount/edit") browser.select_form(name="edit_websearch_settings") browser['group_records'] = ["10"] browser.submit() expected_response = "User settings saved correctly" changed_settings_body = browser.response().read() try: changed_settings_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, changed_settings_body)) # Logging out! browser.open(CFG_SITE_SECURE_URL + "/youraccount/logout") expected_response = "You are no longer recognized" logout_response_body = browser.response().read() try: logout_response_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, logout_response_body)) # Logging in again browser.open(CFG_SITE_SECURE_URL + "/youraccount/login") browser.select_form(nr=0) browser['p_un'] = 'admin' browser['p_pw'] = '' browser.submit() expected_response = "You are logged in as admin" login_response_body = browser.response().read() try: login_response_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, login_response_body)) # Let's go to search and check that the setting is still there browser.open(CFG_SITE_SECURE_URL) browser.select_form(nr=0) browser.submit() expected_response = "1 - 10" records_found_body = browser.response().read() try: records_found_body.index(expected_response) except ValueError: self.fail("Expected to see %s, got %s." % \ (expected_response, records_found_body)) return TEST_SUITE = make_test_suite(WebSessionYourSettingsTests, IsUserSuperAdminTests) if __name__ == "__main__": run_test_suite(TEST_SUITE, warn_user=True) diff --git a/modules/websubmit/lib/websubmit_regression_tests.py b/modules/websubmit/lib/websubmit_regression_tests.py index c90819997..85410f84c 100644 --- a/modules/websubmit/lib/websubmit_regression_tests.py +++ b/modules/websubmit/lib/websubmit_regression_tests.py @@ -1,273 +1,274 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. -## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2013 CERN. +## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2013, 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """WebSubmit Regression Test Suite.""" __revision__ = "$Id$" import unittest import os from logging import StreamHandler, DEBUG from cStringIO import StringIO from invenio.websubmit_file_converter import get_file_converter_logger from invenio.errorlib import register_exception from invenio.config import CFG_SITE_URL, CFG_PREFIX, CFG_TMPDIR, CFG_PATH_PDFTK from invenio.testutils import make_test_suite, run_test_suite, \ test_web_page_content, merge_error_messages from invenio import websubmit_file_stamper class WebSubmitWebPagesAvailabilityTest(unittest.TestCase): """Check WebSubmit web pages whether they are up or not.""" def test_submission_pages_availability(self): """websubmit - availability of submission pages""" baseurl = CFG_SITE_URL + '/submit/' _exports = ['', 'direct'] error_messages = [] for url in [baseurl + page for page in _exports]: error_messages.extend(test_web_page_content(url)) if error_messages: self.fail(merge_error_messages(error_messages)) return def test_publiline_pages_availability(self): """websubmit - availability of approval pages""" baseurl = CFG_SITE_URL _exports = ['/approve.py', '/publiline.py', '/yourapprovals.py'] error_messages = [] for url in [baseurl + page for page in _exports]: error_messages.extend(test_web_page_content(url)) if error_messages: self.fail(merge_error_messages(error_messages)) return def test_your_submissions_pages_availability(self): """websubmit - availability of Your Submissions pages""" baseurl = CFG_SITE_URL _exports = ['/yoursubmissions.py'] error_messages = [] for url in [baseurl + page for page in _exports]: error_messages.extend(test_web_page_content(url)) if error_messages: self.fail(merge_error_messages(error_messages)) return def test_help_page_availability(self): """websubmit - availability of WebSubmit help page""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/help/submit-guide', expected_text="Submit Guide")) class WebSubmitLegacyURLsTest(unittest.TestCase): """ Check that the application still responds to legacy URLs""" def test_legacy_help_page_link(self): """websubmit - legacy Submit Guide page link""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/help/submit', expected_text="Submit Guide")) self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/help/submit/', expected_text="Submit Guide")) self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/help/submit/index.en.html', expected_text="Submit Guide")) self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/help/submit/access.en.html', expected_text="Submit Guide")) class WebSubmitXSSVulnerabilityTest(unittest.TestCase): """Test possible XSS vulnerabilities of the submission engine.""" def test_xss_in_submission_doctype(self): """websubmit - no XSS vulnerability in doctype parameter""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/submit?doctype=%3CSCRIPT%3Ealert%28%22XSS%22%29%3B%3C%2FSCRIPT%3E', expected_text='Unable to find document type: <SCRIPT>alert("XSS")', username="jekyll", password="j123ekyll")) def test_xss_in_submission_act(self): """websubmit - no XSS vulnerability in act parameter""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/submit?doctype=DEMOTHE&access=1_1&act=%3CSCRIPT%3Ealert%28%22XSS%22%29%3B%3C%2FSCRIPT%3E', expected_text='Invalid doctype and act parameters', username="jekyll", password="j123ekyll")) def test_xss_in_submission_page(self): """websubmit - no XSS vulnerability in access parameter""" self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/submit?doctype=DEMOTHE&access=/../../../etc/passwd&act=SBI&startPg=1&ln=en&ln=en', expected_text='Invalid parameters', username="jekyll", password="j123ekyll")) self.assertEqual([], test_web_page_content(CFG_SITE_URL + '/submit?doctype=DEMOTHE&access=%3CSCRIPT%3Ealert%28%22XSS%22%29%3B%3C%2FSCRIPT%3E&act=SBI', expected_text='Invalid parameters', username="jekyll", password="j123ekyll")) def WebSubmitFileConverterTestGenerator(): from invenio.websubmit_file_converter import get_conversion_map, can_convert if can_convert('.odt', '.txt'): ## Special test for unoconv/LibreOffice yield WebSubmitFileConverterTest(os.path.join(CFG_PREFIX, 'lib', 'webtest', 'invenio', 'test.odt'), '.odt', '.txt') if can_convert('.doc', '.txt'): ## Special test for unoconv/LibreOffice yield WebSubmitFileConverterTest(os.path.join(CFG_PREFIX, 'lib', 'webtest', 'invenio', 'test.doc'), '.doc', '.txt') for from_format in get_conversion_map().keys(): input_file = os.path.join(CFG_PREFIX, 'lib', 'webtest', 'invenio', 'test%s' % from_format) if not os.path.exists(input_file): ## Can't run such a test because there is no test example continue for to_format in get_conversion_map().keys(): if from_format == to_format: continue conversion_map = can_convert(from_format, to_format) if conversion_map: if [converter for converter in conversion_map if converter[0].__name__ == 'unoconv']: ## We don't want to test unoconv which is tested separately continue yield WebSubmitFileConverterTest(input_file, from_format, to_format) class WebSubmitFileConverterTest(unittest.TestCase): """Test WebSubmit file converter tool""" def __init__(self, input_file, from_format, to_format): super(WebSubmitFileConverterTest, self).__init__('_run_test') self.from_format = from_format self.to_format = to_format self.input_file = input_file def setUp(self): logger = get_file_converter_logger() self.log = StringIO() logger.setLevel(DEBUG) for handler in logger.handlers: logger.removeHandler(handler) handler = StreamHandler(self.log) handler.setLevel(DEBUG) logger.addHandler(handler) def shortDescription(self): return """websubmit - test %s to %s conversion""" % (self.from_format, self.to_format) def _run_test(self): from invenio.websubmit_file_converter import InvenioWebSubmitFileConverterError, convert_file try: tmpdir_snapshot1 = set(os.listdir(CFG_TMPDIR)) output_file = convert_file(self.input_file, output_format=self.to_format) tmpdir_snapshot2 = set(os.listdir(CFG_TMPDIR)) tmpdir_snapshot2.discard(os.path.basename(output_file)) if not os.path.exists(output_file): raise InvenioWebSubmitFileConverterError("output_file %s was not correctly created" % output_file) if tmpdir_snapshot2 - tmpdir_snapshot1: raise InvenioWebSubmitFileConverterError("Some temporary files were left over: %s" % (tmpdir_snapshot2 - tmpdir_snapshot1)) except Exception, err: register_exception(alert_admin=True) self.fail("ERROR: when converting from %s to %s: %s, the log contained: %s" % (self.from_format, self.to_format, err, self.log.getvalue())) if CFG_PATH_PDFTK: class WebSubmitStampingTest(unittest.TestCase): """Test WebSubmit file stamping tool""" def test_stamp_coverpage(self): """websubmit - creation of a PDF cover page stamp (APIs)""" file_stamper_options = { 'latex-template' : "demo-stamp-left.tex", 'latex-template-var' : {'REPORTNUMBER':'TEST-2010','DATE':'10/10/2000'}, 'input-file' : CFG_PREFIX + "/lib/webtest/invenio/test.pdf", 'output-file' : "test-stamp-coverpage.pdf", 'stamp' : "coverpage", 'layer' : "foreground", 'verbosity' : 0, } try: (stamped_file_path_only, stamped_file_name) = \ websubmit_file_stamper.stamp_file(file_stamper_options) except: self.fail("Stamping failed") # Test that file is now bigger... assert os.path.getsize(os.path.join(stamped_file_path_only, stamped_file_name)) > 12695 def test_stamp_firstpage(self): """websubmit - stamping first page of a PDF (APIs)""" file_stamper_options = { 'latex-template' : "demo-stamp-left.tex", 'latex-template-var' : {'REPORTNUMBER':'TEST-2010','DATE':'10/10/2000'}, 'input-file' : CFG_PREFIX + "/lib/webtest/invenio/test.pdf", 'output-file' : "test-stamp-firstpage.pdf", 'stamp' : "first", 'layer' : "background", 'verbosity' : 0, } try: (stamped_file_path_only, stamped_file_name) = \ websubmit_file_stamper.stamp_file(file_stamper_options) except: self.fail("Stamping failed") # Test that file is now bigger... assert os.path.getsize(os.path.join(stamped_file_path_only, stamped_file_name)) > 12695 def test_stamp_allpages(self): """websubmit - stamping all pages of a PDF (APIs)""" file_stamper_options = { 'latex-template' : "demo-stamp-left.tex", 'latex-template-var' : {'REPORTNUMBER':'TEST-2010','DATE':'10/10/2000'}, 'input-file' : CFG_PREFIX + "/lib/webtest/invenio/test.pdf", 'output-file' : "test-stamp-allpages.pdf", 'stamp' : "all", 'layer' : "foreground", 'verbosity' : 0, } try: (stamped_file_path_only, stamped_file_name) = \ websubmit_file_stamper.stamp_file(file_stamper_options) except: self.fail("Stamping failed") # Test that file is now bigger... assert os.path.getsize(os.path.join(stamped_file_path_only, stamped_file_name)) > 12695 else: ## pdftk is not available. Disabling stamping-related ## regression tests. class WebSubmitStampingTest(unittest.TestCase): pass + TEST_SUITE = make_test_suite(WebSubmitWebPagesAvailabilityTest, WebSubmitLegacyURLsTest, WebSubmitXSSVulnerabilityTest, WebSubmitStampingTest) for test in WebSubmitFileConverterTestGenerator(): TEST_SUITE.addTest(test) if __name__ == "__main__": run_test_suite(TEST_SUITE, warn_user=True) diff --git a/requirements-extras.txt b/requirements-extras.txt deleted file mode 100644 index 2a731b8a1..000000000 --- a/requirements-extras.txt +++ /dev/null @@ -1,17 +0,0 @@ -# More requirements files are needed, since e.g gnuplot-py and h5py -# import numpy in their setup.py, which means they have to be -# installed in a second step. -gnuplot-py==1.8 - -# Following packages are optional (if you do development you probably want to install them): -pylint -http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz/download -pep8 -selenium -winpdb -mock -ipython -cython -nose -nosexcover -flake8 diff --git a/.travis.yml b/scripts/Makefile.am similarity index 59% copy from .travis.yml copy to scripts/Makefile.am index 1aaed64e3..a0b2b422a 100644 --- a/.travis.yml +++ b/scripts/Makefile.am @@ -1,33 +1,24 @@ -# -*- coding: utf-8 -*- -## ## This file is part of Invenio. -## Copyright (C) 2013, 2014 CERN. +## Copyright (C) 2016 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -notifications: - email: false - -python: - - "2.6" - -install: - - sudo apt-get update - - git clone https://github.com/tiborsimko/invenio-devscripts /tmp/invenio-devscripts - - CFG_INVENIO_SRCDIR=$(pwd) /tmp/invenio-devscripts/invenio-kickstart --yes-i-know --yes-i-really-know - -script: - - sudo -u www-data nosetests /opt/invenio/lib/python/invenio/*_unit_tests.py - - sudo -u www-data nosetests /opt/invenio/lib/python/invenio/*_regression_tests.py +EXTRA_DIST = \ + create-instance.sh \ + drop-instance.sh \ + make-install.sh \ + populate-instance.sh \ + provision-mysql.sh \ + provision-web.sh diff --git a/scripts/create-instance.sh b/scripts/create-instance.sh new file mode 100755 index 000000000..39257fd04 --- /dev/null +++ b/scripts/create-instance.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_MYSQL_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_MYSQL_HOST=192.168.50.11" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBNAME}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBNAME before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBNAME=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBUSER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBUSER before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBUSER=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBPASS}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBPASS before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBPASS=dbpass123" + exit 1 +fi +if [ "${INVENIO_WEB_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10" + exit 1 +fi +if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio" + exit 1 +fi +if [ "${INVENIO_WEB_USER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_USER=www-data" + exit 1 +fi + +# check optional environment variables: +INVENIO_WEB_SMTP_PORT=${INVENIO_WEB_SMTP_PORT:=25} + +# quit on unbound symbols: +set -o nounset + +# runs inside virtual environment? +VIRTUAL_ENV=${VIRTUAL_ENV:=} + +# runs as root or needs sudo? +if [[ "$EUID" -ne 0 ]]; then + sudo='sudo' +else + sudo='' +fi + +create_apache_vhost_ubuntu_precise () { + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ssl-cert + sudo mkdir -p /etc/apache2/ssl + if [ ! -e /etc/apache2/ssl/apache.pem ]; then + sudo DEBIAN_FRONTEND=noninteractive /usr/sbin/make-ssl-cert \ + /usr/share/ssl-cert/ssleay.cnf /etc/apache2/ssl/apache.pem + fi + if [ ! -L /etc/apache2/sites-available/invenio.conf ]; then + sudo ln -fs "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" \ + /etc/apache2/sites-available/invenio.conf + fi + if [ ! -e "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" ]; then + # create them empty for the time being so that apache would start + sudo mkdir -p "${INVENIO_WEB_DSTDIR}/etc/apache/" + sudo touch "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" + sudo chown -R "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}" + fi + if [ ! -L /etc/apache2/sites-available/invenio-ssl.conf ]; then + sudo ln -fs "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" \ + /etc/apache2/sites-available/invenio-ssl.conf + fi + if [ ! -e "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" ]; then + # create them empty for the time being so that apache would start + sudo mkdir -p "${INVENIO_WEB_DSTDIR}/etc/apache/" + sudo touch "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" + sudo chown -R "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}" + fi + if [ -e /etc/apache2/sites-available/default-ssl ]; then + sudo /usr/sbin/a2dissite "*default*" + fi + sudo /usr/sbin/a2ensite "invenio*" + sudo /usr/sbin/a2enmod ssl + sudo /usr/sbin/a2enmod version || echo "[WARNING] Ignoring 'a2enmod version' command; hoping IfVersion is built-in." + sudo /usr/sbin/a2enmod xsendfile + sudo /etc/init.d/apache2 restart +} + +create_apache_vhost_centos6 () { + if ! grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" /etc/httpd/conf/httpd.conf; then + echo "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" | sudo tee -a /etc/httpd/conf/httpd.conf + fi + if ! grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" /etc/httpd/conf/httpd.conf; then + echo "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" | sudo tee -a /etc/httpd/conf/httpd.conf + fi + if ! grep -q "TraceEnable off" /etc/httpd/conf/httpd.conf; then + echo "TraceEnable off" | sudo tee -a /etc/httpd/conf/httpd.conf + fi + if ! grep -q "SSLProtocol all -SSLv2" /etc/httpd/conf/httpd.conf; then + echo "SSLProtocol all -SSLv2" | sudo tee -a /etc/httpd/conf/httpd.conf + fi + sudo sed -i 's,^Alias /error/,#Alias /error/,g' /etc/httpd/conf/httpd.conf +} + +create_symlinks () { + $sudo mkdir -p "${INVENIO_WEB_DSTDIR}" + $sudo chown "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}" + $sudo -u "${INVENIO_WEB_USER}" mkdir -p "${INVENIO_WEB_DSTDIR}/lib/python/invenio" + for pythonversion in python2.4 python2.6 python2.7; do + for libversion in lib lib64 local/lib local/lib64; do + for packageversion in site-packages dist-packages; do + if [ -d "/usr/$libversion/$pythonversion/$packageversion/" ] && [ ! -L "/usr/$libversion/$pythonversion/$packageversion/invenio" ]; then + $sudo ln -s "${INVENIO_WEB_DSTDIR}/lib/python/invenio" "/usr/$libversion/$pythonversion/$packageversion/invenio" + fi + done + done + done +} + +install_sources () { + cd "${INVENIO_SRCDIR}" + rm -rf autom4te.cache/ + aclocal + automake -a + autoconf + ./configure --prefix="${INVENIO_WEB_DSTDIR}" + make clean -s + make -s + sudo -u "${INVENIO_WEB_USER}" make -s install + #sudo -u "${INVENIO_WEB_USER}" make -s install-jquery-plugins + sudo -u "${INVENIO_WEB_USER}" make -s install-mathjax-plugin + sudo -u "${INVENIO_WEB_USER}" make -s install-ckeditor-plugin + sudo -u "${INVENIO_WEB_USER}" make -s install-pdfa-helper-files + sudo -u "${INVENIO_WEB_USER}" make -s install-mediaelement +} + +create_openoffice_tmp_space () { + sudo mkdir -p "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files" + sudo chown -R nobody "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files" + sudo chmod -R 755 "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files" +} + +configure_instance () { + # create invenio-local.conf + echo "[Invenio] +CFG_SITE_URL = http://${INVENIO_WEB_HOST} +CFG_SITE_SECURE_URL = https://${INVENIO_WEB_HOST} +CFG_DATABASE_HOST = ${INVENIO_MYSQL_HOST} +CFG_DATABASE_NAME = ${INVENIO_MYSQL_DBNAME} +CFG_DATABASE_USER = ${INVENIO_MYSQL_DBUSER} +CFG_DATABASE_PASS = ${INVENIO_MYSQL_DBPASS} +CFG_SITE_ADMIN_EMAIL = ${INVENIO_ADMIN_EMAIL} +CFG_SITE_SUPPORT_EMAIL = ${INVENIO_ADMIN_EMAIL} +CFG_WEBALERT_ALERT_ENGINE_EMAIL = ${INVENIO_ADMIN_EMAIL} +CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL = ${INVENIO_ADMIN_EMAIL} +CFG_WEBCOMMENT_DEFAULT_MODERATOR = ${INVENIO_ADMIN_EMAIL} +CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL = ${INVENIO_ADMIN_EMAIL} +CFG_BIBCATALOG_SYSTEM_EMAIL_ADDRESS = ${INVENIO_ADMIN_EMAIL} +CFG_BIBSCHED_PROCESS_USER = ${INVENIO_WEB_USER} +CFG_MISCUTIL_SMTP_PORT = ${INVENIO_WEB_SMTP_PORT} +" | \ + sudo -u "${INVENIO_WEB_USER}" tee "${INVENIO_WEB_DSTDIR}/etc/invenio-local.conf" + + # update instance with this information: + sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --update-all +} + +create_tables () { + sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --create-tables --yes-i-know +} + +create_apache_configuration () { + sudo -u "${INVENIO_WEB_USER}" VIRTUAL_ENV="${VIRTUAL_ENV}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --create-apache-conf +} + +restart_apache_ubuntu_precise () { + $sudo /etc/init.d/apache2 restart +} + +restart_apache_centos6 () { + $sudo /etc/init.d/httpd restart +} + +main () { + + # detect OS distribution and release version: + if hash lsb_release 2> /dev/null; then + os_distribution=$(lsb_release -i | cut -f 2) + os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1) + elif [ -e /etc/redhat-release ]; then + os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release) + os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1) + else + os_distribution="UNDETECTED" + os_release="UNDETECTED" + fi + + # call appropriate provisioning functions: + if [ "$os_distribution" = "Ubuntu" ]; then + if [ "$os_release" = "12" ]; then + create_apache_vhost_ubuntu_precise + create_symlinks + install_sources + create_openoffice_tmp_space + configure_instance + create_tables + create_apache_configuration + restart_apache_ubuntu_precise + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + elif [ "$os_distribution" = "CentOS" ]; then + if [ "$os_release" = "6" ]; then + create_apache_vhost_centos6 + create_symlinks + install_sources + create_openoffice_tmp_space + configure_instance + create_tables + create_apache_configuration + restart_apache_centos6 + exit 1 + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + else + echo "[ERROR] Sorry, unsupported distribution ${os_distribution}." + exit 1 + fi + +} + +main diff --git a/scripts/drop-instance.sh b/scripts/drop-instance.sh new file mode 100755 index 000000000..ae23a0443 --- /dev/null +++ b/scripts/drop-instance.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_MYSQL_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_MYSQL_HOST=192.168.50.11" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBNAME}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBNAME before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBNAME=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBUSER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBUSER before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBUSER=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBPASS}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBPASS before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBPASS=dbpass123" + exit 1 +fi +if [ "${INVENIO_WEB_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10" + exit 1 +fi +if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio" + exit 1 +fi +if [ "${INVENIO_WEB_USER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_USER=www-data" + exit 1 +fi + +# quit on unbound symbols: +set -o nounset + +# runs as root or needs sudo? +if [[ "$EUID" -ne 0 ]]; then + sudo='sudo' +else + sudo='' +fi + + +start_apache_ubuntu_precise () { + $sudo /etc/init.d/apache2 start +} + +stop_apache_ubuntu_precise () { + $sudo /etc/init.d/apache2 stop +} + +start_apache_centos6 () { + $sudo /etc/init.d/httpd start +} + +stop_apache_centos6 () { + $sudo /etc/init.d/httpd stop +} + +drop_apache_vhost_ubuntu_precise () { + stop_apache_ubuntu_precise + if [ -e /etc/apache2/sites-available/default-ssl ]; then + $sudo /usr/sbin/a2ensite "*default*" + fi + if [ -L /etc/apache2/sites-enabled/invenio.conf ]; then + $sudo /usr/sbin/a2dissite "invenio*" + fi + start_apache_ubuntu_precise +} + +drop_apache_vhost_centos6 () { + stop_apache_centos6 + if grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" /etc/httpd/conf/httpd.conf; then + sudo sed -i "s,^Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf,#Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf,g" /etc/httpd/conf/httpd.conf + fi + if grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" /etc/httpd/conf/httpd.conf; then + sudo sed -i "s,^Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf,#Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf,g" /etc/httpd/conf/httpd.conf + fi + sudo sed -i 's,^#Alias /error/,Alias /error/,g' /etc/httpd/conf/httpd.conf + start_apache_centos6 +} + +drop_symlinks () { + for pythonversion in python2.4 python2.6 python2.7; do + for libversion in lib lib64 local/lib local/lib64; do + for packageversion in site-packages dist-packages; do + if [ -d /usr/$libversion/$pythonversion/$packageversion/ ] && [ ! -L /usr/$libversion/$pythonversion/$packageversion/invenio ]; then + $sudo rm /usr/$libversion/$pythonversion/$packageversion/invenio + fi + done + done + done +} + +drop_instance_folder () { + $sudo rm -rf "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files" + $sudo -u "${INVENIO_WEB_USER}" rm -rf "${INVENIO_WEB_DSTDIR}/*" +} + +drop_instance_tables () { + if [ -e "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" ]; then + $sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --drop-tables --yes-i-know + fi +} + +main () { + + # detect OS distribution and release version: + if hash lsb_release 2> /dev/null; then + os_distribution=$(lsb_release -i | cut -f 2) + os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1) + elif [ -e /etc/redhat-release ]; then + os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release) + os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1) + else + os_distribution="UNDETECTED" + os_release="UNDETECTED" + fi + + # call appropriate provisioning functions: + if [ "$os_distribution" = "Ubuntu" ]; then + if [ "$os_release" = "12" ]; then + stop_apache_ubuntu_precise + drop_instance_tables + start_apache_ubuntu_precise + drop_apache_vhost_ubuntu_precise + drop_instance_folder + drop_symlinks + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + elif [ "$os_distribution" = "CentOS" ]; then + if [ "$os_release" = "6" ]; then + stop_apache_centos6 + drop_instance_tables + start_apache_centos6 + drop_apache_vhost_centos6 + drop_instance_folder + drop_symlinks + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + else + echo "[ERROR] Sorry, unsupported distribution ${os_distribution}." + exit 1 + fi + +} + +main diff --git a/scripts/make-install.sh b/scripts/make-install.sh new file mode 100755 index 000000000..d5bf78b6a --- /dev/null +++ b/scripts/make-install.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio" + exit 1 +fi +if [ "${INVENIO_WEB_USER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_USER=www-data" + exit 1 +fi + +# quit on unbound symbols: +set -o nounset + +# install sources and restart WSGI application: +(make -s && sudo -u "${INVENIO_WEB_USER}" make -s install && \ + sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --update-all && \ + sudo -u "${INVENIO_WEB_USER}" touch "${INVENIO_WEB_DSTDIR}/var/www-wsgi/invenio.wsgi") > /dev/null diff --git a/scripts/populate-instance.sh b/scripts/populate-instance.sh new file mode 100755 index 000000000..0ad36b368 --- /dev/null +++ b/scripts/populate-instance.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio" + exit 1 +fi +if [ "${INVENIO_WEB_USER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_USER=www-data" + exit 1 +fi + +# quit on unbound symbols: +set -o nounset + +# runs inside virtual environment? +VIRTUAL_ENV=${VIRTUAL_ENV:=} + +# runs as root or needs sudo? +if [[ "$EUID" -ne 0 ]]; then + sudo='sudo' +else + sudo='' +fi + +create_demo_site () { + $sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --create-demo-site --yes-i-know +} + +load_demo_records () { + $sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --load-demo-records --yes-i-know +} + +apache_wsgi_restart () { + $sudo -u "${INVENIO_WEB_USER}" touch "${INVENIO_WEB_DSTDIR}/var/www-wsgi/invenio.wsgi" +} + +start_apache_ubuntu_precise () { + $sudo /etc/init.d/apache2 start +} + +stop_apache_ubuntu_precise () { + $sudo /etc/init.d/apache2 stop +} + +start_apache_centos6 () { + $sudo /etc/init.d/httpd start +} + +stop_apache_centos6 () { + $sudo /etc/init.d/httpd stop +} + +main () { + + # detect OS distribution and release version: + if hash lsb_release 2> /dev/null; then + os_distribution=$(lsb_release -i | cut -f 2) + os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1) + elif [ -e /etc/redhat-release ]; then + os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release) + os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1) + else + os_distribution="UNDETECTED" + os_release="UNDETECTED" + fi + + # call appropriate provisioning functions: + if [ "$os_distribution" = "Ubuntu" ]; then + if [ "$os_release" = "12" ]; then + stop_apache_ubuntu_precise + create_demo_site + start_apache_ubuntu_precise + load_demo_records + apache_wsgi_restart + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + elif [ "$os_distribution" = "CentOS" ]; then + if [ "$os_release" = "6" ]; then + stop_apache_centos6 + create_demo_site + start_apache_centos6 + load_demo_records + apache_wsgi_restart + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + else + echo "[ERROR] Sorry, unsupported distribution ${os_distribution}." + exit 1 + fi + +} + +main diff --git a/scripts/provision-mysql.sh b/scripts/provision-mysql.sh new file mode 100755 index 000000000..d21d8ea19 --- /dev/null +++ b/scripts/provision-mysql.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_MYSQL_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_MYSQL_HOST=192.168.50.11" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBNAME}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBNAME before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBNAME=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBUSER}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBUSER before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBUSER=invenio1" + exit 1 +fi +if [ "${INVENIO_MYSQL_DBPASS}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBPASS before runnning this script." + echo "[ERROR] Example: INVENIO_MYSQL_DBPASS=dbpass123" + exit 1 +fi +if [ "${INVENIO_WEB_HOST}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script." + echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10" + exit 1 +fi + +# quit on unbound symbols: +set -o nounset + +provision_mysql_ubuntu_precise () { + + # update list of available packages + sudo DEBIAN_FRONTEND=noninteractive apt-get update + + # install MySQL server: + sudo DEBIAN_FRONTEND=noninteractive apt-get -y install \ + mysql-server + + # allow network connections: + if ! grep -q "${INVENIO_MYSQL_HOST}" /etc/mysql/my.cnf; then + sudo sed -i "s/127.0.0.1/${INVENIO_MYSQL_HOST}/" /etc/mysql/my.cnf + fi + + # restart MySQL server: + sudo /usr/sbin/service mysql restart + +} + +provision_mysql_centos6 () { + + # update list of available packages + sudo yum update -y + + # install MySQL server: + sudo yum install -y \ + mysql-server + + # open firewall ports: + if [ -e /sbin/iptables ]; then + thisinputchain="INPUT" + if sudo /sbin/iptables -nL | grep -q 'Chain RH-Firewall-1-INPUT'; then + thisinputchain="RH-Firewall-1-INPUT" + fi + if ! sudo /sbin/iptables -nL | grep -q dpt:3306; then + sudo /sbin/iptables -I $thisinputchain -p tcp -m tcp --dport 3306 -j ACCEPT + sudo /sbin/iptables -I OUTPUT -p tcp -m tcp --dport 3306 -j ACCEPT + fi + if ! sudo /sbin/iptables -nL | grep -q dpt:3306; then + sudo /sbin/iptables -I $thisinputchain -p tcp -m tcp --dport 3306 -j ACCEPT + sudo /sbin/iptables -I OUTPUT -p tcp -m tcp --dport 3306 -j ACCEPT + fi + fi + + # save new firewall rules to survive reboot: + sudo /etc/init.d/iptables save + + # enable MySQL upon reboot: + sudo /sbin/chkconfig mysqld on + + # restart MySQL server: + sudo /etc/init.d/mysqld restart + +} + +setup_db () { + + # create database if it does not exist: + echo "CREATE DATABASE IF NOT EXISTS ${INVENIO_MYSQL_DBNAME} DEFAULT CHARACTER SET utf8;" | \ + mysql -u root -B + + # grant privileges to the user on this database: + echo "GRANT ALL PRIVILEGES ON ${INVENIO_MYSQL_DBNAME}.* TO ${INVENIO_MYSQL_DBUSER}@${INVENIO_WEB_HOST} IDENTIFIED BY '${INVENIO_MYSQL_DBPASS}';" | \ + mysql -u root -B +} + +main () { + + # detect OS distribution and release version: + if hash lsb_release 2> /dev/null; then + os_distribution=$(lsb_release -i | cut -f 2) + os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1) + elif [ -e /etc/redhat-release ]; then + os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release) + os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1) + else + os_distribution="UNDETECTED" + os_release="UNDETECTED" + fi + + # call appropriate provisioning functions: + if [ "$os_distribution" = "Ubuntu" ]; then + if [ "$os_release" = "12" ]; then + provision_mysql_ubuntu_precise + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + elif [ "$os_distribution" = "CentOS" ]; then + if [ "$os_release" = "6" ]; then + provision_mysql_centos6 + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + else + echo "[ERROR] Sorry, unsupported distribution ${os_distribution}." + exit 1 + fi + + # finish with common setups: + setup_db + +} + +main diff --git a/scripts/provision-web.sh b/scripts/provision-web.sh new file mode 100755 index 000000000..690435c62 --- /dev/null +++ b/scripts/provision-web.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +# +# This file is part of Invenio. +# Copyright (C) 2015, 2016 CERN. +# +# Invenio is free software; you can redistribute it +# and/or modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Invenio is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Invenio; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, +# MA 02111-1307, USA. +# +# In applying this license, CERN does not +# waive the privileges and immunities granted to it by virtue of its status +# as an Intergovernmental Organization or submit itself to any jurisdiction. + +# quit on errors: +set -o errexit + +# check environment variables: +if [ "${INVENIO_SRCDIR}" = "" ]; then + echo "[ERROR] Please set environment variable INVENIO_SRCDIR before runnning this script." + echo "[ERROR] Example: export INVENIO_SRCDIR=/vagrant" + exit 1 +fi + +# quit on unbound symbols: +set -o nounset + +# runs inside virtual environment? +VIRTUAL_ENV=${VIRTUAL_ENV:=} + +# runs as root or needs sudo? +if [[ "$EUID" -ne 0 ]]; then + sudo='sudo' +else + sudo='' +fi + +# unattended installation: +export DEBIAN_FRONTEND=noninteractive + +provision_web_ubuntu_precise () { + + # update list of available packages + $sudo DEBIAN_FRONTEND=noninteractive apt-get update + + # install useful system packages + $sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + apache2-mpm-worker \ + automake \ + clisp \ + curl \ + cython \ + gettext \ + giflib-tools \ + git \ + gnuplot poppler-utils \ + html2text \ + ipython \ + libapache2-mod-wsgi \ + libapache2-mod-xsendfile \ + libffi-dev \ + libfreetype6-dev \ + libjpeg-dev \ + libmsgpack-dev \ + libmysqlclient-dev \ + libssl-dev \ + libtiff-dev \ + libxml2-dev \ + libxslt-dev \ + make \ + mysql-client \ + netpbm \ + openOffice.org \ + pdftk \ + pep8 \ + postfix \ + pstotext \ + pylint \ + python-dev \ + python-gnuplot \ + python-libxml2 \ + python-libxslt1 \ + python-nose \ + python-nosexcover \ + python-pip \ + python-uno \ + rlwrap \ + sbcl \ + screen \ + texlive \ + unzip \ + vim + + # grant Apache user the nobody user rights for OpenOffice integration: + echo "www-data ALL=(nobody) NOPASSWD: ALL" | $sudo tee /etc/sudoers.d/www-data + $sudo chmod o-r /etc/sudoers.d/www-data + +} + +provision_web_centos6 () { + + # update list of available packages + $sudo yum update -y + + # add EPEL external repository: + $sudo yum install -y epel-release + + # install useful system tools: + $sudo yum install -y \ + automake \ + curl \ + cython \ + file \ + freetype-devel \ + gcc \ + gcc-c++ \ + gettext \ + gettext-devel \ + git \ + gnuplot-py \ + hdf5-devel \ + ipython \ + libffi-devel \ + libpng-devel \ + libreoffice \ + libreoffice-headless \ + libreoffice-pyuno \ + libxml2-devel \ + libxml2-python \ + libxslt-devel \ + libxslt-python \ + mod_ssl \ + mod_wsgi \ + mysql-devel \ + poppler-utils \ + python-devel \ + python-pip \ + redis \ + rlwrap \ + screen \ + sendmail \ + sudo \ + texlive \ + unzip \ + vim \ + w3m \ + wget + + # open firewall ports: + if [ -e /sbin/iptables ]; then + thisinputchain="INPUT" + if sudo /sbin/iptables -nL | grep -q 'Chain RH-Firewall-1-INPUT'; then + thisinputchain="RH-Firewall-1-INPUT" + fi + if ! sudo /sbin/iptables -nL | grep -q dpt:http; then + sudo /sbin/iptables -I $thisinputchain -p tcp -m tcp --dport 80 -j ACCEPT + sudo /sbin/iptables -I OUTPUT -p tcp -m tcp --dport 80 -j ACCEPT + fi + if ! sudo /sbin/iptables -nL | grep -q dpt:https; then + sudo /sbin/iptables -I $thisinputchain -p tcp -m tcp --dport 443 -j ACCEPT + sudo /sbin/iptables -I OUTPUT -p tcp -m tcp --dport 443 -j ACCEPT + fi + fi + + # save new firewall rules to survive reboot: + sudo /etc/init.d/iptables save + + # enable Apache upon reboot: + sudo /sbin/chkconfig httpd on + + # grant Apache user the nobody user rights for OpenOffice integration: + echo "apache ALL=(nobody) NOPASSWD: ALL" | $sudo tee /etc/sudoers.d/apache + $sudo chmod o-r /etc/sudoers.d/apache + +} + +provision_web_pypi () { + + # install Python packages from PyPI + olddir=$(pwd) + cd "${INVENIO_SRCDIR}" + for reqfile in requirements*.txt; do + if [ -e "$reqfile" ]; then + if [ "$VIRTUAL_ENV" != "" ]; then + pip install -r "$reqfile" + else + sudo pip install -r "$reqfile" + fi + fi + done + cd "${olddir}" + +} + +main () { + + # detect OS distribution and release version: + if hash lsb_release 2> /dev/null; then + os_distribution=$(lsb_release -i | cut -f 2) + os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1) + elif [ -e /etc/redhat-release ]; then + os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release) + os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1) + else + os_distribution="UNDETECTED" + os_release="UNDETECTED" + fi + + # call appropriate provisioning functions: + if [ "$os_distribution" = "Ubuntu" ]; then + if [ "$os_release" = "12" ]; then + provision_web_ubuntu_precise + provision_web_pypi + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + elif [ "$os_distribution" = "CentOS" ]; then + if [ "$os_release" = "6" ]; then + provision_web_centos6 + provision_web_pypi + else + echo "[ERROR] Sorry, unsupported release ${os_release}." + exit 1 + fi + else + echo "[ERROR] Sorry, unsupported distribution ${os_distribution}." + exit 1 + fi + +} + +main