diff --git a/configure.ac b/configure.ac index 968b6b901..0d4d32287 100644 --- a/configure.ac +++ b/configure.ac @@ -1,703 +1,703 @@ ## $Id$ ## Purpose: CDS Invenio main configure.ac file. ## Note: If you change this file, please run "autoreconf" to regenerate the "configure" script. ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## General Public License for more details. +## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. ## Initialize autoconf and automake: AC_INIT(cds-invenio, 0.93.50.20070904, cds.support@cern.ch) AM_INIT_AUTOMAKE(cds-invenio, 0.93.50.20070904) ## Check for install: AC_PROG_INSTALL ## Define WEBURL URL: AC_ARG_WITH(weburl, AC_HELP_STRING([--with-weburl], [specify URL where web files will be reached (e.g. http://webserver.domain.com)]), WEBURL=${withval}) if test -z "$WEBURL"; then AC_MSG_ERROR([ - You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached. + You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached. Example: "--with-weburl=http://webserver.domain.com".]) -fi +fi AC_ARG_WITH(sweburl, AC_HELP_STRING([--with-sweburl], [specify URL where HTTPS files will be reached (e.g. https://webserver.domain.com)]), SWEBURL=${withval}) if test -z "${SWEBURL}"; then SWEBURL="${WEBURL}" AC_MSG_WARN([ - You have not specified secure URL for accessing the site (--with-sweburl). + You have not specified secure URL for accessing the site (--with-sweburl). User authentication will be done in clear text across the network. ]) -fi +fi ## Define DBHOST database host: AC_ARG_WITH(dbhost, AC_HELP_STRING([--with-dbhost], [specify DB server (e.g. sqlserver.domain.com)]), DBHOST=${withval}) if test -z "$DBHOST"; then AC_MSG_ERROR([ - You have not used the "--with-dbhost" argument that should define which DB server to use. + You have not used the "--with-dbhost" argument that should define which DB server to use. Example: "--with-dbhost=sqlserver.domain.com".]) -fi +fi ## Define DBNAME database name: AC_ARG_WITH(dbname, AC_HELP_STRING([--with-dbname], [specify DB name (e.g. cdsinvenio)]), DBNAME=${withval}) if test -z "$DBNAME"; then AC_MSG_ERROR([ - You have not used the "--with-dbname" argument that should define DB name. + You have not used the "--with-dbname" argument that should define DB name. Example: "--with-dbname=cdsinvenio".]) -fi +fi ## Define DBUSER user name: AC_ARG_WITH(dbuser, AC_HELP_STRING([--with-dbuser], [specify DB user name (e.g. cdsinvenio)]), DBUSER=${withval}) if test -z "$DBUSER"; then AC_MSG_ERROR([ - You have not used the "--with-dbuser" argument that should define DB user name. + You have not used the "--with-dbuser" argument that should define DB user name. Example: "--with-dbuser=cdsinvenio".]) -fi +fi ## Define DBPASS user password: AC_ARG_WITH(dbpass, AC_HELP_STRING([--with-dbpass], [specify DB user password (e.g. myp1ss)]), DBPASS=${withval}) if test -z "$DBPASS"; then AC_MSG_ERROR([ - You have not used the "--with-dbpass" argument that should define DB user password. + You have not used the "--with-dbpass" argument that should define DB user password. Example: "--with-dbpass=myp1ss".]) -fi +fi ## Check for gettext support AM_GNU_GETTEXT(external) AM_GNU_GETTEXT_VERSION(0.14.4) ## Check for WML: AC_PATH_PROG(WML, wml) if test -z "$WML"; then AC_MSG_ERROR([ - WML (Website META Language) was not found in your PATH. Please install it first. + WML (Website META Language) was not found in your PATH. Please install it first. Available from .]) fi ## Set option -c in WML so that it can work from a separate build tree WML="${WML} -c" ## Check for MySQL client: AC_PATH_PROG(MYSQL, mysql) if test -z "$MYSQL"; then AC_MSG_ERROR([ MySQL command-line client was not found in your PATH. Please install it first. Available from .]) fi ## Check for Python: AC_MSG_CHECKING(for python) AC_ARG_WITH(python, AC_HELP_STRING([--with-python], [path to a specific Python binary (optional)]), PYTHON=${withval}) if test -n "$PYTHON"; then AC_MSG_RESULT($PYTHON) else AC_PATH_PROG(PYTHON, python) if test -z "$PYTHON"; then AC_MSG_ERROR([ - Python was not found in your PATH. Please either install it + Python was not found in your PATH. Please either install it in your PATH or specify --with-python configure option. Python is available from .]) fi fi ## Check for Python version and modules: AC_MSG_CHECKING(for required Python modules) $PYTHON ${srcdir}/configure-tests.py if test $? -ne 0; then AC_MSG_ERROR([Please fix the above Python problem before continuing.]) fi AC_MSG_RESULT(found) ## Check for PHP AC_PATH_PROG(PHP, php) ## Check for Acrobat Reader: AC_PATH_PROG(ACROREAD, acroread) if test -z "$ACROREAD"; then AC_MSG_WARN([ - Acrobat Reader was not found in your PATH. It is used in - the WebSubmit module for automatic conversion of submitted documents. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + Acrobat Reader was not found in your PATH. It is used in + the WebSubmit module for automatic conversion of submitted documents. + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Acrobat Reader is available from .]) fi ## Check for gzip: AC_PATH_PROG(GZIP, gzip) if test -z "$GZIP"; then AC_MSG_WARN([ - Gzip was not found in your PATH. It is used in + Gzip was not found in your PATH. It is used in the WebSubmit module to compress the data submitted in an archive. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Gzip is available from .]) fi ## Check for gunzip: AC_PATH_PROG(GUNZIP, gunzip) if test -z "$GUNZIP"; then AC_MSG_WARN([ - Gunzip was not found in your PATH. It is used in + Gunzip was not found in your PATH. It is used in the WebSubmit module to correctly deal with submitted compressed files. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Gunzip is available from .]) fi ## Check for tar: AC_PATH_PROG(TAR, tar) if test -z "$TAR"; then AC_MSG_WARN([ - Tar was not found in your PATH. It is used in + Tar was not found in your PATH. It is used in the WebSubmit module to pack the submitted data into an archive. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Tar is available from .]) fi ## Check for ps2pdf: AC_PATH_PROG(PS2PDF, ps2pdf) if test -z "$PS2PDF"; then AC_MSG_WARN([ - ps2pdf was not found in your PATH. It is used in + ps2pdf was not found in your PATH. It is used in the WebSubmit module to convert submitted PostScripts into PDF. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. PS2PDF is available from .]) fi ## Check for pdftotext: AC_PATH_PROG(PDFTOTEXT, pdftotext) if test -z "$PDFTOTEXT"; then AC_MSG_WARN([ - pdftotext was not found in your PATH. It is used for the fulltext indexation + pdftotext was not found in your PATH. It is used for the fulltext indexation of PDF files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure - script. pdftotext is available from . + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure + script. pdftotext is available from . ]) fi ## Check for pstotext: AC_PATH_PROG(PSTOTEXT, pstotext) if test -z "$PSTOTEXT"; then AC_MSG_WARN([ - pstotext was not found in your PATH. It is used for the fulltext indexation + pstotext was not found in your PATH. It is used for the fulltext indexation of PDF and PostScript files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. pstotext is available from . ]) fi ## Check for ps2ascii: AC_PATH_PROG(PSTOASCII, ps2ascii) if test -z "$PSTOASCII"; then AC_MSG_WARN([ - ps2ascii was not found in your PATH. It is used for the fulltext indexation + ps2ascii was not found in your PATH. It is used for the fulltext indexation of PostScript files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. ps2ascii is available from . ]) fi ## Check for antiword: AC_PATH_PROG(ANTIWORD, antiword) if test -z "$ANTIWORD"; then AC_MSG_WARN([ - antiword was not found in your PATH. It is used for the fulltext indexation + antiword was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure - script. antiword is available from . + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure + script. antiword is available from . ]) fi ## Check for catdoc: AC_PATH_PROG(CATDOC, catdoc) if test -z "$CATDOC"; then AC_MSG_WARN([ - catdoc was not found in your PATH. It is used for the fulltext indexation + catdoc was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. catdoc is available from . ]) fi ## Check for wvText: AC_PATH_PROG(WVTEXT, wvText) if test -z "$WVTEXT"; then AC_MSG_WARN([ - wvText was not found in your PATH. It is used for the fulltext indexation + wvText was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. wvText is available from . ]) -fi +fi ## Check for ppthtml: AC_PATH_PROG(PPTHTML, ppthtml) if test -z "$PPTHTML"; then AC_MSG_WARN([ - ppthtml was found in your PATH. It is used for the fulltext indexation + ppthtml was found in your PATH. It is used for the fulltext indexation of Microsoft PowerPoint files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. ppthtml is available from . ]) fi ## Check for xlhtml: AC_PATH_PROG(XLHTML, xlhtml) if test -z "$XLHTML"; then AC_MSG_WARN([ - xlhtml was found in your PATH. It is used for the fulltext indexation + xlhtml was found in your PATH. It is used for the fulltext indexation of Microsoft Excel files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. xlhtml is available from . ]) fi ## Check for html2text: AC_PATH_PROG(HTMLTOTEXT, html2text) if test -z "$HTMLTOTEXT"; then AC_MSG_WARN([ - html2text was found in your PATH. It is used for the fulltext indexation + html2text was found in your PATH. It is used for the fulltext indexation of Microsoft PowerPoint and Excel files. - You can continue without it but you may miss fulltext searching capability - of CDS Invenio. We recomend you to install it first and to rerun the configure + You can continue without it but you may miss fulltext searching capability + of CDS Invenio. We recomend you to install it first and to rerun the configure script. html2text is available from . ]) fi ## Check for Giftext: AC_PATH_PROG(GIFTEXT, giftext) if test -z "$GIFTEXT"; then AC_MSG_WARN([ - Giftext was not found in your PATH. It is used in + Giftext was not found in your PATH. It is used in the WebSubmit module to create an icon from a submitted picture. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Giftext is available from .]) fi ## Check for file: AC_PATH_PROG(FILE, file) if test -z "$FILE"; then AC_MSG_WARN([ - File was not found in your PATH. It is used in + File was not found in your PATH. It is used in the WebSubmit module to check the validity of the submitted files. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. File is available from .]) fi ## Check for convert: AC_PATH_PROG(CONVERT, convert) if test -z "$CONVERT"; then AC_MSG_WARN([ - Convert was not found in your PATH. It is used in + Convert was not found in your PATH. It is used in the WebSubmit module to create an icon from a submitted picture. - You can continue without it but you will miss some CDS Invenio - functionality. We recommend you to install it first and to rerun + You can continue without it but you will miss some CDS Invenio + functionality. We recommend you to install it first and to rerun the configure script. Convert is available from .]) fi ## Check for CLISP: AC_MSG_CHECKING(for clisp) AC_ARG_WITH(clisp, AC_HELP_STRING([--with-clisp], [path to a specific CLISP binary (optional)]), CLISP=${withval}) if test -n "$CLISP"; then AC_MSG_RESULT($CLISP) else AC_PATH_PROG(CLISP, clisp) if test -z "$CLISP"; then AC_MSG_WARN([ GNU CLISP was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, SBCL or CMUCL can be used instead of CLISP.) - You can continue without it but you will miss this feature. We + You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CMUCL - nor SBCL) and to rerun the configure script. + nor SBCL) and to rerun the configure script. GNU CLISP is available from .]) fi fi ## Check for CMUCL: AC_MSG_CHECKING(for cmucl) AC_ARG_WITH(cmucl, AC_HELP_STRING([--with-cmucl], [path to a specific CMUCL binary (optional)]), CMUCL=${withval}) if test -n "$CMUCL"; then AC_MSG_RESULT($CMUCL) else AC_PATH_PROG(CMUCL, cmucl) if test -z "$CMUCL"; then AC_MSG_CHECKING(for lisp) # CMUCL can also be installed under `lisp' exec name - AC_PATH_PROG(CMUCL, lisp) + AC_PATH_PROG(CMUCL, lisp) fi - if test -z "$CMUCL"; then + if test -z "$CMUCL"; then AC_MSG_WARN([ CMUCL was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, CLISP or SBCL can be used instead of CMUCL.) - You can continue without it but you will miss this feature. We + You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP - nor SBCL) and to rerun the configure script. + nor SBCL) and to rerun the configure script. CMUCL is available from .]) - fi + fi fi ## Check for SBCL: AC_MSG_CHECKING(for sbcl) AC_ARG_WITH(sbcl, AC_HELP_STRING([--with-sbcl], [path to a specific SBCL binary (optional)]), SBCL=${withval}) if test -n "$SBCL"; then AC_MSG_RESULT($SBCL) else AC_PATH_PROG(SBCL, sbcl) if test -z "$SBCL"; then AC_MSG_WARN([ SBCL was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, CLISP or CMUCL can be used instead of SBCL.) - You can continue without it but you will miss this feature. We + You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP - nor CMUCL) and to rerun the configure script. + nor CMUCL) and to rerun the configure script. SBCL is available from .]) - fi + fi fi ## Check for gnuplot: AC_PATH_PROG(GNUPLOT, gnuplot) if test -z "$GNUPLOT"; then AC_MSG_WARN([ Gnuplot was not found in your PATH. It is used by the BibRank module to produce graphs about download and citation history. - You can continue without it but you will miss these graphs. We - recommend you to install it first and to rerun the configure script. + You can continue without it but you will miss these graphs. We + recommend you to install it first and to rerun the configure script. Gnuplot is available from .]) fi ## Substitute variables: AC_SUBST(VERSION) AC_SUBST(WML) AC_SUBST(MYSQL) AC_SUBST(PHP) AC_SUBST(PYTHON) AC_SUBST(CLIDIR) AC_SUBST(WEBURL) AC_SUBST(SWEBURL) AC_SUBST(DBHOST) AC_SUBST(DBNAME) AC_SUBST(DBUSER) AC_SUBST(DBPASS) AC_SUBST(PDFTOTEXT) AC_SUBST(PSTOTEXT) AC_SUBST(PSTOASCII) AC_SUBST(ANTIWORD) AC_SUBST(CATDOC) AC_SUBST(WVTEXT) AC_SUBST(PPTHTML) AC_SUBST(XLHTML) AC_SUBST(HTMLTOTEXT) AC_SUBST(localstatedir, `eval echo "${localstatedir}"`) AC_SUBST(CACHEDIR) AC_SUBST(CLISP) AC_SUBST(CMUCL) AC_SUBST(SBCL) AC_SUBST(GNUPLOT) AC_SUBST(DJPEG) AC_SUBST(CONVERT) AC_SUBST(GIFTEXT) AC_SUBST(JPEGSIZE) AC_SUBST(PNMSCALE) AC_SUBST(PPMQUANT) AC_SUBST(PPMTOGIF) AC_SUBST(GIFINTER) AC_SUBST(GIFRSIZE) ## Define output files: AC_CONFIG_FILES([config.nice \ Makefile \ po/Makefile.in \ config/Makefile \ config/configbis.wml \ modules/Makefile \ modules/bibconvert/Makefile \ modules/bibconvert/bin/Makefile \ modules/bibconvert/bin/bibconvert \ modules/bibconvert/doc/Makefile \ modules/bibconvert/doc/admin/Makefile \ modules/bibconvert/doc/hacking/Makefile \ modules/bibconvert/etc/Makefile \ modules/bibconvert/lib/Makefile \ modules/bibmatch/Makefile \ modules/bibmatch/bin/Makefile \ modules/bibmatch/bin/bibmatch \ modules/bibmatch/doc/Makefile \ modules/bibmatch/doc/admin/Makefile \ modules/bibmatch/etc/Makefile \ modules/bibmatch/lib/Makefile \ modules/bibedit/Makefile \ modules/bibedit/bin/Makefile \ modules/bibedit/bin/refextract \ modules/bibedit/bin/xmlmarclint \ modules/bibedit/bin/xmlmarc2textmarc \ modules/bibedit/doc/Makefile \ modules/bibedit/doc/admin/Makefile \ modules/bibedit/doc/hacking/Makefile \ modules/bibedit/etc/Makefile \ modules/bibedit/lib/Makefile \ modules/bibedit/web/Makefile \ modules/bibedit/web/admin/Makefile \ modules/bibformat/Makefile \ modules/bibformat/bin/Makefile \ modules/bibformat/bin/bibformat \ modules/bibformat/bin/bibreformat \ modules/bibformat/doc/Makefile \ modules/bibformat/doc/admin/Makefile \ modules/bibformat/doc/hacking/Makefile \ modules/bibformat/etc/Makefile \ modules/bibformat/etc/output_formats/Makefile \ modules/bibformat/etc/format_templates/Makefile \ modules/bibformat/lib/Makefile \ modules/bibformat/lib/elements/Makefile \ modules/bibformat/lib/core/Makefile \ modules/bibformat/lib/common/Makefile \ modules/bibformat/web/Makefile \ modules/bibformat/web/admin/Makefile \ modules/bibharvest/Makefile \ modules/bibharvest/bin/Makefile \ modules/bibharvest/bin/bibharvest \ modules/bibharvest/bin/oaiharvest \ modules/bibharvest/bin/oaiarchive \ modules/bibharvest/doc/Makefile \ modules/bibharvest/doc/admin/Makefile \ modules/bibharvest/doc/hacking/Makefile \ modules/bibharvest/lib/Makefile \ modules/bibharvest/web/Makefile \ modules/bibharvest/web/admin/Makefile \ modules/bibclassify/Makefile \ modules/bibclassify/bin/Makefile \ modules/bibclassify/bin/bibclassify \ modules/bibclassify/bin/bibclassifyd \ modules/bibclassify/doc/Makefile \ modules/bibclassify/doc/admin/Makefile \ modules/bibclassify/doc/hacking/Makefile \ modules/bibclassify/etc/Makefile modules/bibclassify/lib/Makefile modules/bibindex/Makefile \ modules/bibindex/bin/Makefile \ modules/bibindex/bin/bibindex \ modules/bibindex/bin/bibstat \ modules/bibindex/doc/Makefile \ modules/bibindex/doc/admin/Makefile \ modules/bibindex/doc/hacking/Makefile \ modules/bibindex/lib/Makefile \ modules/bibindex/web/Makefile \ modules/bibindex/web/admin/Makefile \ modules/bibrank/Makefile \ modules/bibrank/bin/Makefile \ modules/bibrank/bin/bibrank \ modules/bibrank/bin/bibrankgkb \ modules/bibrank/doc/Makefile \ modules/bibrank/doc/admin/Makefile \ modules/bibrank/doc/hacking/Makefile \ modules/bibrank/etc/Makefile \ modules/bibrank/etc/bibrankgkb.cfg \ modules/bibrank/etc/demo_jif.cfg \ modules/bibrank/etc/template_single_tag_rank_method.cfg \ modules/bibrank/lib/Makefile \ modules/bibrank/web/Makefile \ modules/bibrank/web/admin/Makefile \ modules/bibsched/Makefile \ modules/bibsched/bin/Makefile \ modules/bibsched/bin/bibsched \ modules/bibsched/bin/bibtaskex \ modules/bibsched/doc/Makefile \ modules/bibsched/doc/admin/Makefile \ modules/bibsched/doc/hacking/Makefile \ modules/bibsched/lib/Makefile \ modules/bibupload/Makefile \ modules/bibupload/bin/Makefile \ modules/bibupload/bin/bibupload \ modules/bibupload/doc/Makefile \ modules/bibupload/doc/admin/Makefile \ modules/bibupload/doc/hacking/Makefile \ modules/bibupload/lib/Makefile \ modules/elmsubmit/Makefile \ modules/elmsubmit/bin/Makefile \ modules/elmsubmit/bin/elmsubmit \ modules/elmsubmit/doc/Makefile \ modules/elmsubmit/doc/admin/Makefile \ modules/elmsubmit/doc/hacking/Makefile \ modules/elmsubmit/etc/Makefile \ modules/elmsubmit/etc/elmsubmit.cfg \ modules/elmsubmit/lib/Makefile \ modules/elmsubmit/lib/magic/Makefile \ modules/miscutil/Makefile \ modules/miscutil/bin/Makefile \ modules/miscutil/bin/dbexec \ modules/miscutil/bin/dbtest \ modules/miscutil/bin/testsuite \ modules/miscutil/bin/regressiontestsuite \ modules/miscutil/lib/Makefile \ modules/miscutil/demo/Makefile \ modules/miscutil/sql/Makefile \ modules/miscutil/web/Makefile \ modules/miscutil/doc/Makefile \ modules/miscutil/doc/hacking/Makefile \ modules/webaccess/Makefile \ modules/webaccess/bin/Makefile \ modules/webaccess/bin/authaction \ modules/webaccess/bin/webaccessadmin \ modules/webaccess/doc/Makefile \ modules/webaccess/doc/admin/Makefile \ modules/webaccess/doc/hacking/Makefile \ modules/webaccess/lib/Makefile \ modules/webaccess/web/Makefile \ modules/webaccess/web/admin/Makefile \ modules/webalert/Makefile \ modules/webalert/bin/Makefile \ modules/webalert/bin/alertengine \ modules/webalert/doc/Makefile \ modules/webalert/doc/admin/Makefile \ modules/webalert/doc/hacking/Makefile \ modules/webalert/lib/Makefile \ modules/webalert/web/Makefile \ modules/webhelp/Makefile \ modules/webhelp/web/Makefile \ modules/webhelp/web/hacking/Makefile \ modules/webhelp/web/admin/Makefile \ modules/webhelp/web/admin/howto/Makefile \ modules/websearch/Makefile \ modules/websearch/bin/Makefile \ modules/websearch/bin/webcoll \ modules/websearch/doc/Makefile \ modules/websearch/doc/admin/Makefile \ modules/websearch/doc/hacking/Makefile \ modules/websearch/lib/Makefile \ modules/websearch/web/Makefile \ modules/websearch/web/admin/Makefile \ modules/websession/Makefile \ modules/websession/bin/Makefile \ - modules/websession/bin/sessiongc \ + modules/websession/bin/inveniogc \ modules/websession/doc/Makefile \ modules/websession/doc/admin/Makefile \ modules/websession/doc/hacking/Makefile \ modules/websession/lib/Makefile \ modules/websession/web/Makefile \ modules/webstat/Makefile \ modules/webstat/bin/Makefile \ modules/webstat/bin/webstat \ modules/webstat/bin/webstatadmin \ modules/webstat/doc/Makefile \ modules/webstat/doc/admin/Makefile \ modules/webstat/doc/hacking/Makefile \ modules/webstat/etc/Makefile \ modules/webstat/lib/Makefile \ modules/webstyle/Makefile \ modules/webstyle/css/Makefile \ modules/webstyle/doc/Makefile \ modules/webstyle/doc/admin/Makefile \ modules/webstyle/doc/hacking/Makefile \ modules/webstyle/img/Makefile \ modules/webstyle/lib/Makefile \ modules/webcomment/Makefile \ modules/webcomment/doc/Makefile \ modules/webcomment/doc/admin/Makefile \ modules/webcomment/doc/hacking/Makefile \ modules/webcomment/lib/Makefile \ modules/webcomment/web/Makefile \ modules/webcomment/web/admin/Makefile \ modules/webbasket/Makefile \ modules/webbasket/doc/Makefile \ modules/webbasket/doc/admin/Makefile \ modules/webbasket/doc/hacking/Makefile \ modules/webbasket/lib/Makefile \ modules/webbasket/web/Makefile \ modules/webmessage/Makefile \ modules/webmessage/bin/Makefile \ modules/webmessage/bin/webmessageadmin \ modules/webmessage/doc/Makefile \ modules/webmessage/doc/admin/Makefile \ modules/webmessage/doc/hacking/Makefile \ modules/webmessage/lib/Makefile \ modules/webmessage/web/Makefile \ modules/websubmit/Makefile \ modules/websubmit/bin/Makefile \ modules/websubmit/bin/thumbmaker \ modules/websubmit/etc/Makefile \ modules/websubmit/etc/bibconvert/Makefile \ modules/websubmit/etc/bibconvert/KB/Makefile \ modules/websubmit/etc/bibconvert/config/Makefile \ modules/websubmit/etc/bibconvert/config/DEMOPICcreate.tpl \ modules/websubmit/etc/bibconvert/config/EDSTEXTcreate.tpl \ modules/websubmit/etc/bibconvert/config/EDSRTEXTcreate.tpl \ modules/websubmit/doc/Makefile \ modules/websubmit/doc/admin/Makefile \ modules/websubmit/doc/hacking/Makefile \ modules/websubmit/lib/Makefile \ modules/websubmit/lib/functions/Makefile \ modules/websubmit/web/Makefile \ modules/websubmit/web/admin/Makefile \ ]) AC_CONFIG_COMMANDS(.wmlrc, [ cat<.wmlrc -Iconfig -I${srcdir}/config EOF ]) ## Finally, write output files: AC_OUTPUT ## Write help: AC_MSG_RESULT([****************************************************************************]) -AC_MSG_RESULT([** Your CDS Invenio installation is now ready for building. **]) -AC_MSG_RESULT([** You have entered the following parameters: **]) +AC_MSG_RESULT([** Your CDS Invenio installation is now ready for building. **]) +AC_MSG_RESULT([** You have entered the following parameters: **]) AC_MSG_RESULT([** - CDS Invenio main install directory: ${prefix}]) AC_MSG_RESULT([** - Web home URL: $WEBURL]) AC_MSG_RESULT([** - Secure web home URL: $SWEBURL]) AC_MSG_RESULT([** - DB server: $DBHOST]) AC_MSG_RESULT([** - DB name: $DBNAME]) AC_MSG_RESULT([** - DB username: $DBUSER]) AC_MSG_RESULT([** - DB password: $DBPASS]) AC_MSG_RESULT([** - Python executable: $PYTHON]) AC_MSG_RESULT([** - CLISP executable: $CLISP]) AC_MSG_RESULT([** - CMUCL executable: $CMUCL]) AC_MSG_RESULT([** - SBCL executable: $SBCL]) AC_MSG_RESULT([** Here are the steps to continue the building process: **]) AC_MSG_RESULT([** 1) Customize self-explanatory config file './config/config.wml'. **]) AC_MSG_RESULT([** 2) Type 'make' to build your customized CDS Invenio installation. **]) AC_MSG_RESULT([** 3) Type 'make create-tables' if you have not created tables before. **]) AC_MSG_RESULT([** 4) Type 'make install' to install the system. **]) AC_MSG_RESULT([** 5) Type 'make create-demo-site' to install and test the demo site. **]) AC_MSG_RESULT([** Good luck, and thanks for choosing CDS Invenio. **]) AC_MSG_RESULT([** -- CDS Development Group **]) AC_MSG_RESULT([****************************************************************************]) ## end of file diff --git a/modules/bibsched/lib/bibsched.py b/modules/bibsched/lib/bibsched.py index 9c1982651..8534f6892 100644 --- a/modules/bibsched/lib/bibsched.py +++ b/modules/bibsched/lib/bibsched.py @@ -1,892 +1,892 @@ # -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """BibSched - task management, scheduling and executing system for CDS Invenio """ __revision__ = "$Id$" ### -- local configuration section starts here --- # which tasks are recognized as valid? cfg_valid_processes = ["bibindex", "bibupload", "bibreformat", "webcoll", "bibtaskex", "bibrank", - "oaiharvest", "oaiarchive", "sessiongc", + "oaiharvest", "oaiarchive", "inveniogc", "webstatadmin", "bibclassifyd"] ### -- local configuration section ends here --- import os import string import sys import time import re import marshal import getopt import curses import curses.panel from curses.wrapper import wrapper import signal from invenio.config import \ CFG_PREFIX, \ CFG_BIBSCHED_REFRESHTIME, \ CFG_BIBSCHED_LOG_PAGER, \ bindir, \ logdir from invenio.dbquery import run_sql def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"): """Returns a date string according to the format string. It can handle normal date strings and shifts with respect to now.""" try: date = time.time() shift_re = re.compile("([-\+]{0,1})([\d]+)([dhms])") factors = {"d":24*3600, "h":3600, "m":60, "s":1} m = shift_re.match(var) if m: sign = m.groups()[0] == "-" and -1 or 1 factor = factors[m.groups()[2]] value = float(m.groups()[1]) date = time.localtime(date + sign * factor * value) date = time.strftime(format_string, date) else: date = time.strptime(var, format_string) date = time.strftime(format_string, date) return date except: return None def get_my_pid(process, args=''): if sys.platform.startswith('freebsd'): COMMAND = "ps -o pid,args | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, args) else: COMMAND = "ps -C %s o '%%p%%a' | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, process, args) answer = string.strip(os.popen(COMMAND).read()) if answer == '': answer = 0 else: answer = answer[:string.find(answer,' ')] return int(answer) def get_task_pid(task_name, task_id): """Return the pid of task_name/task_id""" try: pid = int(open(os.path.join(CFG_PREFIX, 'var', 'run', 'bibsched_task_%d.pid' % task_id)).read()) except IOError: return get_my_pid(task_name, str(task_id)) try: os.kill(pid, signal.SIGCONT) except OSError: return get_my_pid(task_name, str(task_id)) return pid def get_output_channelnames(task_id): "Construct and return filename for stdout and stderr for the task 'task_id'." filenamebase = "%s/bibsched_task_%d" % (logdir, task_id) return [filenamebase + ".log", filenamebase + ".err"] def is_task_scheduled(task_name): """Check if a certain task_name is due for execution (WAITING or RUNNING)""" sql = "SELECT COUNT(proc) FROM schTASK WHERE proc = %s AND (status = 'WAITING' or status = 'RUNNING')" return run_sql(sql, (task_name,))[0][0] > 0 def get_task_ids_by_descending_date(task_name, statuses=['SCHEDULED']): """Returns list of task ids, ordered by descending runtime.""" sql = "SELECT id FROM schTASK WHERE proc=%s AND (" + \ " OR ".join(["status = '%s'" % x for x in statuses]) + ") ORDER BY runtime DESC" return [x[0] for x in run_sql(sql, (task_name,))] def get_task_options(task_id): """Returns options for task_id read from the BibSched task queue table.""" res = run_sql("SELECT arguments FROM schTASK WHERE id=%s", (task_id,)) try: return marshal.loads(res[0][0]) except IndexError: return {} class Manager: def __init__(self): self.helper_modules = cfg_valid_processes self.running = 1 self.footer_move_mode = "[KeyUp/KeyDown Move] [M Select mode] [Q Quit]" self.footer_auto_mode = "[A Manual mode] [1/2 Display Type] [P Purge Done] [Q Quit]" self.footer_select_mode = "[KeyUp/KeyDown/PgUp/PgDown Select] [L View Log] [1/2 Display Type] [M Move mode] [A Auto mode] [Q Quit]" self.footer_waiting_item = "[R Run] [D Delete]" self.footer_running_item = "[S Sleep] [T Stop] [K Kill]" self.footer_stopped_item = "[I Initialise] [D Delete]" self.footer_sleeping_item = "[W Wake Up]" self.item_status = "" self.selected_line = 2 self.rows = [] self.panel = None self.display = 2 self.first_visible_line = 0 self.move_mode = 0 self.auto_mode = 0 self.currentrow = ["", "", "", "", "", "", ""] wrapper(self.start) def handle_keys(self, chr): if chr == -1: return if self.auto_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN, curses.KEY_PPAGE, curses.KEY_NPAGE, ord("q"), ord("Q"), ord("a"), ord("A"), ord("1"), ord("2"), ord("p"), ord("P"))): self.display_in_footer("in automatic mode") self.stdscr.refresh() elif self.move_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN, ord("m"), ord("M"), ord("q"), ord("Q"))): self.display_in_footer("in move mode") self.stdscr.refresh() else: if chr == curses.KEY_UP: if self.move_mode: self.move_up() else: self.selected_line = max(self.selected_line - 1, 2) self.repaint() if chr == curses.KEY_PPAGE: self.selected_line = max(self.selected_line - 10, 2) self.repaint() elif chr == curses.KEY_DOWN: if self.move_mode: self.move_down() else: self.selected_line = min(self.selected_line + 1, len(self.rows) + 1 ) self.repaint() elif chr == curses.KEY_NPAGE: self.selected_line = min(self.selected_line + 10, len(self.rows) + 1 ) self.repaint() elif chr == curses.KEY_HOME: self.first_visible_line = 0 self.selected_line = 2 elif chr in (ord("a"), ord("A")): self.change_auto_mode() elif chr in (ord("l"), ord("L")): self.openlog() elif chr in (ord("w"), ord("W")): self.wakeup() elif chr in (ord("r"), ord("R")): self.run() elif chr in (ord("s"), ord("S")): self.sleep() elif chr in (ord("k"), ord("K")): self.kill() elif chr in (ord("t"), ord("T")): self.stop() elif chr in (ord("d"), ord("D")): self.delete() elif chr in (ord("i"), ord("I")): self.init() elif chr in (ord("m"), ord("M")): self.change_select_mode() elif chr in (ord("p"), ord("P")): self.purge_done() elif chr == ord("1"): self.display = 1 self.first_visible_line = 0 self.selected_line = 2 self.display_in_footer("only done processes are displayed") elif chr == ord("2"): self.display = 2 self.first_visible_line = 0 self.selected_line = 2 self.display_in_footer("only not done processes are displayed") elif chr in (ord("q"), ord("Q")): if curses.panel.top_panel() == self.panel: self.panel.bottom() curses.panel.update_panels() else: self.running = 0 return def set_status(self, task_id, status): return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id)) def set_progress(self, task_id, progress): return run_sql("UPDATE schTASK set progress=%s WHERE id=%s", (progress, task_id)) def openlog(self): task_id = self.currentrow[0] status = self.currentrow[5] if status != 'WAITING': tmpname = os.tmpnam() tmpfile = open(tmpname, "w") try: tmpfile.write(open(os.path.join(logdir, 'bibsched_task_%d.log' % task_id)).read()) except IOError: pass try: tmpfile.write(open(os.path.join(logdir, 'bibsched_task_%d.err' % task_id)).read()) except IOError: pass tmpfile.close() if CFG_BIBSCHED_LOG_PAGER: pager = CFG_BIBSCHED_LOG_PAGER else: pager = os.environ.get('PAGER', '/bin/more') curses.endwin() os.spawnlp(os.P_WAIT, pager, pager, tmpname) os.remove(tmpname) curses.panel.update_panels() def count_processes(self, status): out = 0 res = run_sql("SELECT COUNT(id) FROM schTASK WHERE status=%s GROUP BY status", (status,)) try: out = res[0][0] except: pass return out def wakeup(self): task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: self.display_in_footer("a process is already running!") elif status == "SLEEPING": mypid = get_task_pid(process, task_id) if mypid != 0: os.kill(mypid, signal.SIGCONT) self.display_in_footer("process woken up") else: self.display_in_footer("process is not sleeping") self.stdscr.refresh() def _display_YN_box(self, msg): msg += ' (Y/N)' rows = msg.split('\n') height = len(rows) + 2 width = max([len(row) for row in rows]) + 4 self.win = curses.newwin( height, width, (self.height - height) / 2 + 1, (self.width - width) / 2 + 1 ) self.panel = curses.panel.new_panel( self.win ) self.panel.top() self.win.border() i = 1 for row in rows: self.win.addstr(i, 2, row) i += 1 self.win.refresh() while 1: c = self.win.getch() if c in (ord('y'), ord('Y')): return True elif c in (ord('n'), ord('N')): return False def purge_done(self): if self._display_YN_box("You are going to purge all the list of DONE tasks.\n" "This will definitely alter your task history.\nAre you sure?"): run_sql("DELETE FROM schTASK WHERE status='DONE'") curses.panel.update_panels() self.display_in_footer("DONE processes purged") else: curses.panel.update_panels() def run(self): task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] sleeptime = self.currentrow[4] if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: self.display_in_footer("a process is already running!") elif status == "STOPPED" or status == "WAITING": if process in self.helper_modules: program = os.path.join(bindir, process) fdout, fderr = get_output_channelnames(task_id) COMMAND = "%s %s >> %s 2>> %s &" % (program, str(task_id), fdout, fderr) os.system(COMMAND) Log("manually running task #%d (%s)" % (task_id, process)) if sleeptime: new_runtime = get_datetime(sleeptime) new_task_arguments = marshal.loads(self.currentrow[7]) new_task_arguments["runtime"] = new_runtime new_task_id = run_sql("INSERT INTO schTASK (proc,user,runtime,sleeptime,arguments,status)"\ " VALUES (%s,%s,%s,%s,%s,'WAITING')", (process, self.currentrow[2], new_runtime, sleeptime, self.currentrow[7])) new_task_arguments["task"] = new_task_id run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""", (marshal.dumps(new_task_arguments), new_task_id)) else: self.display_in_footer("process status should be STOPPED or WAITING!") self.stdscr.refresh() def sleep(self): task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] if status != 'RUNNING' and status != 'CONTINUING': self.display_in_footer("this process is not running!") else: mypid = get_task_pid(process, task_id) if mypid != 0: os.kill(mypid, signal.SIGUSR1) self.display_in_footer("USR1 signal sent to process #%s" % mypid) else: self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def kill(self): task_id = self.currentrow[0] process = self.currentrow[1] #status = self.currentrow[5] mypid = get_task_pid(process, task_id) if mypid != 0: os.kill(mypid, signal.SIGKILL) self.set_status(task_id, 'STOPPED') self.display_in_footer("KILL signal sent to process #%s" % mypid) else: self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def stop(self): task_id = self.currentrow[0] process = self.currentrow[1] #status = self.currentrow[5] mypid = get_task_pid(process, task_id) if mypid != 0: os.kill(mypid, signal.SIGTERM) self.display_in_footer("TERM signal sent to process #%s" % mypid) else: self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def delete(self): task_id = self.currentrow[0] #process = self.currentrow[1] status = self.currentrow[5] if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING': self.set_status(task_id, "%s_DELETED" % status) self.display_in_footer("process deleted") self.selected_line = max(self.selected_line, 2) else: self.display_in_footer("cannot delete running processes") self.stdscr.refresh() def init(self): task_id = self.currentrow[0] #process = self.currentrow[1] status = self.currentrow[5] if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING': self.set_status(task_id, "WAITING") self.set_progress(task_id, "None") self.display_in_footer("process initialised") else: self.display_in_footer("cannot initialise running processes") self.stdscr.refresh() def change_select_mode(self): if self.move_mode: self.move_mode = 0 else: status = self.currentrow[5] if status in ( "RUNNING" , "CONTINUING" , "SLEEPING" ): self.display_in_footer("cannot move running processes!") else: self.move_mode = 1 self.stdscr.refresh() def change_auto_mode(self): if self.auto_mode: program = os.path.join(bindir, "bibsched") COMMAND = "%s -q stop" % program os.system(COMMAND) self.auto_mode = 0 else: program = os.path.join( bindir, "bibsched") COMMAND = "%s -q start" % program os.system(COMMAND) self.auto_mode = 1 self.move_mode = 0 self.stdscr.refresh() def move_up(self): self.display_in_footer("not implemented yet") self.stdscr.refresh() def move_down(self): self.display_in_footer("not implemented yet") self.stdscr.refresh() def put_line(self, row): col_w = [ 5 , 11 , 21 , 21 , 7 , 11 , 25 ] maxx = self.width if self.y == self.selected_line - self.first_visible_line and self.y > 1: if self.auto_mode: attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD elif self.move_mode: attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD else: attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD self.item_status = row[5] self.currentrow = row elif self.y == 0: if self.auto_mode: attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD elif self.move_mode: attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD else: attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD elif row[5] == "DONE": attr = curses.color_pair(5) + curses.A_BOLD elif row[5] == "STOPPED": attr = curses.color_pair(6) + curses.A_BOLD elif row[5].find("ERROR") > -1: attr = curses.color_pair(4) + curses.A_BOLD elif row[5] == "WAITING": attr = curses.color_pair(3) + curses.A_BOLD elif row[5] in ("RUNNING","CONTINUING") : attr = curses.color_pair(2) + curses.A_BOLD else: attr = curses.A_BOLD myline = str(row[0]).ljust(col_w[0]) myline += str(row[1]).ljust(col_w[1]) myline += str(row[2]).ljust(col_w[2]) myline += str(row[3])[:19].ljust(col_w[3]) myline += str(row[4]).ljust(col_w[4]) myline += str(row[5]).ljust(col_w[5]) myline += str(row[6]).ljust(col_w[6]) myline = myline.ljust(maxx) self.stdscr.addnstr(self.y, 0, myline, maxx, attr) self.y = self.y+1 def display_in_footer(self, footer, i = 0, print_time_p=0): if print_time_p: footer = "%s %s" % (footer, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) maxx = self.stdscr.getmaxyx()[1] footer = footer.ljust(maxx) if self.auto_mode: colorpair = 2 elif self.move_mode: colorpair = 7 else: colorpair = 1 self.stdscr.addnstr(self.y - i, 0, footer, maxx - 1, curses.A_STANDOUT + curses.color_pair(colorpair) + curses.A_BOLD ) def repaint(self): self.y = 0 self.stdscr.clear() self.height, self.width = self.stdscr.getmaxyx() maxy = self.height - 2 #maxx = self.width self.put_line( ("ID", "PROC", "USER", "RUNTIME", "SLEEP", "STATUS", "PROGRESS") ) self.put_line( ("---", "----", "----", "-------------------", "-----", "-----", "--------") ) if self.selected_line > maxy + self.first_visible_line - 1: self.first_visible_line = self.selected_line - maxy + 1 if self.selected_line < self.first_visible_line + 2: self.first_visible_line = self.selected_line - 2 for row in self.rows[self.first_visible_line:self.first_visible_line+maxy-2]: task_id, proc, user, runtime, sleeptime, status, progress, arguments = row self.put_line( row ) self.y = self.stdscr.getmaxyx()[0] - 1 if self.auto_mode: self.display_in_footer(self.footer_auto_mode, print_time_p=1) elif self.move_mode: self.display_in_footer(self.footer_move_mode, print_time_p=1) else: self.display_in_footer(self.footer_select_mode, print_time_p=1) footer2 = "" if self.item_status.find("DONE") > -1 or self.item_status == "ERROR" or self.item_status == "STOPPED": footer2 += self.footer_stopped_item elif self.item_status == "RUNNING" or self.item_status == "CONTINUING": footer2 += self.footer_running_item elif self.item_status == "SLEEPING": footer2 += self.footer_sleeping_item elif self.item_status == "WAITING": footer2 += self.footer_waiting_item self.display_in_footer(footer2, 1) self.stdscr.refresh() def start(self, stdscr): ring = 0 if curses.has_colors(): curses.start_color() curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK) curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_RED) curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK) curses.init_pair(3, curses.COLOR_MAGENTA, curses.COLOR_BLACK) curses.init_pair(4, curses.COLOR_RED, curses.COLOR_BLACK) curses.init_pair(5, curses.COLOR_BLUE, curses.COLOR_BLACK) curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK) curses.init_pair(7, curses.COLOR_YELLOW, curses.COLOR_BLACK) self.stdscr = stdscr self.base_panel = curses.panel.new_panel( self.stdscr ) self.base_panel.bottom() curses.panel.update_panels() self.height, self.width = stdscr.getmaxyx() self.stdscr.clear() if server_pid (): self.auto_mode = 1 if self.display == 1: where = "and status='DONE'" order = "DESC" else: where = "and status!='DONE'" order = "ASC" self.rows = run_sql("""SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s""" % (where, order)) self.repaint() ring = 0 while self.running: ring += 1 char = -1 try: char = timed_out(self.stdscr.getch, 1) if char == 27: # escaping sequence char = self.stdscr.getch() if char == 79: # arrow char = self.stdscr.getch() if char == 65: #arrow up char = curses.KEY_UP elif char == 66: #arrow down char = curses.KEY_DOWN elif char == 72: char = curses.KEY_PPAGE elif char == 70: char = curses.KEY_NPAGE elif char == 91: char = self.stdscr.getch() if char == 53: char = self.stdscr.getch() if char == 126: char = curses.KEY_HOME except TimedOutExc: char = -1 self.handle_keys(char) if ring == 4: if self.display == 1: where = "and status='DONE'" order = "DESC" else: where = "and status!='DONE'" order = "ASC" self.rows = run_sql("""SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s""" % (where, order)) ring = 0 self.repaint() class BibSched: def __init__(self): self.helper_modules = cfg_valid_processes self.running = {} self.sleep_done = {} self.sleep_sent = {} self.stop_sent = {} self.suicide_sent = {} def set_status(self, task_id, status): return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id)) def can_run( self, proc ): return len( self.running.keys() ) == 0 def get_running_processes(self): row = None res = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK "\ " WHERE status='RUNNING' or status='CONTINUING' LIMIT 1") try: row = res[0] except: pass return row def handle_row( self, row ): task_id, proc, user, runtime, sleeptime, arguments, status = row if status == "SLEEP": if task_id in self.running.keys(): self.set_status( task_id, "SLEEP SENT" ) os.kill( self.running[task_id], signal.SIGUSR1 ) self.sleep_sent[task_id] = self.running[task_id] elif status == "SLEEPING": if task_id in self.sleep_sent.keys(): self.sleep_done[task_id] = self.sleep_sent[task_id] del self.sleep_sent[task_id] if status == "WAKEUP": if task_id in self.sleep_done.keys(): self.running[task_id] = self.sleep_done[task_id] del self.sleep_done[task_id] os.kill( self.running[task_id], signal.SIGCONT ) self.set_status( task_id, "RUNNING" ) elif status == "STOP": if task_id in self.running.keys(): self.set_status( task_id, "STOP SENT" ) os.kill( self.running[task_id], signal.SIGTERM ) self.stop_sent[task_id] = self.running[task_id] del self.running[task_id] elif status == "STOPPED" and task_id in self.stop_sent.keys(): del self.stop_sent[task_id] elif status == "SUICIDE": if task_id in self.running.keys(): self.set_status( task_id, "SUICIDE SENT" ) os.kill( self.running[task_id], signal.SIGABRT ) self.suicide_sent[task_id] = self.running[task_id] del self.running[task_id] elif status == "SUICIDED" and task_id in self.suicide_sent.keys(): del self.suicide_sent[task_id] elif status.find("DONE") > -1 and task_id in self.running.keys(): del self.running[task_id] elif self.can_run(proc) and status == "WAITING" and runtime <= time.time(): if proc in self.helper_modules: program = os.path.join(bindir, proc) fdout, fderr = get_output_channelnames(task_id) COMMAND = "%s %s >> %s 2>> %s" % (program, str(task_id), fdout, fderr) Log("task #%d (%s) started" % (task_id, proc)) os.system(COMMAND) Log("task #%d (%s) ended" % (task_id, proc)) self.running[task_id] = get_task_pid(proc, task_id) if sleeptime: new_runtime = get_datetime(sleeptime) new_task_arguments = marshal.loads(arguments) new_task_arguments["runtime"] = new_runtime new_task_id = run_sql("INSERT INTO schTASK (proc,user,runtime,sleeptime,arguments,status)"\ " VALUES (%s,%s,%s,%s,%s,'WAITING')", (proc, user, new_runtime, sleeptime, arguments)) new_task_arguments["task"] = new_task_id run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""", (marshal.dumps(new_task_arguments), new_task_id)) def watch_loop(self): running_process = self.get_running_processes() if running_process: proc = running_process[ 1 ] task_id = running_process[ 0 ] if get_task_pid(proc, task_id): self.running[task_id] = get_task_pid(proc, task_id) else: self.set_status(task_id,"ERROR") rows = [] while 1: for row in rows: self.handle_row( row ) time.sleep(CFG_BIBSCHED_REFRESHTIME) rows = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK ORDER BY runtime ASC") class TimedOutExc(Exception): def __init__(self, value = "Timed Out"): self.value = value def __str__(self): return repr(self.value) def timed_out(f, timeout, *args, **kwargs): def handler(signum, frame): raise TimedOutExc() old = signal.signal(signal.SIGALRM, handler) signal.alarm(timeout) try: result = f(*args, **kwargs) finally: signal.signal(signal.SIGALRM, old) signal.alarm(0) return result def Log(message): log = open(logdir + "/bibsched.log","a") log.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime())) log.write(message) log.write("\n") log.close() def redirect_stdout_and_stderr(): "This function redirects stdout and stderr to bibsched.log and bibsched.err file." sys.stdout = open(logdir + "/bibsched.log", "a") sys.stderr = open(logdir + "/bibsched.err", "a") def usage(exitcode=1, msg=""): """Prints usage info.""" if msg: sys.stderr.write("Error: %s.\n" % msg) sys.stderr.write ("""\ Usage: %s [options] [start|stop|restart|monitor] The following commands are available for bibsched: - start: start bibsched in background - stop: stop a running bibsched - restart: restart a running bibsched - monitor: enter the interactive monitor Command options: -d, --daemon\t Launch BibSched in the daemon mode (deprecated, use 'start') General options: -h, --help \t\t Print this help. -V, --version \t\t Print version information. """ % sys.argv [0]) #sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n") sys.exit(exitcode) pidfile = os.path.join(CFG_PREFIX, 'var', 'run', 'bibsched.pid') def error (msg): print >> sys.stderr, "error: " + msg sys.exit (1) def server_pid (): # The pid must be stored on the filesystem try: pid = int (open (pidfile).read ()) except IOError: return None # Even if the pid is available, we check if it corresponds to an # actual process, as it might have been killed externally try: os.kill (pid, signal.SIGCONT) except OSError: return None return pid def start (verbose = True): """ Fork this process in the background and start processing requests. The process PID is stored in a pid file, so that it can be stopped later on.""" if verbose: sys.stdout.write ("starting bibsched: ") sys.stdout.flush () pid = server_pid () if pid: error ("another instance of bibsched (pid %d) is running" % pid) # start the child process using the "double fork" technique pid = os.fork () if pid > 0: sys.exit (0) os.setsid () os.chdir ('/') pid = os.fork () if pid > 0: if verbose: sys.stdout.write ('pid %d\n' % pid) Log ("daemon started (pid %d)" % pid) open (pidfile, 'w').write ('%d' % pid) return sys.stdin.close () redirect_stdout_and_stderr () sched = BibSched() sched.watch_loop () return def stop (verbose = True): pid = server_pid () if not pid: error ('bibsched seems not to be running.') try: os.kill (pid, signal.SIGKILL) except OSError: print >> sys.stderr, 'no bibsched process found' Log ("daemon stopped (pid %d)" % pid) if verbose: print "stopping bibsched: pid %d" % pid os.unlink (pidfile) return def monitor(verbose = True): redirect_stdout_and_stderr() manager = Manager() return def restart(verbose = True): stop(verbose) start(verbose) return def main(): verbose = True try: opts, args = getopt.getopt(sys.argv[1:], "hVdq", [ "help","version","daemon", "quiet"]) except getopt.GetoptError, err: Log ("Error: %s" % err) usage(1, err) for opt, arg in opts: if opt in ["-h", "--help"]: usage (0) elif opt in ["-V", "--version"]: print __revision__ sys.exit(0) elif opt in ["-d", "--daemon"]: redirect_stdout_and_stderr () sched = BibSched() Log("daemon started") sched.watch_loop() elif opt in ['-q', '--quiet']: verbose = False else: usage(1) try: cmd = args [0] except IndexError: cmd = 'monitor' try: { 'start': start, 'stop': stop, 'restart': restart, 'monitor': monitor } [cmd] (verbose) except KeyError: usage (1, 'unkown command: %s' % `cmd`) return if __name__ == '__main__': main() diff --git a/modules/websession/bin/Makefile.am b/modules/websession/bin/Makefile.am index 2a3f6b8b9..68e9fed80 100644 --- a/modules/websession/bin/Makefile.am +++ b/modules/websession/bin/Makefile.am @@ -1,24 +1,24 @@ ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## General Public License for more details. +## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -bin_SCRIPTS = sessiongc +bin_SCRIPTS = inveniogc -EXTRA_DIST = sessiongc.in +EXTRA_DIST = inveniogc.in CLEANFILES = *~ *.tmp \ No newline at end of file diff --git a/modules/websession/bin/sessiongc.in b/modules/websession/bin/inveniogc.in similarity index 96% rename from modules/websession/bin/sessiongc.in rename to modules/websession/bin/inveniogc.in index 589a9a643..47ee91fda 100644 --- a/modules/websession/bin/sessiongc.in +++ b/modules/websession/bin/inveniogc.in @@ -1,36 +1,36 @@ #!@PYTHON@ ## -*- mode: python; coding: utf-8; -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Guest user sessions garbage collector. To be run via cron once per day. (say) """ __revision__ = "$Id$" try: - from invenio.sessiongc import main + from invenio.inveniogc import main except ImportError, e: print "Error: %s" % e import sys sys.exit(1) main() diff --git a/modules/websession/lib/Makefile.am b/modules/websession/lib/Makefile.am index 1b267f6f1..14fdec003 100644 --- a/modules/websession/lib/Makefile.am +++ b/modules/websession/lib/Makefile.am @@ -1,33 +1,33 @@ ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. pylibdir = $(libdir)/python/invenio pylib_DATA = websession.py session.py webuser.py webuser_tests.py \ websession_templates.py websession_webinterface.py \ webgroup.py webgroup_dblayer.py websession_config.py \ webaccount.py websession_regression_tests.py \ webgroup_regression_tests.py webuser_regression_tests.py \ - webgroup_tests.py sessiongc.py + webgroup_tests.py inveniogc.py noinst_DATA = password_migration_kit.py EXTRA_DIST = $(pylib_DATA) $(noinst_DATA) CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/websession/lib/sessiongc.py b/modules/websession/lib/inveniogc.py similarity index 65% rename from modules/websession/lib/sessiongc.py rename to modules/websession/lib/inveniogc.py index 79e129ac5..c0b6ea4ec 100644 --- a/modules/websession/lib/sessiongc.py +++ b/modules/websession/lib/inveniogc.py @@ -1,506 +1,387 @@ ## -*- mode: python; coding: utf-8; -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Guest user sessions garbage collector. """ __revision__ = "$Id$" import sys try: from invenio.dbquery import run_sql from invenio.config import logdir, tmpdir from invenio.websession_config import CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS from invenio.bibtask import task_init, task_set_option, task_get_option, \ write_message, write_messages from invenio.access_control_mailcookie import mail_cookie_gc + from invenio.file import BibDoc import time import os except ImportError, e: print "Error: %s" % (e, ) sys.exit(1) # configure variables CFG_MYSQL_ARGUMENTLIST_SIZE = 100 # After how many days to remove obsolete log/err files CFG_MAX_ATIME_RM_LOG = 28 # After how many days to zip obsolete log/err files CFG_MAX_ATIME_ZIP_LOG = 7 # After how many days to remove obsolete bibreformat fmt xml files CFG_MAX_ATIME_RM_FMT = 28 # After how many days to zip obsolete bibreformat fmt xml files CFG_MAX_ATIME_ZIP_FMT = 7 # After how many days to remove obsolete bibharvest fmt xml files CFG_MAX_ATIME_RM_OAI = 28 # After how many days to zip obsolete bibharvest fmt xml files CFG_MAX_ATIME_ZIP_OAI = 7 +# After how many days to remove deleted bibdocs +CFG_DELETED_BIBDOC_MAXLIFE = 365*10 def gc_exec_command(command): """ Exec the command logging in appropriate way its output.""" write_message(' %s' % command, verbose=9) (dontcare, output, errors) = os.popen3(command) write_messages(errors.read()) write_messages(output.read()) -def clean_filesystem(): - """ Clean the filesystem from obsolete files. """ - write_message("""FILESYSTEM CLEANING STARTED""") +def clean_logs(): + """ Clean the logs from obsolete files. """ + write_message("""LOGS CLEANING STARTED""") write_message("- deleting/gzipping bibsched empty/old err/log " "BibSched files") vstr = task_get_option('verbose') > 1 and '-v' or '' gc_exec_command('find %s -name "bibsched_task_*"' ' -size 0c -exec rm %s -f {} \;' \ % (logdir, vstr)) gc_exec_command('find %s -name "bibsched_task_*"' ' -atime +%s -exec rm %s -f {} \;' \ % (logdir, CFG_MAX_ATIME_RM_LOG, vstr)) gc_exec_command('find %s -name "bibsched_task_*"' ' -atime +%s -exec gzip %s -9 {} \;' \ % (logdir, CFG_MAX_ATIME_ZIP_LOG, vstr)) write_message("- deleting/gzipping temporary empty/old " "BibReformat xml files") gc_exec_command('find %s -name "rec_fmt_*"' ' -size 0c -exec rm %s -f {} \;' \ % (tmpdir, vstr)) gc_exec_command('find %s -name "rec_fmt_*"' ' -atime +%s -exec rm %s -f {} \;' \ % (tmpdir, CFG_MAX_ATIME_RM_FMT, vstr)) gc_exec_command('find %s -name "rec_fmt_*"' ' -atime +%s -exec gzip %s -9 {} \;' \ % (tmpdir, CFG_MAX_ATIME_ZIP_FMT, vstr)) write_message("- deleting/gzipping temporary old " "BibHarvest xml files") gc_exec_command('find %s -name "bibharvestadmin.*"' ' -exec rm %s -f {} \;' \ % (tmpdir, vstr)) gc_exec_command('find %s -name "bibconvertrun.*"' ' -exec rm %s -f {} \;' \ % (tmpdir, vstr)) gc_exec_command('find %s -name "oaiharvest*"' ' -atime +%s -exec gzip %s -9 {} \;' \ % (tmpdir, CFG_MAX_ATIME_ZIP_OAI, vstr)) gc_exec_command('find %s -name "oaiharvest*"' ' -atime +%s -exec rm %s -f {} \;' \ % (tmpdir, CFG_MAX_ATIME_RM_OAI, vstr)) gc_exec_command('find %s -name "oai_archive*"' ' -atime +%s -exec rm %s -f {} \;' \ % (tmpdir, CFG_MAX_ATIME_RM_OAI, vstr)) - write_message("""FILESYSTEM CLEANING FINISHED""") - + write_message("""LOGS CLEANING FINISHED""") + +def clean_documents(): + """Delete all the bibdocs that have been set as deleted and have not been + modified since CFG_DELETED_BIBDOC_MAXLIFE days. Returns the number of + bibdocs involved.""" + write_message("""OBSOLETED DELETED DOCUMENTS CLEANING STARTED""") + write_message("select id from bibdoc where status='DELETED' and NOW()>ADDTIME(modification_date, '%s 0:0:0')" % CFG_DELETED_BIBDOC_MAXLIFE, verbose=9) + records = run_sql("select id from bibdoc where status='DELETED' and NOW()>ADDTIME(modification_date, '%s 0:0:0')" % CFG_DELETED_BIBDOC_MAXLIFE) + for record in records: + bibdoc = BibDoc(record[0]) + bibdoc.expunge() + write_message("DELETE FROM bibdoc WHERE id=%i" % int(record[0]), verbose=9) + run_sql("DELETE FROM bibdoc WHERE id=%s", (record[0], )) + write_message("""%s Obsoleted deleted documents cleaned""" % len(records)) + write_message("""OBSOLETED DELETED DOCUMENTS CLEANING FINISHED""") + return len(records) def guest_user_garbage_collector(): """Session Garbage Collector program flow/tasks: 1: delete expired sessions 1b:delete guest users without session 2: delete queries not attached to any user 3: delete baskets not attached to any user 4: delete alerts not attached to any user 5: delete expired mailcookies 5b: delete expired not confirmed email address 6: delete expired roles memberships verbose - level of program output. 0 - nothing 1 - default 9 - max, debug""" # dictionary used to keep track of number of deleted entries delcount = {'session': 0, 'user': 0, 'user_query': 0, 'query': 0, 'bskBASKET': 0, 'user_bskBASKET': 0, 'bskREC': 0, 'bskRECORDCOMMENT': 0, 'bskEXTREC': 0, 'bskEXTFMT': 0, 'user_query_basket': 0, 'mail_cookie': 0, 'email_addresses': 0, 'role_membership' : 0} write_message("GUEST USER SESSIONS GARBAGE" " COLLECTOR STARTED") # 1 - DELETE EXPIRED SESSIONS write_message("- deleting expired sessions") timelimit = time.time() write_message(" DELETE FROM session WHERE" " session_expiry < %d \n" % (timelimit, ), verbose=9) delcount['session'] += run_sql("DELETE FROM session WHERE" " session_expiry < %s """ % (timelimit, )) # 1b - DELETE GUEST USERS WITHOUT SESSION write_message("- deleting guest users without session") # get uids write_message(""" SELECT u.id\n FROM user AS u LEFT JOIN session AS s\n ON u.id = s.uid\n WHERE s.uid IS NULL AND u.email = ''""", verbose=9) result = run_sql("""SELECT u.id FROM user AS u LEFT JOIN session AS s ON u.id = s.uid WHERE s.uid IS NULL AND u.email = ''""") write_message(result, verbose=9) if result: # work on slices of result list in case of big result for i in range(0, len(result), CFG_MYSQL_ARGUMENTLIST_SIZE): # create string of uids uidstr = '' for (id_user, ) in result[i:i+CFG_MYSQL_ARGUMENTLIST_SIZE]: if uidstr: uidstr += ',' uidstr += "%s" % (id_user, ) # delete users write_message(" DELETE FROM user WHERE" " id IN (TRAVERSE LAST RESULT) AND email = '' \n", verbose=9) delcount['user'] += run_sql("DELETE FROM user WHERE" " id IN (%s) AND email = ''" % (uidstr, )) # 2 - DELETE QUERIES NOT ATTACHED TO ANY USER # first step, delete from user_query write_message("- deleting user_queries referencing" " non-existent users") # find user_queries referencing non-existent users write_message(" SELECT DISTINCT uq.id_user\n" " FROM user_query AS uq LEFT JOIN user AS u\n" " ON uq.id_user = u.id\n WHERE u.id IS NULL", verbose=9) result = run_sql("""SELECT DISTINCT uq.id_user FROM user_query AS uq LEFT JOIN user AS u ON uq.id_user = u.id WHERE u.id IS NULL""") write_message(result, verbose=9) # delete in user_query one by one write_message(" DELETE FROM user_query WHERE" " id_user = 'TRAVERSE LAST RESULT' \n", verbose=9) for (id_user, ) in result: delcount['user_query'] += run_sql("""DELETE FROM user_query WHERE id_user = %s""" % (id_user, )) # delete the actual queries write_message("- deleting queries not attached to any user") # select queries that must be deleted write_message(""" SELECT DISTINCT q.id\n FROM query AS q LEFT JOIN user_query AS uq\n ON uq.id_query = q.id\n WHERE uq.id_query IS NULL AND\n q.type <> 'p' """, verbose=9) result = run_sql("""SELECT DISTINCT q.id FROM query AS q LEFT JOIN user_query AS uq ON uq.id_query = q.id WHERE uq.id_query IS NULL AND q.type <> 'p'""") write_message(result, verbose=9) # delete queries one by one write_message(""" DELETE FROM query WHERE id = 'TRAVERSE LAST RESULT \n""", verbose=9) for (id_user, ) in result: delcount['query'] += run_sql("""DELETE FROM query WHERE id = %s""", (id_user, )) # 3 - DELETE BASKETS NOT OWNED BY ANY USER write_message("- deleting baskets not owned by any user") # select basket ids write_message(""" SELECT ub.id_bskBASKET\n FROM user_bskBASKET AS ub LEFT JOIN user AS u\n ON u.id = ub.id_user\n WHERE u.id IS NULL""", verbose=9) try: result = run_sql("""SELECT ub.id_bskBASKET FROM user_bskBASKET AS ub LEFT JOIN user AS u ON u.id = ub.id_user WHERE u.id IS NULL""") except: result = [] write_message(result, verbose=9) # delete from user_basket and basket one by one write_message(""" DELETE FROM user_bskBASKET WHERE id_bskBASKET = 'TRAVERSE LAST RESULT' """, verbose=9) write_message(""" DELETE FROM bskBASKET WHERE id = 'TRAVERSE LAST RESULT' """, verbose=9) write_message(""" DELETE FROM bskREC WHERE id_bskBASKET = 'TRAVERSE LAST RESULT'""", verbose=9) write_message(""" DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET = 'TRAVERSE LAST RESULT' \n""", verbose=9) for (id_basket, ) in result: delcount['user_bskBASKET'] += run_sql("""DELETE FROM user_bskBASKET WHERE id_bskBASKET = %s""", (id_basket, )) delcount['bskBASKET'] += run_sql("""DELETE FROM bskBASKET WHERE id = %s""", (id_basket, )) delcount['bskREC'] += run_sql("""DELETE FROM bskREC WHERE id_bskBASKET = %s""", (id_basket, )) delcount['bskRECORDCOMMENT'] += run_sql("""DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET = %s""", (id_basket, )) write_message(""" SELECT DISTINCT ext.id, rec.id_bibrec_or_bskEXTREC FROM bskEXTREC AS ext \nLEFT JOIN bskREC AS rec ON ext.id=-rec.id_bibrec_or_bskEXTREC WHERE id_bibrec_or_bskEXTREC is NULL""", verbose=9) try: result = run_sql("""SELECT DISTINCT ext.id FROM bskEXTREC AS ext LEFT JOIN bskREC AS rec ON ext.id=-rec.id_bibrec_or_bskEXTREC WHERE id_bibrec_or_bskEXTREC is NULL""") except: result = [] write_message(result, verbose=9) write_message(""" DELETE FROM bskEXTREC WHERE id = 'TRAVERSE LAST RESULT' """, verbose=9) write_message(""" DELETE FROM bskEXTFMT WHERE id_bskEXTREC = 'TRAVERSE LAST RESULT' \n""", verbose=9) for (id_basket, ) in result: delcount['bskEXTREC'] += run_sql("""DELETE FROM bskEXTREC WHERE id=%s""", (id_basket,)) delcount['bskEXTFMT'] += run_sql("""DELETE FROM bskEXTFMT WHERE id_bskEXTREC=%s""", (id_basket,)) # 4 - DELETE ALERTS NOT OWNED BY ANY USER write_message('- deleting alerts not owned by any user') # select user ids in uqb that reference non-existent users write_message("""SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL""", verbose=9) result = run_sql("""SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL""") write_message(result, verbose=9) # delete all these entries for (id_user, ) in result: write_message("""DELETE FROM user_query_basket WHERE id_user = 'TRAVERSE LAST RESULT """, verbose=9) delcount['user_query_basket'] += run_sql("""DELETE FROM user_query_basket WHERE id_user = %s """, (id_user, )) # 5 - delete expired mailcookies write_message("""mail_cookie_gc()""", verbose=9) delcount['mail_cookie'] = mail_cookie_gc() - # 5b - delete expired not confirmed email address + ## 5b - delete expired not confirmed email address write_message("""DELETE FROM user WHERE note='2' AND NOW()>ADDTIME(last_login, '%s 0:0:0')""" % CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS, verbose=9) delcount['email_addresses'] = run_sql("""DELETE FROM user WHERE note='2' AND NOW()>ADDTIME(last_login, '%s 0:0:0')""" % CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS) # 6 - delete expired roles memberships write_message("""DELETE FROM user_accROLE WHERE expiration 0: types = listTypesFromArray(bibdocs) fulltypes = [] for mytype in types: fulltype = { 'name' : mytype, 'content' : [], } for bibdoc in bibdocs: if mytype == bibdoc.getType(): fulltype['content'].append(bibdoc.display(version, ln = ln)) fulltypes.append(fulltype) t = websubmit_templates.tmpl_bibrecdoc_filelist( ln = ln, types = fulltypes, ) return t class BibDoc: """this class represents one file attached to a record there is a one to one mapping between an instance of this class and an entry in the bibdoc db table""" def __init__ (self, bibdocid="", recid="", docname="file", type="Main"): # bibdocid is known, the document already exists if bibdocid != "": if recid == "": recid = None self.type = "" res = run_sql("select id_bibrec,type from bibrec_bibdoc " "where id_bibdoc=%s", (bibdocid,)) if len(res) > 0: recid = res[0][0] self.type = res[0][1] else: res = run_sql("select id_bibdoc1 from bibdoc_bibdoc " "where id_bibdoc2=%s", (bibdocid,)) if len(res) > 0 : main_bibdoc = res[0][0] res = run_sql("select id_bibrec,type from bibrec_bibdoc " "where id_bibdoc=%s", (main_bibdoc,)) if len(res) > 0: recid = res[0][0] self.type = res[0][1] else: res = run_sql("select type from bibrec_bibdoc " "where id_bibrec=%s and id_bibdoc=%s", (recid, bibdocid,)) if len(res) > 0: self.type = res[0][0] else: #this bibdoc isn't associated with the corresponding bibrec. raise StandardError, "No docid associated with the recid %s" % recid # gather the other information res = run_sql("select id,status,docname,creation_date," "modification_date from bibdoc where id=%s", (bibdocid,)) if len(res) > 0: self.cd = res[0][3] self.md = res[0][4] self.recid = recid self.docname = res[0][2] self.id = bibdocid self.status = res[0][1] group = "g" + str(int(int(self.id) / filedirsize)) self.basedir = "%s/%s/%s" % (filedir, group, self.id) else: # this bibdoc doesn't exist raise StandardError, "The docid %s does not exist." % bibdocid # else it is a new document else: if docname == "" or type == "": raise StandardError, "Argument missing for creating a new bibdoc" else: self.recid = recid self.type = type self.docname = docname self.status = '' res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (recid, docname)) if res: raise StandardError, "A bibdoc called %s already exists for recid %s" % (docname, recid) self.id = run_sql("insert into bibdoc " "(status,docname,creation_date,modification_date) " "values(%s,%s,NOW(),NOW())", (self.status, docname,)) if self.id is not None: # we link the document to the record if a recid was # specified if self.recid != "": run_sql("insert into bibrec_bibdoc values(%s,%s,%s)", (recid, self.id, self.type,)) else: raise StandardError, "New docid cannot be created" group = "g" + str(int(int(self.id) / filedirsize)) self.basedir = "%s/%s/%s" % (filedir, group, self.id) # we create the corresponding storage directory if not os.path.exists(self.basedir): old_umask = os.umask(022) os.makedirs(self.basedir) # and save the father record id if it exists if self.recid != "": fp = open("%s/.recid" % self.basedir, "w") fp.write(str(self.recid)) fp.close() if self.type != "": fp = open("%s/.type" % self.basedir, "w") fp.write(str(self.type)) fp.close() os.umask(old_umask) # build list of attached files self.BuildFileList() # link with relatedFiles self.BuildRelatedFileList() def getStatus(self): """Retrieve the status.""" return self.status + def touch(self): + """Update the modification time of the bibdoc.""" + run_sql('UPDATE bibdoc SET modification_date=NOW() WHERE id=%s', self.id) + def setStatus(self, new_status): """Set a new status.""" - run_sql('UPDATE bibdoc SET status=%s WHERE id=%s', (new_status, id)) + run_sql('UPDATE bibdoc SET status=%s WHERE id=%s', (new_status, self.id)) self.BuildFileList() self.BuildRelatedFileList() def addFilesNewVersion(self, files=[]): """add a new version of a file to an archive""" latestVersion = self.getLatestVersion() if latestVersion == "0": myversion = "1" else: myversion = str(int(latestVersion) + 1) for file in files: if os.path.exists(file): dummy, basename, extension = decompose_file(file) if extension: extension = '.' + extension #self.changeName(basename) #destination = propose_unique_name("%s/%s%s;%s" % #(self.basedir, basename, extension, myversion), True) destination = "%s/%s%s;%s" % (self.basedir, self.docname, extension, myversion) shutil.copyfile(file, destination) os.chmod(destination, 0644) + self.touch() self.BuildFileList() def purge(self): """Phisically Remove all the previous version of the given bibdoc""" version = self.getLatestVersion() if version > 1: for file in self.docfiles: if file.getVersion() < version: os.remove(file.getFullPath()) Md5Folder(self.basedir).update() + self.touch() self.BuildFileList() self.BuildRelatedFileList() def expunge(self): """Phisically remove all the traces of a given bibdoc""" for file in self.docfiles: os.remove(file.getFullPath()) Md5Folder(self.basedir).update() + self.touch() self.BuildFileList() self.BuildRelatedFileList() def addFilesNewFormat(self, files=[], version=""): """add a new format of a file to an archive""" if version == "": version = self.getLatestVersion() for file in files: if os.path.exists(file): dummy, basename, extension = decompose_file(file) if extension: extension = '.' + extension destination = "%s/%s%s;%s" % (self.basedir, self.docname, extension, version) if os.path.exists(destination): raise StandardError, "A file for docname '%s' for the recid '%s' already exists for the format '%s'" % (self.docname, self.recid, extension) #destination = propose_unique_name("%s/%s%s;%s" % #(self.basedir, basename, extension, version), True) shutil.copyfile(file, destination) os.chmod(destination, 0644) + self.touch() self.BuildFileList() def getIcon(self): if self.relatedFiles.has_key('Icon'): return self.relatedFiles['Icon'][0] else: return None def addIcon(self, file, basename=''): """link an icon with the bibdoc object""" #first check if an icon already exists existingIcon = self.getIcon() if existingIcon is not None: existingIcon.delete() #then add the new one if not basename: basename = decompose_file(file)[1] try: newicon = BibDoc(type='Icon', docname=basename) except StandardError: pass else: newicon.addFilesNewVersion([file]) run_sql("insert into bibdoc_bibdoc values(%s,%s,'Icon')", (self.id, newicon.getId(),)) if os.path.exists(newicon.getBaseDir()): old_umask = os.umask(022) fp = open("%s/.docid" % newicon.getBaseDir(), "w") fp.write(str(self.id)) fp.close() fp = open("%s/.type" % newicon.getBaseDir(), "w") fp.write(str(self.type)) fp.close() os.umask(old_umask) + self.touch() self.BuildRelatedFileList() def deleteIcon(self): existingIcon = self.getIcon() if existingIcon is not None: existingIcon.delete() + self.touch() self.BuildRelatedFileList() def display(self, version="", ln = cdslang): t = "" if version == "all": docfiles = self.listAllFiles() elif version != "": docfiles = self.listVersionFiles(version) else: docfiles = self.listLatestFiles() existingIcon = self.getIcon() if existingIcon is not None: imagepath = "%s/getfile.py?docid=%s&name=%s&format=gif" % \ (weburl, existingIcon.getId(), urllib.quote(existingIcon.getDocName())) else: imagepath = "%s/smallfiles.gif" % images versions = [] for version in listVersionsFromArray(docfiles): currversion = { 'version' : version, 'previous' : 0, 'content' : [] } if version == self.getLatestVersion() and version != "1": currversion['previous'] = 1 for docfile in docfiles: if docfile.getVersion() == version: currversion['content'].append(docfile.display(ln = ln)) versions.append(currversion) t = websubmit_templates.tmpl_bibdoc_filelist( ln = ln, weburl = weburl, versions = versions, imagepath = imagepath, docname = self.docname, id = self.id, recid = self.recid ) return t def changeName(self, newname): """Rename the bibdoc name. New name must not be already used by the linked bibrecs.""" - res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (recid, docname)) + res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (self.recid, self.docname)) if res: - raise StandardError, "A bibdoc called %s already exists for recid %s" % (docname, recid) + raise StandardError, "A bibdoc called %s already exists for recid %s" % (self.docname, self.recid) run_sql("update bibdoc set docname=%s where id=%s", (newname, self.id,)) for f in os.listdir(self.basedir): if f.startswith(self.docname): - sh.move('%s/%s' % (self.basedir, f), '%s/%s' % (self.basedir, f.replace(self.docname, newname, 1))) + shutil.move('%s/%s' % (self.basedir, f), '%s/%s' % (self.basedir, f.replace(self.docname, newname, 1))) self.docname = newname Md5Folder(self.basedir).update() + self.touch() self.BuildFileList() self.BuildRelatedFileList() def getDocName(self): """retrieve bibdoc name""" return self.docname def getBaseDir(self): """retrieve bibdoc base directory""" return self.basedir def getType(self): """retrieve bibdoc type""" return self.type def getRecid(self): """retrieve bibdoc recid""" return self.recid def getId(self): """retrieve bibdoc id""" return self.id def getFile(self, name, format, version): if version == "": docfiles = self.listLatestFiles() else: docfiles = self.listVersionFiles(version) for docfile in docfiles: if docfile.getName()==name and (docfile.getFormat()==format or not format): return docfile raise StandardError, "No file called '%s' of format '%s', version '%s'" % (name, format, version) def listVersions(self): versions = [] for docfile in self.docfiles: if not docfile.getVersion() in versions: versions.append(docfile.getVersion()) return versions def delete(self): """delete the current bibdoc instance""" run_sql("update bibdoc set status='DELETED' where id=%s", (self.id,)) def BuildFileList(self): """lists all files attached to the bibdoc""" self.docfiles = [] if os.path.exists(self.basedir): self.md5s = Md5Folder(self.basedir) for fil in os.listdir(self.basedir): if fil not in (".recid", ".docid", ".", "..", '.doc_checksum', '.type'): filepath = "%s/%s" % (self.basedir, fil) fileversion = re.sub(".*;", "", fil) fullname = fil.replace(";%s" % fileversion, "") checksum = self.md5s.get_checksum(fil) (dirname, basename, extension) = decompose_file(fullname) # we can append file: self.docfiles.append(BibDocFile(filepath, self.type, fileversion, basename, extension, self.id, self.status, checksum)) def BuildRelatedFileList(self): self.relatedFiles = {} res = run_sql("select ln.id_bibdoc2,ln.type,bibdoc.status from " "bibdoc_bibdoc as ln,bibdoc where id=ln.id_bibdoc2 and " "ln.id_bibdoc1=%s", (self.id,)) for row in res: bibdocid = row[0] type = row[1] if row[2] != 'DELETED': if not self.relatedFiles.has_key(type): self.relatedFiles[type] = [] try: cur_doc = BibDoc(bibdocid=bibdocid) self.relatedFiles[type].append(cur_doc) except StandardError: pass def listAllFiles(self): return self.docfiles def listLatestFiles(self): return self.listVersionFiles(self.getLatestVersion()) def listVersionFiles(self, version): tmp = [] for docfile in self.docfiles: if docfile.getVersion() == str(version): tmp.append(docfile) return tmp def getLatestVersion(self): if len(self.docfiles) > 0: self.docfiles.sort(orderFilesWithVersion) return self.docfiles[0].getVersion() else: return 0 def getFileNumber(self): return len(self.docfiles) def registerDownload(self, addressIp, version, format, userid=0): return run_sql("INSERT INTO rnkDOWNLOADS " "(id_bibrec,id_bibdoc,file_version,file_format," "id_user,client_host,download_time) VALUES " "(%s,%s,%s,%s,%s,INET_ATON(%s),NOW())", (self.recid, self.id, version, format.upper(), userid, addressIp,)) class BibDocFile: """this class represents a physical file in the CDS Invenio filesystem""" def __init__(self, fullpath, type, version, name, format, bibdocid, status, checksum): self.fullpath = fullpath self.type = type self.bibdocid = bibdocid self.version = version self.status = status self.checksum = checksum self.size = os.path.getsize(fullpath) self.md = os.path.getmtime(fullpath) try: self.cd = os.path.getctime(fullpath) except: self.cd = self.md self.name = name self.format = format self.dir = os.path.dirname(fullpath) if format == "": self.mime = "text/plain" self.encoding = "" self.fullname = name else: self.fullname = "%s.%s" % (name, format) (self.mime, self.encoding) = _mimes.guess_type(self.fullname) if self.mime is None: self.mime = "text/plain" def display(self, ln = cdslang): if self.format != "": format = ".%s" % self.format else: format = "" return websubmit_templates.tmpl_bibdocfile_filelist( ln = ln, weburl = weburl, id = self.bibdocid, selfformat = self.format, version = self.version, name = self.name, format = format, size = self.size, ) def isRestricted(self): """return restriction state""" return self.status != '' def getType(self): return self.type def getPath(self): return self.fullpath def getBibDocId(self): return self.bibdocid def getName(self): return self.name def getFullName(self): return self.fullname def getFullPath(self): return self.fullpath def getFormat(self): return self.format def getSize(self): return self.size def getVersion(self): return self.version def getChecksum(self): return self.checksum def getRecid(self): return run_sql("select id_bibrec from bibrec_bibdoc where " "id_bibdoc=%s",(self.bibdocid,))[0][0] def stream(self, req): if self.status: (auth_code, auth_message) = acc_authorize_action(req, 'viewrestrdoc', status=self.status) else: auth_code = 0 if auth_code == 0: if os.path.exists(self.fullpath): req.content_type = self.mime req.encoding = self.encoding req.filename = self.fullname req.headers_out["Content-Disposition"] = \ "attachment; filename=%s" % quoteattr(self.fullname) req.send_http_header() fp = file(self.fullpath, "r") content = fp.read() fp.close() return content else: raise StandardError, "%s does not exists!" % self.fullpath else: raise StandardError, "You are not authorized to download %s: %s" % (self.fullname, auth_message) def readfile(path): """Read the contents of a text file and return them as a string. Return an empty string if unable to access the file for reading. @param path: (string) - path to the file @return: (string) contents of file or empty string """ content = "" if os.access(path, os.F_OK|os.R_OK): try: fp = open(path, "r") except IOError: pass else: content = fp.read() fp.close() return content def listTypesFromArray(bibdocs): types = [] for bibdoc in bibdocs: if not bibdoc.getType() in types: types.append(bibdoc.getType()) return types def listVersionsFromArray(docfiles): versions = [] for docfile in docfiles: if not docfile.getVersion() in versions: versions.append(docfile.getVersion()) return versions def orderFilesWithVersion(docfile1, docfile2): """order docfile objects according to their version""" version1 = int(docfile1.getVersion()) version2 = int(docfile2.getVersion()) return cmp(version2, version1) class Md5Folder: """Manage all the Md5 checksum about a folder""" def __init__(self, folder): """Initialize the class from the md5 checksum of a given path""" self.folder = folder self.load() def update(self, only_new = True): """Update the .doc_checksum file with the current files. If only_new is specified then only not already calculated file are calculated.""" self.md5s = {} for filename in os.listdir(self.folder): if not only_new or self.md5s.get(filename, None) is None and \ not filename.startswith('.'): self.md5s[filename] = md5.new(open("%s/%s" % (self.folder, filename), "rb").read()).hexdigest() self.store() def store(self): """Store the current md5 dictionary into .doc_checksum""" old_umask = os.umask(022) md5file = open("%s/.doc_checksum" % self.folder, "w") for key, value in self.md5s.items(): md5file.write('%s *%s\n' % (value, key)) os.umask(old_umask) def load(self): """Load .doc_checksum into the md5 dictionary""" self.md5s = {} try: for row in open("%s/.doc_checksum" % self.folder, "r"): md5hash = row[:32] filename = row[34:].strip() self.md5s[filename] = md5hash except IOError: pass def check(self, filename = ''): """Check the specified file or all the files for which it exists a hash for being coherent with the stored hash.""" if filename and filename in self.md5s.keys(): return self.md5s[filename] == md5.new(open("%s/%s" % (self.folder, filename), "rb").read()).hexdigest() else: for filename, md5hash in self.md5s.items(): if md5.new(open("%s/%s" % (self.folder, filename), "rb").read()).hexdigest() != md5hash: return False return True def get_checksum(self, filename): md5hash = self.md5s.get(filename, None) if md5hash is None: self.update() # Now it should not fail! md5hash = self.md5s[filename] return md5hash