diff --git a/configure.ac b/configure.ac index 9ab4a6645..668ae79fa 100644 --- a/configure.ac +++ b/configure.ac @@ -1,701 +1,702 @@ ## $Id$ ## Purpose: CDS Invenio main configure.ac file. ## Note: If you change this file, please run "autoreconf" to regenerate the "configure" script. ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. ## Initialize autoconf and automake: -AC_INIT(cds-invenio, 0.90.1.20060822, cds.support@cern.ch) -AM_INIT_AUTOMAKE(cds-invenio, 0.90.1.20060822) +AC_INIT(cds-invenio, 0.90.1.20060920, cds.support@cern.ch) +AM_INIT_AUTOMAKE(cds-invenio, 0.90.1.20060920) ## Check for install: AC_PROG_INSTALL ## Define WEBURL URL: AC_ARG_WITH(weburl, AC_HELP_STRING([--with-weburl], [specify URL where web files will be reached (e.g. http://webserver.domain.com)]), WEBURL=${withval}) if test -z "$WEBURL"; then AC_MSG_ERROR([ You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached. Example: "--with-weburl=http://webserver.domain.com/doc/".]) fi AC_ARG_WITH(sweburl, AC_HELP_STRING([--with-sweburl], [specify URL where HTTPS files will be reached (e.g. https://webserver.domain.com)]), SWEBURL=${withval}) if test -z "${SWEBURL}"; then SWEBURL="${WEBURL}" AC_MSG_WARN([ You have not specified secure URL for accessing the site (--with-sweburl). User authentication will be done in clear text across the network. ]) fi ## Define DBHOST database host: AC_ARG_WITH(dbhost, AC_HELP_STRING([--with-dbhost], [specify DB server (e.g. sqlserver.domain.com)]), DBHOST=${withval}) if test -z "$DBHOST"; then AC_MSG_ERROR([ You have not used the "--with-dbhost" argument that should define which DB server to use. Example: "--with-dbhost=sqlserver.domain.com".]) fi ## Define DBNAME database name: AC_ARG_WITH(dbname, AC_HELP_STRING([--with-dbname], [specify DB name (e.g. cdsinvenio)]), DBNAME=${withval}) if test -z "$DBNAME"; then AC_MSG_ERROR([ You have not used the "--with-dbname" argument that should define DB name. Example: "--with-dbname=cdsinvenio".]) fi ## Define DBUSER user name: AC_ARG_WITH(dbuser, AC_HELP_STRING([--with-dbuser], [specify DB user name (e.g. cdsinvenio)]), DBUSER=${withval}) if test -z "$DBUSER"; then AC_MSG_ERROR([ You have not used the "--with-dbuser" argument that should define DB user name. Example: "--with-dbuser=cdsinvenio".]) fi ## Define DBPASS user password: AC_ARG_WITH(dbpass, AC_HELP_STRING([--with-dbpass], [specify DB user password (e.g. myp1ss)]), DBPASS=${withval}) if test -z "$DBPASS"; then AC_MSG_ERROR([ You have not used the "--with-dbpass" argument that should define DB user password. Example: "--with-dbpass=myp1ss".]) fi ## Check for gettext support AM_GNU_GETTEXT(external) AM_GNU_GETTEXT_VERSION(0.14.4) ## Check for WML: AC_PATH_PROG(WML, wml) if test -z "$WML"; then AC_MSG_ERROR([ WML (Website META Language) was not found in your PATH. Please install it first. Available from .]) fi ## Set option -c in WML so that it can work from a separate build tree WML="${WML} -c" ## Check for MySQL client: AC_PATH_PROG(MYSQL, mysql) if test -z "$MYSQL"; then AC_MSG_ERROR([ MySQL command-line client was not found in your PATH. Please install it first. Available from .]) fi ## Check for Python: AC_MSG_CHECKING(for python) AC_ARG_WITH(python, AC_HELP_STRING([--with-python], [path to a specific Python binary (optional)]), PYTHON=${withval}) if test -n "$PYTHON"; then AC_MSG_RESULT($PYTHON) else AC_PATH_PROG(PYTHON, python) if test -z "$PYTHON"; then AC_MSG_ERROR([ Python was not found in your PATH. Please either install it in your PATH or specify --with-python configure option. Python is available from .]) fi fi ## Check for Python version and modules: AC_MSG_CHECKING(for required Python modules) $PYTHON ${srcdir}/configure-tests.py if test $? -ne 0; then AC_MSG_ERROR([Please fix the above Python problem before continuing.]) fi AC_MSG_RESULT(found) ## Check for PHP AC_PATH_PROG(PHP, php) ## Check for Acrobat Reader: AC_PATH_PROG(ACROREAD, acroread) if test -z "$ACROREAD"; then AC_MSG_WARN([ Acrobat Reader was not found in your PATH. It is used in the WebSubmit module for automatic conversion of submitted documents. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Acrobat Reader is available from .]) fi ## Check for gzip: AC_PATH_PROG(GZIP, gzip) if test -z "$GZIP"; then AC_MSG_WARN([ Gzip was not found in your PATH. It is used in the WebSubmit module to compress the data submitted in an archive. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Gzip is available from .]) fi ## Check for gunzip: AC_PATH_PROG(GUNZIP, gunzip) if test -z "$GUNZIP"; then AC_MSG_WARN([ Gunzip was not found in your PATH. It is used in the WebSubmit module to correctly deal with submitted compressed files. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Gunzip is available from .]) fi ## Check for tar: AC_PATH_PROG(TAR, tar) if test -z "$TAR"; then AC_MSG_WARN([ Tar was not found in your PATH. It is used in the WebSubmit module to pack the submitted data into an archive. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Tar is available from .]) fi ## Check for ps2pdf: AC_PATH_PROG(PS2PDF, ps2pdf) if test -z "$PS2PDF"; then AC_MSG_WARN([ ps2pdf was not found in your PATH. It is used in the WebSubmit module to convert submitted PostScripts into PDF. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. PS2PDF is available from .]) fi ## Check for pdftotext: AC_PATH_PROG(PDFTOTEXT, pdftotext) if test -z "$PDFTOTEXT"; then AC_MSG_WARN([ pdftotext was not found in your PATH. It is used for the fulltext indexation of PDF files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. pdftotext is available from . ]) fi ## Check for pstotext: AC_PATH_PROG(PSTOTEXT, pstotext) if test -z "$PSTOTEXT"; then AC_MSG_WARN([ pstotext was not found in your PATH. It is used for the fulltext indexation of PDF and PostScript files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. pstotext is available from . ]) fi ## Check for ps2ascii: AC_PATH_PROG(PSTOASCII, ps2ascii) if test -z "$PSTOASCII"; then AC_MSG_WARN([ ps2ascii was not found in your PATH. It is used for the fulltext indexation of PostScript files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. ps2ascii is available from . ]) fi ## Check for antiword: AC_PATH_PROG(ANTIWORD, antiword) if test -z "$ANTIWORD"; then AC_MSG_WARN([ antiword was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. antiword is available from . ]) fi ## Check for catdoc: AC_PATH_PROG(CATDOC, catdoc) if test -z "$CATDOC"; then AC_MSG_WARN([ catdoc was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. catdoc is available from . ]) fi ## Check for wvText: AC_PATH_PROG(WVTEXT, wvText) if test -z "$WVTEXT"; then AC_MSG_WARN([ wvText was not found in your PATH. It is used for the fulltext indexation of Microsoft Word files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. wvText is available from . ]) fi ## Check for ppthtml: AC_PATH_PROG(PPTHTML, ppthtml) if test -z "$PPTHTML"; then AC_MSG_WARN([ ppthtml was found in your PATH. It is used for the fulltext indexation of Microsoft PowerPoint files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. ppthtml is available from . ]) fi ## Check for xlhtml: AC_PATH_PROG(XLHTML, xlhtml) if test -z "$XLHTML"; then AC_MSG_WARN([ xlhtml was found in your PATH. It is used for the fulltext indexation of Microsoft Excel files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. xlhtml is available from . ]) fi ## Check for html2text: AC_PATH_PROG(HTMLTOTEXT, html2text) if test -z "$HTMLTOTEXT"; then AC_MSG_WARN([ html2text was found in your PATH. It is used for the fulltext indexation of Microsoft PowerPoint and Excel files. You can continue without it but you may miss fulltext searching capability of CDS Invenio. We recomend you to install it first and to rerun the configure script. html2text is available from . ]) fi ## Check for Giftext: AC_PATH_PROG(GIFTEXT, giftext) if test -z "$GIFTEXT"; then AC_MSG_WARN([ Giftext was not found in your PATH. It is used in the WebSubmit module to create an icon from a submitted picture. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Giftext is available from .]) fi ## Check for file: AC_PATH_PROG(FILE, file) if test -z "$FILE"; then AC_MSG_WARN([ File was not found in your PATH. It is used in the WebSubmit module to check the validity of the submitted files. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. File is available from .]) fi ## Check for convert: AC_PATH_PROG(CONVERT, convert) if test -z "$CONVERT"; then AC_MSG_WARN([ Convert was not found in your PATH. It is used in the WebSubmit module to create an icon from a submitted picture. You can continue without it but you will miss some CDS Invenio functionality. We recommend you to install it first and to rerun the configure script. Convert is available from .]) fi ## Check for CLISP: AC_MSG_CHECKING(for clisp) AC_ARG_WITH(clisp, AC_HELP_STRING([--with-clisp], [path to a specific CLISP binary (optional)]), CLISP=${withval}) if test -n "$CLISP"; then AC_MSG_RESULT($CLISP) else AC_PATH_PROG(CLISP, clisp) if test -z "$CLISP"; then AC_MSG_WARN([ GNU CLISP was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, SBCL or CMUCL can be used instead of CLISP.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CMUCL nor SBCL) and to rerun the configure script. GNU CLISP is available from .]) fi fi ## Check for CMUCL: AC_MSG_CHECKING(for cmucl) AC_ARG_WITH(cmucl, AC_HELP_STRING([--with-cmucl], [path to a specific CMUCL binary (optional)]), CMUCL=${withval}) if test -n "$CMUCL"; then AC_MSG_RESULT($CMUCL) else AC_PATH_PROG(CMUCL, cmucl) if test -z "$CMUCL"; then AC_MSG_CHECKING(for lisp) # CMUCL can also be installed under `lisp' exec name AC_PATH_PROG(CMUCL, lisp) fi if test -z "$CMUCL"; then AC_MSG_WARN([ CMUCL was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, CLISP or SBCL can be used instead of CMUCL.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP nor SBCL) and to rerun the configure script. CMUCL is available from .]) fi fi ## Check for SBCL: AC_MSG_CHECKING(for sbcl) AC_ARG_WITH(sbcl, AC_HELP_STRING([--with-sbcl], [path to a specific SBCL binary (optional)]), SBCL=${withval}) if test -n "$SBCL"; then AC_MSG_RESULT($SBCL) else AC_PATH_PROG(SBCL, sbcl) if test -z "$SBCL"; then AC_MSG_WARN([ SBCL was not found in your PATH. It is used by the WebStat module to produce statistics about CDS Invenio usage. (Alternatively, CLISP or CMUCL can be used instead of SBCL.) You can continue without it but you will miss this feature. We recommend you to install it first (if you don't have neither CLISP nor CMUCL) and to rerun the configure script. SBCL is available from .]) fi fi ## Check for gnuplot: AC_PATH_PROG(GNUPLOT, gnuplot) if test -z "$GNUPLOT"; then AC_MSG_WARN([ Gnuplot was not found in your PATH. It is used by the BibRank module to produce graphs about download and citation history. You can continue without it but you will miss these graphs. We recommend you to install it first and to rerun the configure script. Gnuplot is available from .]) fi ## Substitute variables: AC_SUBST(VERSION) AC_SUBST(WML) AC_SUBST(MYSQL) AC_SUBST(PHP) AC_SUBST(PYTHON) AC_SUBST(CLIDIR) AC_SUBST(WEBURL) AC_SUBST(SWEBURL) AC_SUBST(DBHOST) AC_SUBST(DBNAME) AC_SUBST(DBUSER) AC_SUBST(DBPASS) AC_SUBST(PDFTOTEXT) AC_SUBST(PSTOTEXT) AC_SUBST(PSTOASCII) AC_SUBST(ANTIWORD) AC_SUBST(CATDOC) AC_SUBST(WVTEXT) AC_SUBST(PPTHTML) AC_SUBST(XLHTML) AC_SUBST(HTMLTOTEXT) AC_SUBST(localstatedir, `eval echo "${localstatedir}"`) AC_SUBST(CACHEDIR) AC_SUBST(CLISP) AC_SUBST(CMUCL) AC_SUBST(SBCL) AC_SUBST(GNUPLOT) AC_SUBST(DJPEG) AC_SUBST(CONVERT) AC_SUBST(GIFTEXT) AC_SUBST(JPEGSIZE) AC_SUBST(PNMSCALE) AC_SUBST(PPMQUANT) AC_SUBST(PPMTOGIF) AC_SUBST(GIFINTER) AC_SUBST(GIFRSIZE) ## Define output files: AC_CONFIG_FILES([config.nice \ Makefile \ po/Makefile.in \ config/Makefile \ config/configbis.wml \ modules/Makefile \ modules/bibconvert/Makefile \ modules/bibconvert/bin/Makefile \ modules/bibconvert/bin/bibconvert \ modules/bibconvert/doc/Makefile \ modules/bibconvert/doc/admin/Makefile \ modules/bibconvert/doc/hacking/Makefile \ modules/bibconvert/etc/Makefile \ modules/bibconvert/lib/Makefile \ modules/bibmatch/Makefile \ modules/bibmatch/bin/Makefile \ modules/bibmatch/bin/bibmatch \ modules/bibmatch/doc/Makefile \ modules/bibmatch/doc/admin/Makefile \ modules/bibmatch/etc/Makefile \ modules/bibmatch/lib/Makefile \ modules/bibedit/Makefile \ modules/bibedit/bin/Makefile \ modules/bibedit/bin/refextract \ modules/bibedit/bin/xmlmarclint \ modules/bibedit/bin/xmlmarc2textmarc \ modules/bibedit/doc/Makefile \ modules/bibedit/doc/admin/Makefile \ modules/bibedit/doc/hacking/Makefile \ modules/bibedit/etc/Makefile \ modules/bibedit/lib/Makefile \ modules/bibedit/web/Makefile \ modules/bibedit/web/admin/Makefile \ modules/bibformat/Makefile \ modules/bibformat/bin/Makefile \ modules/bibformat/bin/bibformat \ modules/bibformat/bin/bibreformat \ modules/bibformat/doc/Makefile \ modules/bibformat/doc/admin/Makefile \ modules/bibformat/doc/hacking/Makefile \ modules/bibformat/etc/Makefile \ modules/bibformat/etc/output_formats/Makefile \ modules/bibformat/etc/format_templates/Makefile \ modules/bibformat/lib/Makefile \ modules/bibformat/lib/elements/Makefile \ modules/bibformat/lib/core/Makefile \ modules/bibformat/lib/common/Makefile \ modules/bibformat/web/Makefile \ modules/bibformat/web/admin/Makefile \ modules/bibharvest/Makefile \ modules/bibharvest/bin/Makefile \ modules/bibharvest/bin/bibharvest \ modules/bibharvest/bin/oaiharvest \ modules/bibharvest/bin/oaiarchive \ modules/bibharvest/doc/Makefile \ modules/bibharvest/doc/admin/Makefile \ modules/bibharvest/doc/hacking/Makefile \ modules/bibharvest/lib/Makefile \ modules/bibharvest/web/Makefile \ modules/bibharvest/web/admin/Makefile \ modules/bibclassify/Makefile modules/bibclassify/bin/Makefile modules/bibclassify/bin/bibclassify modules/bibclassify/doc/Makefile \ modules/bibclassify/doc/admin/Makefile \ modules/bibclassify/doc/hacking/Makefile \ modules/bibclassify/etc/Makefile modules/bibclassify/lib/Makefile modules/bibindex/Makefile \ modules/bibindex/bin/Makefile \ modules/bibindex/bin/bibindex \ modules/bibindex/bin/bibstat \ modules/bibindex/doc/Makefile \ modules/bibindex/doc/admin/Makefile \ modules/bibindex/doc/hacking/Makefile \ modules/bibindex/lib/Makefile \ modules/bibindex/web/Makefile \ modules/bibindex/web/admin/Makefile \ modules/bibrank/Makefile \ modules/bibrank/bin/Makefile \ modules/bibrank/bin/bibrank \ modules/bibrank/bin/bibrankgkb \ modules/bibrank/doc/Makefile \ modules/bibrank/doc/admin/Makefile \ modules/bibrank/doc/hacking/Makefile \ modules/bibrank/etc/Makefile \ modules/bibrank/etc/bibrankgkb.cfg \ modules/bibrank/etc/demo_jif.cfg \ modules/bibrank/etc/template_single_tag_rank_method.cfg \ modules/bibrank/lib/Makefile \ modules/bibrank/web/Makefile \ modules/bibrank/web/admin/Makefile \ modules/bibsched/Makefile \ modules/bibsched/bin/Makefile \ modules/bibsched/bin/bibsched \ modules/bibsched/bin/bibtaskex \ modules/bibsched/doc/Makefile \ modules/bibsched/doc/admin/Makefile \ modules/bibsched/doc/hacking/Makefile \ + modules/bibsched/lib/Makefile \ modules/bibupload/Makefile \ modules/bibupload/bin/Makefile \ modules/bibupload/bin/bibupload \ modules/bibupload/doc/Makefile \ modules/bibupload/doc/admin/Makefile \ modules/bibupload/doc/hacking/Makefile \ modules/bibupload/lib/Makefile \ modules/elmsubmit/Makefile \ modules/elmsubmit/bin/Makefile \ modules/elmsubmit/bin/elmsubmit \ modules/elmsubmit/doc/Makefile \ modules/elmsubmit/doc/admin/Makefile \ modules/elmsubmit/doc/hacking/Makefile \ modules/elmsubmit/etc/Makefile \ modules/elmsubmit/etc/elmsubmit.cfg \ modules/elmsubmit/lib/Makefile \ modules/elmsubmit/lib/magic/Makefile \ modules/miscutil/Makefile \ modules/miscutil/bin/Makefile \ modules/miscutil/bin/dbexec \ modules/miscutil/bin/dbtest \ modules/miscutil/bin/testsuite \ modules/miscutil/bin/regressiontestsuite \ modules/miscutil/lib/Makefile \ modules/miscutil/demo/Makefile \ modules/miscutil/sql/Makefile \ modules/miscutil/web/Makefile \ modules/miscutil/doc/Makefile \ modules/miscutil/doc/hacking/Makefile \ modules/webaccess/Makefile \ modules/webaccess/bin/Makefile \ modules/webaccess/bin/authaction \ modules/webaccess/bin/webaccessadmin \ modules/webaccess/doc/Makefile \ modules/webaccess/doc/admin/Makefile \ modules/webaccess/doc/hacking/Makefile \ modules/webaccess/lib/Makefile \ modules/webaccess/web/Makefile \ modules/webaccess/web/admin/Makefile \ modules/webalert/Makefile \ modules/webalert/bin/Makefile \ modules/webalert/bin/alertengine \ modules/webalert/doc/Makefile \ modules/webalert/doc/admin/Makefile \ modules/webalert/doc/hacking/Makefile \ modules/webalert/lib/Makefile \ modules/webalert/web/Makefile \ modules/webhelp/Makefile \ modules/webhelp/web/Makefile \ modules/webhelp/web/hacking/Makefile \ modules/webhelp/web/admin/Makefile \ modules/webhelp/web/admin/howto/Makefile \ modules/websearch/Makefile \ modules/websearch/bin/Makefile \ modules/websearch/bin/webcoll \ modules/websearch/doc/Makefile \ modules/websearch/doc/admin/Makefile \ modules/websearch/doc/hacking/Makefile \ modules/websearch/lib/Makefile \ modules/websearch/web/Makefile \ modules/websearch/web/admin/Makefile \ modules/websession/Makefile \ modules/websession/bin/Makefile \ modules/websession/bin/sessiongc \ modules/websession/doc/Makefile \ modules/websession/doc/admin/Makefile \ modules/websession/doc/hacking/Makefile \ modules/websession/lib/Makefile \ modules/websession/web/Makefile \ modules/webstat/Makefile \ modules/webstat/bin/Makefile \ modules/webstat/bin/webstat \ modules/webstat/doc/Makefile \ modules/webstat/doc/admin/Makefile \ modules/webstat/doc/hacking/Makefile \ modules/webstat/etc/Makefile \ modules/webstat/lib/Makefile \ modules/webstyle/Makefile \ modules/webstyle/css/Makefile \ modules/webstyle/doc/Makefile \ modules/webstyle/doc/admin/Makefile \ modules/webstyle/doc/hacking/Makefile \ modules/webstyle/img/Makefile \ modules/webstyle/lib/Makefile \ modules/webcomment/Makefile \ modules/webcomment/doc/Makefile \ modules/webcomment/doc/admin/Makefile \ modules/webcomment/doc/hacking/Makefile \ modules/webcomment/lib/Makefile \ modules/webcomment/web/Makefile \ modules/webcomment/web/admin/Makefile \ modules/webbasket/Makefile \ modules/webbasket/doc/Makefile \ modules/webbasket/doc/admin/Makefile \ modules/webbasket/doc/hacking/Makefile \ modules/webbasket/lib/Makefile \ modules/webbasket/web/Makefile \ modules/webmessage/Makefile \ modules/webmessage/bin/Makefile \ modules/webmessage/bin/webmessageadmin \ modules/webmessage/doc/Makefile \ modules/webmessage/doc/admin/Makefile \ modules/webmessage/doc/hacking/Makefile \ modules/webmessage/lib/Makefile \ modules/webmessage/web/Makefile \ modules/websubmit/Makefile \ modules/websubmit/bin/Makefile \ modules/websubmit/bin/thumbmaker \ modules/websubmit/etc/Makefile \ modules/websubmit/etc/bibconvert/Makefile \ modules/websubmit/etc/bibconvert/KB/Makefile \ modules/websubmit/etc/bibconvert/config/Makefile \ modules/websubmit/etc/bibconvert/config/EDSPICTcreate.tpl \ modules/websubmit/etc/bibconvert/config/EDSRPICTcreate.tpl \ modules/websubmit/etc/bibconvert/config/EDSTEXTcreate.tpl \ modules/websubmit/etc/bibconvert/config/EDSRTEXTcreate.tpl \ modules/websubmit/doc/Makefile \ modules/websubmit/doc/admin/Makefile \ modules/websubmit/doc/hacking/Makefile \ modules/websubmit/lib/Makefile \ modules/websubmit/lib/functions/Makefile \ modules/websubmit/web/Makefile \ modules/websubmit/web/admin/Makefile \ ]) AC_CONFIG_COMMANDS(.wmlrc, [ cat<.wmlrc -Iconfig -I${srcdir}/config EOF ]) ## Finally, write output files: AC_OUTPUT ## Write help: AC_MSG_RESULT([****************************************************************************]) AC_MSG_RESULT([** Your CDS Invenio installation is now ready for building. **]) AC_MSG_RESULT([** You have entered the following parameters: **]) AC_MSG_RESULT([** - CDS Invenio main install directory: ${prefix}]) AC_MSG_RESULT([** - Web home URL: $WEBURL]) AC_MSG_RESULT([** - Secure web home URL: $SWEBURL]) AC_MSG_RESULT([** - DB server: $DBHOST]) AC_MSG_RESULT([** - DB name: $DBNAME]) AC_MSG_RESULT([** - DB username: $DBUSER]) AC_MSG_RESULT([** - DB password: $DBPASS]) AC_MSG_RESULT([** - Python executable: $PYTHON]) AC_MSG_RESULT([** - CLISP executable: $CLISP]) AC_MSG_RESULT([** - CMUCL executable: $CMUCL]) AC_MSG_RESULT([** - SBCL executable: $SBCL]) AC_MSG_RESULT([** Here are the steps to continue the building process: **]) AC_MSG_RESULT([** 1) Customize self-explanatory config file './config/config.wml'. **]) AC_MSG_RESULT([** 2) Type 'make' to build your customized CDS Invenio installation. **]) AC_MSG_RESULT([** 3) Type 'make create-tables' if you have not created tables before. **]) AC_MSG_RESULT([** 4) Type 'make install' to install the system. **]) AC_MSG_RESULT([** 5) Type 'make create-demo-site' to install and test the demo site. **]) AC_MSG_RESULT([** Good luck, and thanks for choosing CDS Invenio. **]) AC_MSG_RESULT([** -- CDS Development Group **]) AC_MSG_RESULT([****************************************************************************]) ## end of file diff --git a/modules/bibsched/Makefile.am b/modules/bibsched/Makefile.am index dde3dc9ba..c1a8c7c7a 100644 --- a/modules/bibsched/Makefile.am +++ b/modules/bibsched/Makefile.am @@ -1,22 +1,22 @@ ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -SUBDIRS = bin doc +SUBDIRS = bin doc lib CLEANFILES = *~ \ No newline at end of file diff --git a/modules/bibsched/bin/bibsched.in b/modules/bibsched/bin/bibsched.in index 8e95b9747..84ff4e60a 100644 --- a/modules/bibsched/bin/bibsched.in +++ b/modules/bibsched/bin/bibsched.in @@ -1,798 +1,35 @@ #!@PYTHON@ ## -*- mode: python; coding: utf-8; -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """BibSched - task management, scheduling and executing system for CDS Invenio """ __revision__ = "$Id$" -### -- local configuration section starts here --- -cfg_valid_processes = ["bibindex","bibupload","bibreformat","webcoll","bibtaskex","bibrank","oaiharvest","oaiarchive"] # which tasks are reconized as valid? -### -- local configuration section ends here --- - -## import interesting modules: try: - import os - import imp - import string - import sys - import time - import sre - import getopt - import curses - import curses.panel - from curses.wrapper import wrapper - import signal + from invenio.bibsched import main except ImportError, e: print "Error: %s" % e import sys sys.exit(1) -try: - from invenio.config import * - from invenio.dbquery import run_sql -except ImportError, e: - print "Error: %s" % e - import sys - sys.exit(1) - -def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"): - """Returns a date string according to the format string. - It can handle normal date strings and shifts with respect - to now.""" - try: - date = time.time() - shift_re=sre.compile("([-\+]{0,1})([\d]+)([dhms])") - factors = {"d":24*3600, "h":3600, "m":60, "s":1} - m = shift_re.match(var) - if m: - sign = m.groups()[0] == "-" and -1 or 1 - factor = factors[m.groups()[2]] - value = float(m.groups()[1]) - date = time.localtime(date + sign * factor * value) - date = time.strftime(format_string, date) - else: - date = time.strptime(var, format_string) - date = time.strftime(format_string, date) - return date - except: - return None - -def get_my_pid(process,args=''): - if sys.platform.startswith('freebsd'): - COMMAND = "ps -o pid,args | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, args) - else: - COMMAND = "ps -C %s o '%%p%%a' | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, process, args) - answer = string.strip(os.popen(COMMAND).read()) - if answer=='': - answer = 0 - else: - answer = answer[:string.find(answer,' ')] - return int(answer) - -def get_output_channelnames(task_id): - "Construct and return filename for stdout and stderr for the task 'task_id'." - filenamebase = "%s/bibsched_task_%d" % (logdir, task_id) - return [filenamebase + ".log", filenamebase + ".err"] - -class Manager: - def __init__(self): - self.helper_modules = cfg_valid_processes - self.running = 1 - self.footer_move_mode = "[KeyUp/KeyDown Move] [M Select mode] [Q Quit]" - self.footer_auto_mode = "[A Manual mode] [1/2 Display Type] [Q Quit]" - self.footer_select_mode = "[KeyUp/KeyDown/PgUp/PgDown Select] [L View Log] [1/2 Display Type] [M Move mode] [A Auto mode] [Q Quit]" - self.footer_waiting_item = "[R Run] [D Delete]" - self.footer_running_item = "[S Sleep] [T Stop] [K Kill]" - self.footer_stopped_item = "[I Initialise] [D Delete]" - self.footer_sleeping_item = "[W Wake Up]" - self.item_status = "" - self.selected_line = 2 - self.rows = [] - self.panel = None - self.display = 2 - self.first_visible_line = 0 - self.move_mode = 0 - self.auto_mode = 0 - self.currentrow = ["","","","","","",""] - wrapper( self.start ) - - def handle_keys(self, chr): - if chr == -1: - return - if self.auto_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,curses.KEY_PPAGE,curses.KEY_NPAGE,ord("q"),ord("Q"),ord("a"),ord("A"),ord("1"),ord("2"))): - self.display_in_footer("in automatic mode") - self.stdscr.refresh() - elif self.move_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,ord("m"),ord("M"),ord("q"),ord("Q"))): - self.display_in_footer("in move mode") - self.stdscr.refresh() - else: - if chr == curses.KEY_UP: - if self.move_mode: - self.move_up() - else: - self.selected_line = max( self.selected_line - 1 , 2 ) - self.repaint() - if chr == curses.KEY_PPAGE: - self.selected_line = max( self.selected_line - 10 , 2 ) - self.repaint() - elif chr == curses.KEY_DOWN: - if self.move_mode: - self.move_down() - else: - self.selected_line = min(self.selected_line + 1, len(self.rows) + 1 ) - self.repaint() - elif chr == curses.KEY_NPAGE: - self.selected_line = min(self.selected_line + 10, len(self.rows) + 1 ) - self.repaint() - elif chr == curses.KEY_HOME: - self.first_visible_line = 0 - self.selected_line = 2 - elif chr in (ord("a"), ord("A")): - self.change_auto_mode() - elif chr in (ord("l"), ord("L")): - self.openlog() - elif chr in (ord("w"), ord("W")): - self.wakeup() - elif chr in (ord("r"), ord("R")): - self.run() - elif chr in (ord("s"), ord("S")): - self.sleep() - elif chr in (ord("k"), ord("K")): - self.kill() - elif chr in (ord("t"), ord("T")): - self.stop() - elif chr in (ord("d"), ord("D")): - self.delete() - elif chr in (ord("i"), ord("I")): - self.init() - elif chr in (ord("m"), ord("M")): - self.change_select_mode() - elif chr == ord("1"): - self.display = 1 - self.first_visible_line = 0 - self.selected_line = 2 - self.display_in_footer("only done processes are displayed") - elif chr == ord("2"): - self.display = 2 - self.first_visible_line = 0 - self.selected_line = 2 - self.display_in_footer("only not done processes are displayed") - elif chr in (ord("q"), ord("Q")): - if curses.panel.top_panel() == self.panel: - self.panel.bottom() - curses.panel.update_panels() - else: - self.running = 0 - return - - def set_status(self, id, status): - return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, id)) - - def set_progress(self, id, progress): - return run_sql("UPDATE schTASK set progress=%s WHERE id=%s", (progress, id)) - - def openlog(self): - self.win = curses.newwin( self.height-2, self.width-2, 1, 1 ) - self.panel = curses.panel.new_panel( self.win ) - self.panel.top() - self.win.border() - self.win.addstr(1, 1, "Not implemented yet...") - self.win.refresh() - curses.panel.update_panels() - - def count_processes(self,status): - out = 0 - res = run_sql("SELECT COUNT(id) FROM schTASK WHERE status=%s GROUP BY status", (status,)) - try: - out = res[0][0] - except: - pass - return out - - def wakeup(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: - self.display_in_footer("a process is already running!") - elif status == "SLEEPING": - mypid = get_my_pid(process,str(id)) - if mypid!=0: - os.kill(mypid, signal.SIGCONT) - self.display_in_footer("process woken up") - else: - self.display_in_footer("process is not sleeping") - self.stdscr.refresh() - - def run(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - sleeptime = self.currentrow[4] - if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: - self.display_in_footer("a process is already running!") - elif status == "STOPPED" or status == "WAITING": - if process in self.helper_modules: - program=os.path.join( bindir, process ) - fdout, fderr = get_output_channelnames(id) - COMMAND = "%s %s >> %s 2>> %s &" % (program, str(id), fdout, fderr) - os.system(COMMAND) - Log("manually running task #%d (%s)" % (id, process)) - if sleeptime: - next_runtime=get_datetime(sleeptime) - run_sql("INSERT INTO schTASK (proc, user, runtime, sleeptime, arguments, status) "\ - " VALUES (%s,%s,%s,%s,%s,'WAITING')", - (process,self.currentrow[2], next_runtime,sleeptime,self.currentrow[7])) - else: - self.display_in_footer("process status should be STOPPED or WAITING!") - self.stdscr.refresh() - - def sleep(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING': - self.display_in_footer("this process is not running!") - else: - mypid = get_my_pid(process,str(id)) - if mypid!=0: - os.kill(mypid, signal.SIGUSR1) - self.display_in_footer("SLEEP signal sent to process #%s" % mypid) - else: - self.set_status(id,'STOPPED') - self.display_in_footer("cannot find process...") - self.stdscr.refresh() - - def kill(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - mypid = get_my_pid(process,str(id)) - if mypid!=0: - os.kill(mypid, signal.SIGKILL) - self.set_status(id,'STOPPED') - self.display_in_footer("KILL signal sent to process #%s" % mypid) - else: - self.set_status(id,'STOPPED') - self.display_in_footer("cannot find process...") - self.stdscr.refresh() - - def stop(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - mypid = get_my_pid(process, str(id)) - if mypid!=0: - os.kill(mypid, signal.SIGINT) - self.display_in_footer("INT signal sent to process #%s" % mypid) - else: - self.set_status(id,'STOPPED') - self.display_in_footer("cannot find process...") - self.stdscr.refresh() - - def delete(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING' and status!='SLEEPING': - self.set_status(id,"%s_DELETED" % status) - self.display_in_footer("process deleted") - self.selected_line = max(self.selected_line, 2) - else: - self.display_in_footer("cannot delete running processes") - self.stdscr.refresh() - - def init(self): - id = self.currentrow[0] - process = self.currentrow[1] - status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING' and status!='SLEEPING': - self.set_status(id,"WAITING") - self.set_progress(id,"None") - self.display_in_footer("process initialised") - else: - self.display_in_footer("cannot initialise running processes") - self.stdscr.refresh() - - def change_select_mode(self): - if self.move_mode: - self.move_mode = 0 - else: - status = self.currentrow[5] - if status in ( "RUNNING" , "CONTINUING" , "SLEEPING" ): - self.display_in_footer("cannot move running processes!") - else: - self.move_mode = 1 - self.stdscr.refresh() - - def change_auto_mode(self): - if self.auto_mode: - program = os.path.join( bindir, "bibsched") - COMMAND = "%s -q stop" % program - os.system(COMMAND) - - self.auto_mode = 0 - else: - program = os.path.join( bindir, "bibsched") - COMMAND = "%s -q start" % program - os.system(COMMAND) - - self.auto_mode = 1 - self.move_mode = 0 - self.stdscr.refresh() - - def move_up(self): - self.display_in_footer("not implemented yet") - self.stdscr.refresh() - - def move_down(self): - self.display_in_footer("not implemented yet") - self.stdscr.refresh() - - def put_line(self, row): - col_w = [ 5 , 11 , 21 , 21 , 7 , 11 , 25 ] - maxx = self.width - if self.y == self.selected_line - self.first_visible_line and self.y > 1: - if self.auto_mode: - attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD - elif self.move_mode: - attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD - else: - attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD - self.item_status = row[5] - self.currentrow = row - elif self.y == 0: - if self.auto_mode: - attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD - elif self.move_mode: - attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD - else: - attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD - elif row[5] == "DONE": - attr = curses.color_pair(5) + curses.A_BOLD - elif row[5] == "STOPPED": - attr = curses.color_pair(6) + curses.A_BOLD - elif sre.search("ERROR",row[5]): - attr = curses.color_pair(4) + curses.A_BOLD - elif row[5] == "WAITING": - attr = curses.color_pair(3) + curses.A_BOLD - elif row[5] in ("RUNNING","CONTINUING") : - attr = curses.color_pair(2) + curses.A_BOLD - else: - attr = curses.A_BOLD - myline = str(row[0]).ljust(col_w[0]) - myline += str(row[1]).ljust(col_w[1]) - myline += str(row[2]).ljust(col_w[2]) - myline += str(row[3])[:19].ljust(col_w[3]) - myline += str(row[4]).ljust(col_w[4]) - myline += str(row[5]).ljust(col_w[5]) - myline += str(row[6]).ljust(col_w[6]) - myline = myline.ljust(maxx) - self.stdscr.addnstr(self.y, 0, myline, maxx, attr) - self.y = self.y+1 - - def display_in_footer(self, footer, i = 0, print_time_p=0): - if print_time_p: - footer = "%s %s" % (footer, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) - maxx = self.stdscr.getmaxyx()[1] - footer = footer.ljust(maxx) - if self.auto_mode: - colorpair = 2 - elif self.move_mode: - colorpair = 7 - else: - colorpair = 1 - self.stdscr.addnstr(self.y - i, 0, footer, maxx - 1, curses.A_STANDOUT + curses.color_pair(colorpair) + curses.A_BOLD ) - - def repaint(self): - self.y = 0 - self.stdscr.clear() - self.height,self.width = self.stdscr.getmaxyx() - maxy = self.height - 2 - maxx = self.width - self.put_line( ("ID","PROC","USER","RUNTIME","SLEEP","STATUS","PROGRESS") ) - self.put_line( ("---","----","----","-------------------","-----","-----","--------") ) - if self.selected_line > maxy + self.first_visible_line - 1: - self.first_visible_line = self.selected_line - maxy + 1 - if self.selected_line < self.first_visible_line + 2: - self.first_visible_line = self.selected_line - 2 - for row in self.rows[self.first_visible_line:self.first_visible_line+maxy-2]: - id,proc,user,runtime,sleeptime,status,progress,arguments = row - self.put_line( row ) - self.y = self.stdscr.getmaxyx()[0] - 1 - if self.auto_mode: - self.display_in_footer(self.footer_auto_mode, print_time_p=1) - elif self.move_mode: - self.display_in_footer(self.footer_move_mode, print_time_p=1) - else: - self.display_in_footer(self.footer_select_mode, print_time_p=1) - footer2 = "" - if sre.search("DONE",self.item_status) or self.item_status == "ERROR" or self.item_status == "STOPPED": - footer2 += self.footer_stopped_item - elif self.item_status == "RUNNING" or self.item_status == "CONTINUING": - footer2 += self.footer_running_item - elif self.item_status == "SLEEPING": - footer2 += self.footer_sleeping_item - elif self.item_status == "WAITING": - footer2 += self.footer_waiting_item - self.display_in_footer(footer2,1) - self.stdscr.refresh() - - def start(self, stdscr): - ring = 0 - if curses.has_colors(): - curses.start_color() - curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK) - curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_RED) - curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK) - curses.init_pair(3, curses.COLOR_MAGENTA, curses.COLOR_BLACK) - curses.init_pair(4, curses.COLOR_RED, curses.COLOR_BLACK) - curses.init_pair(5, curses.COLOR_BLUE, curses.COLOR_BLACK) - curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK) - curses.init_pair(7, curses.COLOR_YELLOW, curses.COLOR_BLACK) - - self.stdscr = stdscr - self.base_panel = curses.panel.new_panel( self.stdscr ) - self.base_panel.bottom() - curses.panel.update_panels() - self.height,self.width = stdscr.getmaxyx() - self.stdscr.clear() - if server_pid (): self.auto_mode = 1 - if self.display == 1: - where = "and status='DONE'" - order = "DESC" - else: - where = "and status!='DONE'" - order = "ASC" - self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order)) - self.repaint() - ring=0 - while self.running: - ring += 1 - chr = -1 - try: - chr = timed_out(self.stdscr.getch, 1) - if chr == 27: # escaping sequence - chr = self.stdscr.getch() - if chr == 79: # arrow - chr = self.stdscr.getch() - if chr == 65: #arrow up - chr = curses.KEY_UP - elif chr == 66: #arrow down - chr = curses.KEY_DOWN - elif chr == 72: - chr = curses.KEY_PPAGE - elif chr == 70: - chr = curses.KEY_NPAGE - elif chr == 91: - chr = self.stdscr.getch() - if chr == 53: - chr = self.stdscr.getch() - if chr == 126: - chr = curses.KEY_HOME - except TimedOutExc: - chr = -1 - self.handle_keys(chr) - if ring == 4: - if self.display == 1: - where = "and status='DONE'" - order = "DESC" - else: - where = "and status!='DONE'" - order = "ASC" - self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order)) - ring = 0 - self.repaint() - - -class BibSched: - def __init__(self): - self.helper_modules = cfg_valid_processes - self.running = {} - self.sleep_done = {} - self.sleep_sent ={} - self.stop_sent = {} - self.suicide_sent = {} - - def set_status(self, id, status): - return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, id)) - - def can_run( self, proc ): - return len( self.running.keys() ) == 0 - - def get_running_processes(self): - row = None - res = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK "\ - " WHERE status='RUNNING' or status='CONTINUING' LIMIT 1") - try: - row = res[0] - except: - pass - return row - - def handle_row( self, row ): - id,proc,user,runtime,sleeptime,arguments,status = row - if status == "SLEEP": - if id in self.running.keys(): - self.set_status( id, "SLEEP SENT" ) - os.kill( self.running[id], signal.SIGUSR1 ) - self.sleep_sent[id] = self.running[id] - elif status == "SLEEPING": - if id in self.sleep_sent.keys(): - self.sleep_done[id] = self.sleep_sent[id] - del self.sleep_sent[id] - if status == "WAKEUP": - if id in self.sleep_done.keys(): - self.running[id] = self.sleep_done[id] - del self.sleep_done[id] - os.kill( self.running[id], signal.SIGCONT ) - self.set_status( id, "RUNNING" ) - elif status == "STOP": - if id in self.running.keys(): - self.set_status( id, "STOP SENT" ) - os.kill( self.running[id], signal.SIGUSR2 ) - self.stop_sent[id] = self.running[id] - del self.running[id] - elif status == "STOPPED" and id in self.stop_sent.keys(): - del self.stop_sent[id] - elif status == "SUICIDE": - if id in self.running.keys(): - self.set_status( id, "SUICIDE SENT" ) - os.kill( self.running[id], signal.SIGABRT ) - self.suicide_sent[id] = self.running[id] - del self.running[id] - elif status == "SUICIDED" and id in self.suicide_sent.keys(): - del self.suicide_sent[ id ] - elif sre.search("DONE",status) and id in self.running.keys(): - del self.running[id] - elif self.can_run(proc) and status == "WAITING" and runtime <= time.time(): - if proc in self.helper_modules: - program=os.path.join( bindir, proc ) - fdout, fderr = get_output_channelnames(id) - COMMAND = "%s %s >> %s 2>> %s" % (program, str(id), fdout, fderr) - Log("task #%d (%s) started" % (id, proc)) - os.system(COMMAND) - Log("task #%d (%s) ended" % (id, proc)) - self.running[id] = get_my_pid(proc,str(id)) - if sleeptime: - next_runtime=get_datetime(sleeptime) - run_sql("INSERT INTO schTASK (proc, user, runtime, sleeptime, arguments, status) "\ - " VALUES (%s,%s,%s,%s,%s,'WAITING')", - (proc, user, next_runtime, sleeptime, arguments)) - - def watch_loop(self): - running_process = self.get_running_processes() - if running_process: - proc = running_process[ 1 ] - id = running_process[ 0 ] - if get_my_pid(proc,str(id)): - self.running[id] = get_my_pid(proc,str(id)) - else: - self.set_status(id,"ERROR") - rows = [] - while 1: - for row in rows: - self.handle_row( row ) - time.sleep(1) - rows = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK ORDER BY runtime ASC") - -class TimedOutExc(Exception): - def __init__(self, value = "Timed Out"): - self.value = value - def __str__(self): - return repr(self.value) - -def timed_out(f, timeout, *args, **kwargs): - def handler(signum, frame): - raise TimedOutExc() - - old = signal.signal(signal.SIGALRM, handler) - signal.alarm(timeout) - try: - result = f(*args, **kwargs) - finally: - signal.signal(signal.SIGALRM, old) - signal.alarm(0) - return result - - -def Log(message): - log=open(logdir + "/bibsched.log","a") - log.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime())) - log.write(message) - log.write("\n") - log.close() - -def redirect_stdout_and_stderr(): - "This function redirects stdout and stderr to bibsched.log and bibsched.err file." - sys.stdout = open(logdir + "/bibsched.log", "a") - sys.stderr = open(logdir + "/bibsched.err", "a") - -def usage(exitcode=1, msg=""): - """Prints usage info.""" - if msg: - sys.stderr.write("Error: %s.\n" % msg) - - sys.stderr.write ("""\ -Usage: %s [options] [start|stop|restart|monitor] - -The following commands are available for bibsched: - - - start: start bibsched in background - - stop: stop a running bibsched - - restart: restart a running bibsched - - monitor: enter the interactive monitor - -Command options: - -d, --daemon\t Launch BibSched in the daemon mode (deprecated, use 'start') -General options: - -h, --help \t\t Print this help. - -V, --version \t\t Print version information. - """ % sys.argv [0]) - - #sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n") - sys.exit(exitcode) - - -import pwd, grp - -prefix = '@prefix@' -pidfile = os.path.join (prefix, 'var', 'run', 'bibsched.pid') - -def error (msg): - print >> sys.stderr, "error: " + msg - sys.exit (1) - - -def server_pid (): - # The pid must be stored on the filesystem - try: - pid = int (open (pidfile).read ()) - except IOError: - return None - - # Even if the pid is available, we check if it corresponds to an - # actual process, as it might have been killed externally - try: - os.kill (pid, signal.SIGCONT) - except OSError: - return None - - return pid - - -def start (verbose = True): - """ Fork this process in the background and start processing - requests. The process PID is stored in a pid file, so that it can - be stopped later on.""" - - if verbose: - sys.stdout.write ("starting bibsched: ") - sys.stdout.flush () - - pid = server_pid () - if pid: - error ("another instance of bibsched (pid %d) is running" % pid) - - # start the child process using the "double fork" technique - pid = os.fork () - if pid > 0: sys.exit (0) - - os.setsid () - os.chdir ('/') - - pid = os.fork () - - if pid > 0: - if verbose: - sys.stdout.write ('pid %d\n' % pid) - - Log ("daemon started (pid %d)" % pid) - open (pidfile, 'w').write ('%d' % pid) - return - - sys.stdin.close () - redirect_stdout_and_stderr () - - sched = BibSched() - sched.watch_loop () - - return - -def stop (verbose = True): - - pid = server_pid () - if not pid: - error ('bibsched seems not to be running.') - - try: os.kill (pid, signal.SIGKILL) - except OSError: - print >> sys.stderr, 'no bibsched process found' - - Log ("daemon stopped (pid %d)" % pid) - - if verbose: print "stopping bibsched: pid %d" % pid - os.unlink (pidfile) - return - - -def monitor (verbose = True): - redirect_stdout_and_stderr() - manager = Manager () - return - - -def restart (verbose = True): - stop (verbose) - start (verbose) - return - - -def main(): - verbose = True - - try: - opts, args = getopt.getopt(sys.argv[1:], "hVdq", [ - "help","version","daemon", "quiet"]) - except getopt.GetoptError, err: - Log ("Error: %s" % err) - usage(1, err) - - for opt, arg in opts: - if opt in ["-h", "--help"]: - usage (0) - - elif opt in ["-V", "--version"]: - print __revision__ - sys.exit(0) - - elif opt in ["-d", "--daemon"]: - redirect_stdout_and_stderr () - sched = BibSched() - Log("daemon started") - sched.watch_loop() - - elif opt in ['-q', '--quiet']: - verbose = False - - else: - usage(1) - - try: cmd = args [0] - except IndexError: cmd = 'monitor' - - try: - { 'start': start, - 'stop': stop, - 'restart': restart, - 'monitor': monitor } [cmd] (verbose) - - except KeyError: - usage (1, 'unkown command: %s' % `cmd`) - - return - -if __name__ == '__main__': - main() - - +main() diff --git a/modules/bibsched/bin/bibtaskex.in b/modules/bibsched/bin/bibtaskex.in index f3d871756..d0c09b6ee 100644 --- a/modules/bibsched/bin/bibtaskex.in +++ b/modules/bibsched/bin/bibtaskex.in @@ -1,345 +1,38 @@ #!@PYTHON@ ## -*- mode: python; coding: utf-8; -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """CDS Invenio Bibliographic Task Example. Demonstrates BibTask <-> BibSched connectivity, signal handling, error handling, etc. """ __revision__ = "$Id$" -## import interesting modules: try: - import sys - from invenio.dbquery import run_sql - from invenio.access_control_engine import acc_authorize_action - import getopt - import getpass - import marshal - import signal - import sre - import string - import time - import traceback + from invenio.bibtaskex import main except ImportError, e: print "Error: %s" % e + import sys sys.exit(1) -options = {} # global variable to hold task options - -cfg_n_default = 30 # how many Fibonacci numbers to calculate if none submitted? - -def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"): - """Returns a date string according to the format string. - It can handle normal date strings and shifts with respect - to now.""" - date = time.time() - shift_re=sre.compile("([-\+]{0,1})([\d]+)([dhms])") - factors = {"d":24*3600, "h":3600, "m":60, "s":1} - m = shift_re.match(var) - if m: - sign = m.groups()[0] == "-" and -1 or 1 - factor = factors[m.groups()[2]] - value = float(m.groups()[1]) - date = time.localtime(date + sign * factor * value) - date = time.strftime(format_string, date) - else: - date = time.strptime(var, format_string) - date = time.strftime(format_string, date) - return date - -def write_message(msg, stream=sys.stdout): - """Write message and flush output stream (may be sys.stdout or sys.stderr). Useful for debugging stuff.""" - if stream == sys.stdout or stream == sys.stderr: - stream.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime())) - stream.write("%s\n" % msg) - stream.flush() - else: - sys.stderr.write("Unknown stream %s. [must be sys.stdout or sys.stderr]\n" % stream) - return - -def task_sig_sleep(sig, frame): - """Signal handler for the 'sleep' signal sent by BibSched.""" - if options["verbose"] >= 9: - write_message("task_sig_sleep(), got signal %s frame %s" % (sig, frame)) - write_message("sleeping...") - task_update_status("SLEEPING") - signal.pause() # wait for wake-up signal - -def task_sig_wakeup(sig, frame): - """Signal handler for the 'wakeup' signal sent by BibSched.""" - if options["verbose"] >= 9: - write_message("task_sig_wakeup(), got signal %s frame %s" % (sig, frame)) - write_message("continuing...") - task_update_status("CONTINUING") - -def task_sig_stop(sig, frame): - """Signal handler for the 'stop' signal sent by BibSched.""" - if options["verbose"] >= 9: - write_message("task_sig_stop(), got signal %s frame %s" % (sig, frame)) - write_message("stopping...") - task_update_status("STOPPING") - write_message("flushing cache or whatever...") - time.sleep(3) - write_message("closing tables or whatever...") - time.sleep(1) - write_message("stopped") - task_update_status("STOPPED") - sys.exit(0) - -def task_sig_suicide(sig, frame): - """Signal handler for the 'suicide' signal sent by BibSched.""" - if options["verbose"] >= 9: - write_message("task_sig_suicide(), got signal %s frame %s" % (sig, frame)) - write_message("suiciding myself now...") - task_update_status("SUICIDING") - write_message("suicided") - task_update_status("SUICIDED") - sys.exit(0) - -def task_sig_unknown(sig, frame): - """Signal handler for the other unknown signals sent by shell or user.""" - # do nothing for unknown signals: - write_message("unknown signal %d (frame %s) ignored" % (sig, frame)) - -def fib(n): - """Returns Fibonacci number for 'n'.""" - out = 1 - if n >= 2: - out = fib(n-2) + fib(n-1) - return out - -def authenticate(user, header="BibTaskEx Task Submission", action="runbibtaskex"): - """Authenticate the user against the user database. - Check for its password, if it exists. - Check for action access rights. - Return user name upon authorization success, - do system exit upon authorization failure. - """ - print header - print "=" * len(header) - if user == "": - print >> sys.stdout, "\rUsername: ", - user = string.strip(string.lower(sys.stdin.readline())) - else: - print >> sys.stdout, "\rUsername:", user - ## first check user pw: - res = run_sql("select id,password from user where email=%s or nickname=%s", (user, user,), 1) - if not res: - print "Sorry, %s does not exist." % user - sys.exit(1) - else: - (uid_db, password_db) = res[0] - if password_db: - password_entered = getpass.getpass() - if password_db == password_entered: - pass - else: - print "Sorry, wrong credentials for %s." % user - sys.exit(1) - ## secondly check authorization for the action: - (auth_code, auth_message) = acc_authorize_action(uid_db, action) - if auth_code != 0: - print auth_message - sys.exit(1) - return user - -def task_submit(): - """Submits task to the BibSched task queue. This is what people will be invoking via command line.""" - global options - ## sanity check: remove eventual "task" option: - if options.has_key("task"): - del options["task"] - ## authenticate user: - user = authenticate(options.get("user", "")) - ## submit task: - if options["verbose"] >= 9: - print "" - write_message("storing task options %s\n" % options) - task_id = run_sql("""INSERT INTO schTASK (id,proc,user,runtime,sleeptime,status,arguments) - VALUES (NULL,'bibtaskex',%s,%s,%s,'WAITING',%s)""", - (user, options["runtime"], options["sleeptime"], marshal.dumps(options))) - ## update task number: - options["task"] = task_id - run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""", (marshal.dumps(options),task_id)) - write_message("Task #%d submitted." % task_id) - return task_id - -def task_update_progress(msg): - """Updates progress information in the BibSched task table.""" - global options - if options["verbose"] >= 9: - write_message("Updating task progress to %s." % msg) - return run_sql("UPDATE schTASK SET progress=%s where id=%s", (msg, options["task"])) - -def task_update_status(val): - """Updates status information in the BibSched task table.""" - global options - if options["verbose"] >= 9: - write_message("Updating task status to %s." % val) - return run_sql("UPDATE schTASK SET status=%s where id=%s", (val, options["task"])) - -def task_read_status(task_id): - """Read status information in the BibSched task table.""" - res = run_sql("SELECT status FROM schTASK where id=%s", (task_id,), 1) - try: - out = res[0][0] - except: - out = 'UNKNOWN' - return out - -def task_get_options(id): - """Returns options for the task 'id' read from the BibSched task queue table.""" - out = {} - res = run_sql("SELECT arguments FROM schTASK WHERE id=%s AND proc='bibtaskex'", (id,)) - try: - out = marshal.loads(res[0][0]) - except: - write_message("Error: BibTaskEx task %d does not seem to exist." % id, sys.stderr) - sys.exit(1) - return out - -def task_run(task_id): - """Runs the task by fetching arguments from the BibSched task queue. This is what BibSched will be invoking via daemon call. - The task prints Fibonacci numbers for up to NUM on the stdout, and some messages on stderr. - Return 1 in case of success and 0 in case of failure.""" - global options - options = task_get_options(task_id) # get options from BibSched task table - ## check task id: - if not options.has_key("task"): - write_message("Error: The task #%d does not seem to be a BibTaskEx task." % task_id, sys.stderr) - return 0 - ## check task status: - task_status = task_read_status(task_id) - if task_status != "WAITING": - write_message("Error: The task #%d is %s. I expected WAITING." % (task_id, task_status), sys.stderr) - return 0 - ## we can run the task now: - if options["verbose"]: - write_message("Task #%d started." % task_id) - task_update_status("RUNNING") - ## initialize signal handler: - signal.signal(signal.SIGUSR1, task_sig_sleep) - signal.signal(signal.SIGTERM, task_sig_stop) - signal.signal(signal.SIGABRT, task_sig_suicide) - signal.signal(signal.SIGCONT, task_sig_wakeup) - signal.signal(signal.SIGINT, task_sig_unknown) - ## run the task: - if options.has_key("number"): - n = options["number"] - else: - n = cfg_n_default - if options["verbose"] >= 9: - write_message("Printing %d Fibonacci numbers." % n) - for i in range(0,n): - if i > 0 and i % 4 == 0: - if options["verbose"] >= 3: - write_message("Error: water in the CPU. Ignoring and continuing.", sys.stderr) - elif i > 0 and i % 5 == 0: - if options["verbose"]: - write_message("Error: floppy drive dropped on the floor. Ignoring and continuing.", sys.stderr) - if options["verbose"]: - write_message("fib(%d)=%d" % (i, fib(i))) - task_update_progress("Done %d out of %d." % (i,n)) - time.sleep(1) - ## we are done: - task_update_progress("Done %d out of %d." % (n,n)) - task_update_status("DONE") - if options["verbose"]: - write_message("Task #%d finished." % task_id) - return 1 - -def usage(exitcode=1, msg=""): - """Prints usage info.""" - if msg: - sys.stderr.write("Error: %s.\n" % msg) - sys.stderr.write("Usage: %s [options]\n" % sys.argv[0]) - sys.stderr.write("Command options:\n") - sys.stderr.write(" -n, --number=NUM\t Print Fibonacci numbers for up to NUM. [default=%d]\n" % cfg_n_default) - sys.stderr.write("Scheduling options:\n") - sys.stderr.write(" -u, --user=USER \t User name to submit the task as, password needed.\n") - sys.stderr.write(" -t, --runtime=TIME \t Time to execute the task (now), e.g.: +15s, 5m, 3h, 2002-10-27 13:57:26\n") - sys.stderr.write(" -s, --sleeptime=SLEEP \t Sleeping frequency after which to repeat task (no), e.g.: 30m, 2h, 1d\n") - sys.stderr.write("General options:\n") - sys.stderr.write(" -h, --help \t\t Print this help.\n") - sys.stderr.write(" -V, --version \t\t Print version information.\n") - sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n") - sys.exit(exitcode) - -def main(): - """Main function that analyzes command line input and calls whatever is appropriate. - Useful for learning on how to write BibSched tasks.""" - global options - ## parse command line: - if len(sys.argv) == 2 and sys.argv[1].isdigit(): - ## A - run the task - task_id = int(sys.argv[1]) - try: - if not task_run(task_id): - write_message("Error occurred. Exiting.", sys.stderr) - except StandardError, e: - write_message("Unexpected error occurred: %s." % e, sys.stderr) - write_message("Traceback is:", sys.stderr) - traceback.print_tb(sys.exc_info()[2]) - write_message("Exiting.", sys.stderr) - task_update_status("ERROR") - else: - ## B - submit the task - # set default values: - options = {} - options["runtime"] = time.strftime("%Y-%m-%d %H:%M:%S") - options["sleeptime"] = "" - options["verbose"] = 1 - # set user-defined options: - try: - opts, args = getopt.getopt(sys.argv[1:], "hVv:n:u:s:t:", ["help", "version", "verbose=","number=","user=","sleep=","time="]) - except getopt.GetoptError, err: - usage(1, err) - try: - for opt in opts: - if opt[0] in ["-h", "--help"]: - usage(0) - elif opt[0] in ["-V", "--version"]: - print __revision__ - sys.exit(0) - elif opt[0] in [ "-u", "--user"]: - options["user"] = opt[1] - elif opt[0] in ["-v", "--verbose"]: - options["verbose"] = int(opt[1]) - elif opt[0] in ["-n", "--number"]: - options["number"]=int(opt[1]) - elif opt[0] in [ "-s", "--sleeptime" ]: - get_datetime(opt[1]) # see if it is a valid shift - options["sleeptime"] = opt[1] - elif opt[0] in [ "-t", "--runtime" ]: - options["runtime"] = get_datetime(opt[1]) - else: - usage(1) - except StandardError, e: - usage(e) - task_submit() - return - -### okay, here we go: -if __name__ == '__main__': - main() +main() diff --git a/modules/bibsched/lib/.cvsignore b/modules/bibsched/lib/.cvsignore new file mode 100644 index 000000000..9638520ce --- /dev/null +++ b/modules/bibsched/lib/.cvsignore @@ -0,0 +1,6 @@ +Makefile +Makefile.in +z_* +*.O +*~ +*.pyc \ No newline at end of file diff --git a/modules/bibsched/Makefile.am b/modules/bibsched/lib/Makefile.am similarity index 85% copy from modules/bibsched/Makefile.am copy to modules/bibsched/lib/Makefile.am index dde3dc9ba..a9a5bcf53 100644 --- a/modules/bibsched/Makefile.am +++ b/modules/bibsched/lib/Makefile.am @@ -1,22 +1,28 @@ ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -SUBDIRS = bin doc +pylibdir = $(libdir)/python/invenio -CLEANFILES = *~ \ No newline at end of file +pylib_DATA = \ + bibsched.py \ + bibtaskex.py + +EXTRA_DIST = $(pylib_DATA) + +CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/bibsched/bin/bibsched.in b/modules/bibsched/lib/bibsched.py similarity index 73% copy from modules/bibsched/bin/bibsched.in copy to modules/bibsched/lib/bibsched.py index 8e95b9747..c72720a70 100644 --- a/modules/bibsched/bin/bibsched.in +++ b/modules/bibsched/lib/bibsched.py @@ -1,798 +1,789 @@ -#!@PYTHON@ -## -*- mode: python; coding: utf-8; -*- +# -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """BibSched - task management, scheduling and executing system for CDS Invenio """ __revision__ = "$Id$" ### -- local configuration section starts here --- -cfg_valid_processes = ["bibindex","bibupload","bibreformat","webcoll","bibtaskex","bibrank","oaiharvest","oaiarchive"] # which tasks are reconized as valid? + +# which tasks are reconized as valid? +cfg_valid_processes = ["bibindex", "bibupload", "bibreformat", + "webcoll", "bibtaskex", "bibrank", + "oaiharvest", "oaiarchive"] + ### -- local configuration section ends here --- -## import interesting modules: -try: - import os - import imp - import string - import sys - import time - import sre - import getopt - import curses - import curses.panel - from curses.wrapper import wrapper - import signal -except ImportError, e: - print "Error: %s" % e - import sys - sys.exit(1) - -try: - from invenio.config import * - from invenio.dbquery import run_sql -except ImportError, e: - print "Error: %s" % e - import sys - sys.exit(1) +import os +import string +import sys +import time +import sre +import getopt +import curses +import curses.panel +from curses.wrapper import wrapper +import signal + +from invenio.config import \ + CFG_PREFIX, \ + bindir, \ + logdir +from invenio.dbquery import run_sql def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"): """Returns a date string according to the format string. It can handle normal date strings and shifts with respect to now.""" try: date = time.time() - shift_re=sre.compile("([-\+]{0,1})([\d]+)([dhms])") + shift_re = sre.compile("([-\+]{0,1})([\d]+)([dhms])") factors = {"d":24*3600, "h":3600, "m":60, "s":1} m = shift_re.match(var) if m: sign = m.groups()[0] == "-" and -1 or 1 factor = factors[m.groups()[2]] value = float(m.groups()[1]) date = time.localtime(date + sign * factor * value) date = time.strftime(format_string, date) else: date = time.strptime(var, format_string) date = time.strftime(format_string, date) return date except: return None -def get_my_pid(process,args=''): +def get_my_pid(process, args=''): if sys.platform.startswith('freebsd'): COMMAND = "ps -o pid,args | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, args) else: COMMAND = "ps -C %s o '%%p%%a' | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, process, args) answer = string.strip(os.popen(COMMAND).read()) - if answer=='': + if answer == '': answer = 0 else: answer = answer[:string.find(answer,' ')] return int(answer) def get_output_channelnames(task_id): "Construct and return filename for stdout and stderr for the task 'task_id'." filenamebase = "%s/bibsched_task_%d" % (logdir, task_id) return [filenamebase + ".log", filenamebase + ".err"] class Manager: def __init__(self): self.helper_modules = cfg_valid_processes self.running = 1 self.footer_move_mode = "[KeyUp/KeyDown Move] [M Select mode] [Q Quit]" self.footer_auto_mode = "[A Manual mode] [1/2 Display Type] [Q Quit]" self.footer_select_mode = "[KeyUp/KeyDown/PgUp/PgDown Select] [L View Log] [1/2 Display Type] [M Move mode] [A Auto mode] [Q Quit]" self.footer_waiting_item = "[R Run] [D Delete]" self.footer_running_item = "[S Sleep] [T Stop] [K Kill]" self.footer_stopped_item = "[I Initialise] [D Delete]" self.footer_sleeping_item = "[W Wake Up]" self.item_status = "" self.selected_line = 2 self.rows = [] self.panel = None self.display = 2 self.first_visible_line = 0 self.move_mode = 0 self.auto_mode = 0 - self.currentrow = ["","","","","","",""] - wrapper( self.start ) + self.currentrow = ["", "", "", "", "", "", ""] + wrapper(self.start) def handle_keys(self, chr): if chr == -1: return - if self.auto_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,curses.KEY_PPAGE,curses.KEY_NPAGE,ord("q"),ord("Q"),ord("a"),ord("A"),ord("1"),ord("2"))): + if self.auto_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN, + curses.KEY_PPAGE, curses.KEY_NPAGE, + ord("q"), ord("Q"), ord("a"), + ord("A"), ord("1"), ord("2"))): self.display_in_footer("in automatic mode") self.stdscr.refresh() - elif self.move_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,ord("m"),ord("M"),ord("q"),ord("Q"))): + elif self.move_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN, + ord("m"), ord("M"), ord("q"), + ord("Q"))): self.display_in_footer("in move mode") self.stdscr.refresh() else: if chr == curses.KEY_UP: if self.move_mode: self.move_up() else: - self.selected_line = max( self.selected_line - 1 , 2 ) + self.selected_line = max(self.selected_line - 1, 2) self.repaint() if chr == curses.KEY_PPAGE: - self.selected_line = max( self.selected_line - 10 , 2 ) + self.selected_line = max(self.selected_line - 10, 2) self.repaint() elif chr == curses.KEY_DOWN: if self.move_mode: self.move_down() else: self.selected_line = min(self.selected_line + 1, len(self.rows) + 1 ) self.repaint() elif chr == curses.KEY_NPAGE: self.selected_line = min(self.selected_line + 10, len(self.rows) + 1 ) self.repaint() elif chr == curses.KEY_HOME: self.first_visible_line = 0 self.selected_line = 2 elif chr in (ord("a"), ord("A")): self.change_auto_mode() elif chr in (ord("l"), ord("L")): self.openlog() elif chr in (ord("w"), ord("W")): self.wakeup() elif chr in (ord("r"), ord("R")): self.run() elif chr in (ord("s"), ord("S")): self.sleep() elif chr in (ord("k"), ord("K")): self.kill() elif chr in (ord("t"), ord("T")): self.stop() elif chr in (ord("d"), ord("D")): self.delete() elif chr in (ord("i"), ord("I")): self.init() elif chr in (ord("m"), ord("M")): self.change_select_mode() elif chr == ord("1"): self.display = 1 self.first_visible_line = 0 self.selected_line = 2 self.display_in_footer("only done processes are displayed") elif chr == ord("2"): self.display = 2 self.first_visible_line = 0 self.selected_line = 2 self.display_in_footer("only not done processes are displayed") elif chr in (ord("q"), ord("Q")): if curses.panel.top_panel() == self.panel: self.panel.bottom() curses.panel.update_panels() else: self.running = 0 return - def set_status(self, id, status): - return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, id)) + def set_status(self, task_id, status): + return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id)) - def set_progress(self, id, progress): - return run_sql("UPDATE schTASK set progress=%s WHERE id=%s", (progress, id)) + def set_progress(self, task_id, progress): + return run_sql("UPDATE schTASK set progress=%s WHERE id=%s", (progress, task_id)) def openlog(self): self.win = curses.newwin( self.height-2, self.width-2, 1, 1 ) self.panel = curses.panel.new_panel( self.win ) self.panel.top() self.win.border() self.win.addstr(1, 1, "Not implemented yet...") self.win.refresh() curses.panel.update_panels() - def count_processes(self,status): + def count_processes(self, status): out = 0 res = run_sql("SELECT COUNT(id) FROM schTASK WHERE status=%s GROUP BY status", (status,)) try: out = res[0][0] except: pass return out def wakeup(self): - id = self.currentrow[0] + task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: self.display_in_footer("a process is already running!") elif status == "SLEEPING": - mypid = get_my_pid(process,str(id)) - if mypid!=0: + mypid = get_my_pid(process, str(task_id)) + if mypid != 0: os.kill(mypid, signal.SIGCONT) self.display_in_footer("process woken up") else: self.display_in_footer("process is not sleeping") self.stdscr.refresh() def run(self): - id = self.currentrow[0] + task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] sleeptime = self.currentrow[4] if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1: self.display_in_footer("a process is already running!") elif status == "STOPPED" or status == "WAITING": if process in self.helper_modules: - program=os.path.join( bindir, process ) - fdout, fderr = get_output_channelnames(id) - COMMAND = "%s %s >> %s 2>> %s &" % (program, str(id), fdout, fderr) + program = os.path.join(bindir, process) + fdout, fderr = get_output_channelnames(task_id) + COMMAND = "%s %s >> %s 2>> %s &" % (program, str(task_id), fdout, fderr) os.system(COMMAND) - Log("manually running task #%d (%s)" % (id, process)) + Log("manually running task #%d (%s)" % (task_id, process)) if sleeptime: - next_runtime=get_datetime(sleeptime) + next_runtime = get_datetime(sleeptime) run_sql("INSERT INTO schTASK (proc, user, runtime, sleeptime, arguments, status) "\ " VALUES (%s,%s,%s,%s,%s,'WAITING')", - (process,self.currentrow[2], next_runtime,sleeptime,self.currentrow[7])) + (process, self.currentrow[2], next_runtime,sleeptime, self.currentrow[7])) else: self.display_in_footer("process status should be STOPPED or WAITING!") self.stdscr.refresh() def sleep(self): - id = self.currentrow[0] + task_id = self.currentrow[0] process = self.currentrow[1] status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING': + if status != 'RUNNING' and status != 'CONTINUING': self.display_in_footer("this process is not running!") else: - mypid = get_my_pid(process,str(id)) - if mypid!=0: + mypid = get_my_pid(process, str(task_id)) + if mypid != 0: os.kill(mypid, signal.SIGUSR1) self.display_in_footer("SLEEP signal sent to process #%s" % mypid) else: - self.set_status(id,'STOPPED') + self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def kill(self): - id = self.currentrow[0] + task_id = self.currentrow[0] process = self.currentrow[1] - status = self.currentrow[5] - mypid = get_my_pid(process,str(id)) - if mypid!=0: + #status = self.currentrow[5] + mypid = get_my_pid(process, str(task_id)) + if mypid != 0: os.kill(mypid, signal.SIGKILL) - self.set_status(id,'STOPPED') + self.set_status(task_id, 'STOPPED') self.display_in_footer("KILL signal sent to process #%s" % mypid) else: - self.set_status(id,'STOPPED') + self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def stop(self): - id = self.currentrow[0] + task_id = self.currentrow[0] process = self.currentrow[1] - status = self.currentrow[5] - mypid = get_my_pid(process, str(id)) - if mypid!=0: + #status = self.currentrow[5] + mypid = get_my_pid(process, str(task_id)) + if mypid != 0: os.kill(mypid, signal.SIGINT) self.display_in_footer("INT signal sent to process #%s" % mypid) else: - self.set_status(id,'STOPPED') + self.set_status(task_id, 'STOPPED') self.display_in_footer("cannot find process...") self.stdscr.refresh() def delete(self): - id = self.currentrow[0] - process = self.currentrow[1] + task_id = self.currentrow[0] + #process = self.currentrow[1] status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING' and status!='SLEEPING': - self.set_status(id,"%s_DELETED" % status) + if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING': + self.set_status(task_id, "%s_DELETED" % status) self.display_in_footer("process deleted") self.selected_line = max(self.selected_line, 2) else: self.display_in_footer("cannot delete running processes") self.stdscr.refresh() def init(self): - id = self.currentrow[0] - process = self.currentrow[1] + task_id = self.currentrow[0] + #process = self.currentrow[1] status = self.currentrow[5] - if status!='RUNNING' and status!='CONTINUING' and status!='SLEEPING': - self.set_status(id,"WAITING") - self.set_progress(id,"None") + if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING': + self.set_status(task_id, "WAITING") + self.set_progress(task_id, "None") self.display_in_footer("process initialised") else: self.display_in_footer("cannot initialise running processes") self.stdscr.refresh() def change_select_mode(self): if self.move_mode: self.move_mode = 0 else: status = self.currentrow[5] if status in ( "RUNNING" , "CONTINUING" , "SLEEPING" ): self.display_in_footer("cannot move running processes!") else: self.move_mode = 1 self.stdscr.refresh() def change_auto_mode(self): if self.auto_mode: - program = os.path.join( bindir, "bibsched") + program = os.path.join(bindir, "bibsched") COMMAND = "%s -q stop" % program os.system(COMMAND) self.auto_mode = 0 else: program = os.path.join( bindir, "bibsched") COMMAND = "%s -q start" % program os.system(COMMAND) self.auto_mode = 1 self.move_mode = 0 self.stdscr.refresh() def move_up(self): self.display_in_footer("not implemented yet") self.stdscr.refresh() def move_down(self): self.display_in_footer("not implemented yet") self.stdscr.refresh() def put_line(self, row): col_w = [ 5 , 11 , 21 , 21 , 7 , 11 , 25 ] maxx = self.width if self.y == self.selected_line - self.first_visible_line and self.y > 1: if self.auto_mode: attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD elif self.move_mode: attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD else: attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD self.item_status = row[5] self.currentrow = row elif self.y == 0: if self.auto_mode: attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD elif self.move_mode: attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD else: attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD elif row[5] == "DONE": attr = curses.color_pair(5) + curses.A_BOLD elif row[5] == "STOPPED": attr = curses.color_pair(6) + curses.A_BOLD elif sre.search("ERROR",row[5]): attr = curses.color_pair(4) + curses.A_BOLD elif row[5] == "WAITING": attr = curses.color_pair(3) + curses.A_BOLD elif row[5] in ("RUNNING","CONTINUING") : attr = curses.color_pair(2) + curses.A_BOLD else: attr = curses.A_BOLD myline = str(row[0]).ljust(col_w[0]) myline += str(row[1]).ljust(col_w[1]) myline += str(row[2]).ljust(col_w[2]) myline += str(row[3])[:19].ljust(col_w[3]) myline += str(row[4]).ljust(col_w[4]) myline += str(row[5]).ljust(col_w[5]) myline += str(row[6]).ljust(col_w[6]) myline = myline.ljust(maxx) self.stdscr.addnstr(self.y, 0, myline, maxx, attr) self.y = self.y+1 def display_in_footer(self, footer, i = 0, print_time_p=0): if print_time_p: footer = "%s %s" % (footer, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) maxx = self.stdscr.getmaxyx()[1] footer = footer.ljust(maxx) if self.auto_mode: colorpair = 2 elif self.move_mode: colorpair = 7 else: colorpair = 1 self.stdscr.addnstr(self.y - i, 0, footer, maxx - 1, curses.A_STANDOUT + curses.color_pair(colorpair) + curses.A_BOLD ) def repaint(self): self.y = 0 self.stdscr.clear() - self.height,self.width = self.stdscr.getmaxyx() + self.height, self.width = self.stdscr.getmaxyx() maxy = self.height - 2 - maxx = self.width - self.put_line( ("ID","PROC","USER","RUNTIME","SLEEP","STATUS","PROGRESS") ) - self.put_line( ("---","----","----","-------------------","-----","-----","--------") ) + #maxx = self.width + self.put_line( ("ID", "PROC", "USER", "RUNTIME", "SLEEP", "STATUS", "PROGRESS") ) + self.put_line( ("---", "----", "----", "-------------------", "-----", "-----", "--------") ) if self.selected_line > maxy + self.first_visible_line - 1: self.first_visible_line = self.selected_line - maxy + 1 if self.selected_line < self.first_visible_line + 2: self.first_visible_line = self.selected_line - 2 for row in self.rows[self.first_visible_line:self.first_visible_line+maxy-2]: - id,proc,user,runtime,sleeptime,status,progress,arguments = row + task_id, proc, user, runtime, sleeptime, status, progress, arguments = row self.put_line( row ) self.y = self.stdscr.getmaxyx()[0] - 1 if self.auto_mode: self.display_in_footer(self.footer_auto_mode, print_time_p=1) elif self.move_mode: self.display_in_footer(self.footer_move_mode, print_time_p=1) else: self.display_in_footer(self.footer_select_mode, print_time_p=1) footer2 = "" - if sre.search("DONE",self.item_status) or self.item_status == "ERROR" or self.item_status == "STOPPED": - footer2 += self.footer_stopped_item + if sre.search("DONE", self.item_status) or self.item_status == "ERROR" or self.item_status == "STOPPED": + footer2 += self.footer_stopped_item elif self.item_status == "RUNNING" or self.item_status == "CONTINUING": - footer2 += self.footer_running_item + footer2 += self.footer_running_item elif self.item_status == "SLEEPING": - footer2 += self.footer_sleeping_item + footer2 += self.footer_sleeping_item elif self.item_status == "WAITING": - footer2 += self.footer_waiting_item - self.display_in_footer(footer2,1) + footer2 += self.footer_waiting_item + self.display_in_footer(footer2, 1) self.stdscr.refresh() def start(self, stdscr): ring = 0 if curses.has_colors(): curses.start_color() curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK) curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_RED) curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK) curses.init_pair(3, curses.COLOR_MAGENTA, curses.COLOR_BLACK) curses.init_pair(4, curses.COLOR_RED, curses.COLOR_BLACK) curses.init_pair(5, curses.COLOR_BLUE, curses.COLOR_BLACK) curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK) curses.init_pair(7, curses.COLOR_YELLOW, curses.COLOR_BLACK) self.stdscr = stdscr self.base_panel = curses.panel.new_panel( self.stdscr ) self.base_panel.bottom() curses.panel.update_panels() - self.height,self.width = stdscr.getmaxyx() + self.height, self.width = stdscr.getmaxyx() self.stdscr.clear() if server_pid (): self.auto_mode = 1 if self.display == 1: where = "and status='DONE'" order = "DESC" else: where = "and status!='DONE'" order = "ASC" - self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order)) + self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where, order)) self.repaint() - ring=0 + ring = 0 while self.running: ring += 1 - chr = -1 + char = -1 try: - chr = timed_out(self.stdscr.getch, 1) - if chr == 27: # escaping sequence - chr = self.stdscr.getch() - if chr == 79: # arrow - chr = self.stdscr.getch() - if chr == 65: #arrow up - chr = curses.KEY_UP - elif chr == 66: #arrow down - chr = curses.KEY_DOWN - elif chr == 72: - chr = curses.KEY_PPAGE - elif chr == 70: - chr = curses.KEY_NPAGE - elif chr == 91: - chr = self.stdscr.getch() - if chr == 53: - chr = self.stdscr.getch() - if chr == 126: - chr = curses.KEY_HOME + char = timed_out(self.stdscr.getch, 1) + if char == 27: # escaping sequence + char = self.stdscr.getch() + if char == 79: # arrow + char = self.stdscr.getch() + if char == 65: #arrow up + char = curses.KEY_UP + elif char == 66: #arrow down + char = curses.KEY_DOWN + elif char == 72: + char = curses.KEY_PPAGE + elif char == 70: + char = curses.KEY_NPAGE + elif char == 91: + char = self.stdscr.getch() + if char == 53: + char = self.stdscr.getch() + if char == 126: + char = curses.KEY_HOME except TimedOutExc: - chr = -1 - self.handle_keys(chr) + char = -1 + self.handle_keys(char) if ring == 4: if self.display == 1: where = "and status='DONE'" order = "DESC" else: where = "and status!='DONE'" order = "ASC" - self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order)) + self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where, order)) ring = 0 self.repaint() class BibSched: def __init__(self): self.helper_modules = cfg_valid_processes self.running = {} self.sleep_done = {} - self.sleep_sent ={} + self.sleep_sent = {} self.stop_sent = {} self.suicide_sent = {} - def set_status(self, id, status): - return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, id)) + def set_status(self, task_id, status): + return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id)) def can_run( self, proc ): return len( self.running.keys() ) == 0 def get_running_processes(self): row = None res = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK "\ " WHERE status='RUNNING' or status='CONTINUING' LIMIT 1") try: row = res[0] except: pass return row def handle_row( self, row ): - id,proc,user,runtime,sleeptime,arguments,status = row + task_id, proc, user, runtime, sleeptime, arguments, status = row if status == "SLEEP": - if id in self.running.keys(): - self.set_status( id, "SLEEP SENT" ) - os.kill( self.running[id], signal.SIGUSR1 ) - self.sleep_sent[id] = self.running[id] + if task_id in self.running.keys(): + self.set_status( task_id, "SLEEP SENT" ) + os.kill( self.running[task_id], signal.SIGUSR1 ) + self.sleep_sent[task_id] = self.running[task_id] elif status == "SLEEPING": - if id in self.sleep_sent.keys(): - self.sleep_done[id] = self.sleep_sent[id] - del self.sleep_sent[id] + if task_id in self.sleep_sent.keys(): + self.sleep_done[task_id] = self.sleep_sent[task_id] + del self.sleep_sent[task_id] if status == "WAKEUP": - if id in self.sleep_done.keys(): - self.running[id] = self.sleep_done[id] - del self.sleep_done[id] - os.kill( self.running[id], signal.SIGCONT ) - self.set_status( id, "RUNNING" ) + if task_id in self.sleep_done.keys(): + self.running[task_id] = self.sleep_done[task_id] + del self.sleep_done[task_id] + os.kill( self.running[task_id], signal.SIGCONT ) + self.set_status( task_id, "RUNNING" ) elif status == "STOP": - if id in self.running.keys(): - self.set_status( id, "STOP SENT" ) - os.kill( self.running[id], signal.SIGUSR2 ) - self.stop_sent[id] = self.running[id] - del self.running[id] - elif status == "STOPPED" and id in self.stop_sent.keys(): - del self.stop_sent[id] + if task_id in self.running.keys(): + self.set_status( task_id, "STOP SENT" ) + os.kill( self.running[task_id], signal.SIGUSR2 ) + self.stop_sent[task_id] = self.running[task_id] + del self.running[task_id] + elif status == "STOPPED" and task_id in self.stop_sent.keys(): + del self.stop_sent[task_id] elif status == "SUICIDE": - if id in self.running.keys(): - self.set_status( id, "SUICIDE SENT" ) - os.kill( self.running[id], signal.SIGABRT ) - self.suicide_sent[id] = self.running[id] - del self.running[id] - elif status == "SUICIDED" and id in self.suicide_sent.keys(): - del self.suicide_sent[ id ] - elif sre.search("DONE",status) and id in self.running.keys(): - del self.running[id] + if task_id in self.running.keys(): + self.set_status( task_id, "SUICIDE SENT" ) + os.kill( self.running[task_id], signal.SIGABRT ) + self.suicide_sent[task_id] = self.running[task_id] + del self.running[task_id] + elif status == "SUICIDED" and task_id in self.suicide_sent.keys(): + del self.suicide_sent[task_id] + elif sre.search("DONE", status) and task_id in self.running.keys(): + del self.running[task_id] elif self.can_run(proc) and status == "WAITING" and runtime <= time.time(): if proc in self.helper_modules: - program=os.path.join( bindir, proc ) - fdout, fderr = get_output_channelnames(id) - COMMAND = "%s %s >> %s 2>> %s" % (program, str(id), fdout, fderr) - Log("task #%d (%s) started" % (id, proc)) + program = os.path.join(bindir, proc) + fdout, fderr = get_output_channelnames(task_id) + COMMAND = "%s %s >> %s 2>> %s" % (program, str(task_id), fdout, fderr) + Log("task #%d (%s) started" % (task_id, proc)) os.system(COMMAND) - Log("task #%d (%s) ended" % (id, proc)) - self.running[id] = get_my_pid(proc,str(id)) + Log("task #%d (%s) ended" % (task_id, proc)) + self.running[task_id] = get_my_pid(proc, str(task_id)) if sleeptime: - next_runtime=get_datetime(sleeptime) + next_runtime = get_datetime(sleeptime) run_sql("INSERT INTO schTASK (proc, user, runtime, sleeptime, arguments, status) "\ " VALUES (%s,%s,%s,%s,%s,'WAITING')", (proc, user, next_runtime, sleeptime, arguments)) def watch_loop(self): running_process = self.get_running_processes() if running_process: proc = running_process[ 1 ] - id = running_process[ 0 ] - if get_my_pid(proc,str(id)): - self.running[id] = get_my_pid(proc,str(id)) + task_id = running_process[ 0 ] + if get_my_pid(proc, str(task_id)): + self.running[task_id] = get_my_pid(proc, str(task_id)) else: - self.set_status(id,"ERROR") + self.set_status(task_id,"ERROR") rows = [] while 1: for row in rows: self.handle_row( row ) time.sleep(1) rows = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK ORDER BY runtime ASC") class TimedOutExc(Exception): def __init__(self, value = "Timed Out"): self.value = value def __str__(self): return repr(self.value) def timed_out(f, timeout, *args, **kwargs): def handler(signum, frame): raise TimedOutExc() old = signal.signal(signal.SIGALRM, handler) signal.alarm(timeout) try: result = f(*args, **kwargs) finally: signal.signal(signal.SIGALRM, old) signal.alarm(0) return result def Log(message): - log=open(logdir + "/bibsched.log","a") + log = open(logdir + "/bibsched.log","a") log.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime())) log.write(message) log.write("\n") log.close() def redirect_stdout_and_stderr(): "This function redirects stdout and stderr to bibsched.log and bibsched.err file." sys.stdout = open(logdir + "/bibsched.log", "a") sys.stderr = open(logdir + "/bibsched.err", "a") def usage(exitcode=1, msg=""): """Prints usage info.""" if msg: sys.stderr.write("Error: %s.\n" % msg) sys.stderr.write ("""\ Usage: %s [options] [start|stop|restart|monitor] The following commands are available for bibsched: - start: start bibsched in background - stop: stop a running bibsched - restart: restart a running bibsched - monitor: enter the interactive monitor Command options: -d, --daemon\t Launch BibSched in the daemon mode (deprecated, use 'start') General options: -h, --help \t\t Print this help. -V, --version \t\t Print version information. """ % sys.argv [0]) #sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n") sys.exit(exitcode) - -import pwd, grp - -prefix = '@prefix@' -pidfile = os.path.join (prefix, 'var', 'run', 'bibsched.pid') +pidfile = os.path.join(CFG_PREFIX, 'var', 'run', 'bibsched.pid') def error (msg): print >> sys.stderr, "error: " + msg sys.exit (1) def server_pid (): # The pid must be stored on the filesystem try: pid = int (open (pidfile).read ()) except IOError: return None # Even if the pid is available, we check if it corresponds to an # actual process, as it might have been killed externally try: os.kill (pid, signal.SIGCONT) except OSError: return None return pid def start (verbose = True): """ Fork this process in the background and start processing requests. The process PID is stored in a pid file, so that it can be stopped later on.""" if verbose: sys.stdout.write ("starting bibsched: ") sys.stdout.flush () pid = server_pid () if pid: error ("another instance of bibsched (pid %d) is running" % pid) # start the child process using the "double fork" technique pid = os.fork () if pid > 0: sys.exit (0) os.setsid () os.chdir ('/') pid = os.fork () if pid > 0: if verbose: sys.stdout.write ('pid %d\n' % pid) Log ("daemon started (pid %d)" % pid) open (pidfile, 'w').write ('%d' % pid) return sys.stdin.close () redirect_stdout_and_stderr () sched = BibSched() sched.watch_loop () return def stop (verbose = True): pid = server_pid () if not pid: error ('bibsched seems not to be running.') try: os.kill (pid, signal.SIGKILL) except OSError: print >> sys.stderr, 'no bibsched process found' Log ("daemon stopped (pid %d)" % pid) if verbose: print "stopping bibsched: pid %d" % pid os.unlink (pidfile) return - -def monitor (verbose = True): +def monitor(verbose = True): redirect_stdout_and_stderr() - manager = Manager () + manager = Manager() return - -def restart (verbose = True): - stop (verbose) - start (verbose) +def restart(verbose = True): + stop(verbose) + start(verbose) return - def main(): verbose = True try: opts, args = getopt.getopt(sys.argv[1:], "hVdq", [ "help","version","daemon", "quiet"]) except getopt.GetoptError, err: Log ("Error: %s" % err) usage(1, err) for opt, arg in opts: if opt in ["-h", "--help"]: usage (0) elif opt in ["-V", "--version"]: print __revision__ sys.exit(0) elif opt in ["-d", "--daemon"]: redirect_stdout_and_stderr () sched = BibSched() Log("daemon started") sched.watch_loop() elif opt in ['-q', '--quiet']: verbose = False else: usage(1) try: cmd = args [0] except IndexError: cmd = 'monitor' try: { 'start': start, 'stop': stop, 'restart': restart, 'monitor': monitor } [cmd] (verbose) except KeyError: usage (1, 'unkown command: %s' % `cmd`) return if __name__ == '__main__': main() - - diff --git a/modules/bibsched/bin/bibtaskex.in b/modules/bibsched/lib/bibtaskex.py similarity index 94% copy from modules/bibsched/bin/bibtaskex.in copy to modules/bibsched/lib/bibtaskex.py index f3d871756..340b75fff 100644 --- a/modules/bibsched/bin/bibtaskex.in +++ b/modules/bibsched/lib/bibtaskex.py @@ -1,345 +1,339 @@ -#!@PYTHON@ -## -*- mode: python; coding: utf-8; -*- +# -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """CDS Invenio Bibliographic Task Example. Demonstrates BibTask <-> BibSched connectivity, signal handling, error handling, etc. """ __revision__ = "$Id$" -## import interesting modules: -try: - import sys - from invenio.dbquery import run_sql - from invenio.access_control_engine import acc_authorize_action - import getopt - import getpass - import marshal - import signal - import sre - import string - import time - import traceback -except ImportError, e: - print "Error: %s" % e - sys.exit(1) +import sys +from invenio.dbquery import run_sql +from invenio.access_control_engine import acc_authorize_action +import getopt +import getpass +import marshal +import signal +import sre +import string +import time +import traceback options = {} # global variable to hold task options cfg_n_default = 30 # how many Fibonacci numbers to calculate if none submitted? def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"): """Returns a date string according to the format string. It can handle normal date strings and shifts with respect to now.""" date = time.time() - shift_re=sre.compile("([-\+]{0,1})([\d]+)([dhms])") + shift_re = sre.compile("([-\+]{0,1})([\d]+)([dhms])") factors = {"d":24*3600, "h":3600, "m":60, "s":1} m = shift_re.match(var) if m: sign = m.groups()[0] == "-" and -1 or 1 factor = factors[m.groups()[2]] value = float(m.groups()[1]) date = time.localtime(date + sign * factor * value) date = time.strftime(format_string, date) else: date = time.strptime(var, format_string) date = time.strftime(format_string, date) return date def write_message(msg, stream=sys.stdout): """Write message and flush output stream (may be sys.stdout or sys.stderr). Useful for debugging stuff.""" if stream == sys.stdout or stream == sys.stderr: stream.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime())) stream.write("%s\n" % msg) stream.flush() else: sys.stderr.write("Unknown stream %s. [must be sys.stdout or sys.stderr]\n" % stream) return def task_sig_sleep(sig, frame): """Signal handler for the 'sleep' signal sent by BibSched.""" if options["verbose"] >= 9: write_message("task_sig_sleep(), got signal %s frame %s" % (sig, frame)) write_message("sleeping...") task_update_status("SLEEPING") signal.pause() # wait for wake-up signal def task_sig_wakeup(sig, frame): """Signal handler for the 'wakeup' signal sent by BibSched.""" if options["verbose"] >= 9: write_message("task_sig_wakeup(), got signal %s frame %s" % (sig, frame)) write_message("continuing...") task_update_status("CONTINUING") def task_sig_stop(sig, frame): """Signal handler for the 'stop' signal sent by BibSched.""" if options["verbose"] >= 9: write_message("task_sig_stop(), got signal %s frame %s" % (sig, frame)) write_message("stopping...") task_update_status("STOPPING") write_message("flushing cache or whatever...") time.sleep(3) write_message("closing tables or whatever...") time.sleep(1) write_message("stopped") task_update_status("STOPPED") sys.exit(0) def task_sig_suicide(sig, frame): """Signal handler for the 'suicide' signal sent by BibSched.""" if options["verbose"] >= 9: write_message("task_sig_suicide(), got signal %s frame %s" % (sig, frame)) write_message("suiciding myself now...") task_update_status("SUICIDING") write_message("suicided") task_update_status("SUICIDED") sys.exit(0) def task_sig_unknown(sig, frame): """Signal handler for the other unknown signals sent by shell or user.""" # do nothing for unknown signals: write_message("unknown signal %d (frame %s) ignored" % (sig, frame)) def fib(n): """Returns Fibonacci number for 'n'.""" out = 1 if n >= 2: out = fib(n-2) + fib(n-1) return out def authenticate(user, header="BibTaskEx Task Submission", action="runbibtaskex"): """Authenticate the user against the user database. Check for its password, if it exists. Check for action access rights. Return user name upon authorization success, do system exit upon authorization failure. """ print header print "=" * len(header) if user == "": print >> sys.stdout, "\rUsername: ", user = string.strip(string.lower(sys.stdin.readline())) else: print >> sys.stdout, "\rUsername:", user ## first check user pw: res = run_sql("select id,password from user where email=%s or nickname=%s", (user, user,), 1) if not res: print "Sorry, %s does not exist." % user sys.exit(1) else: (uid_db, password_db) = res[0] if password_db: password_entered = getpass.getpass() if password_db == password_entered: pass else: print "Sorry, wrong credentials for %s." % user sys.exit(1) ## secondly check authorization for the action: (auth_code, auth_message) = acc_authorize_action(uid_db, action) if auth_code != 0: print auth_message sys.exit(1) return user def task_submit(): """Submits task to the BibSched task queue. This is what people will be invoking via command line.""" global options ## sanity check: remove eventual "task" option: if options.has_key("task"): del options["task"] ## authenticate user: user = authenticate(options.get("user", "")) ## submit task: if options["verbose"] >= 9: print "" write_message("storing task options %s\n" % options) task_id = run_sql("""INSERT INTO schTASK (id,proc,user,runtime,sleeptime,status,arguments) VALUES (NULL,'bibtaskex',%s,%s,%s,'WAITING',%s)""", (user, options["runtime"], options["sleeptime"], marshal.dumps(options))) ## update task number: options["task"] = task_id - run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""", (marshal.dumps(options),task_id)) + run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""", (marshal.dumps(options), task_id)) write_message("Task #%d submitted." % task_id) return task_id def task_update_progress(msg): """Updates progress information in the BibSched task table.""" global options if options["verbose"] >= 9: write_message("Updating task progress to %s." % msg) return run_sql("UPDATE schTASK SET progress=%s where id=%s", (msg, options["task"])) def task_update_status(val): """Updates status information in the BibSched task table.""" global options if options["verbose"] >= 9: write_message("Updating task status to %s." % val) return run_sql("UPDATE schTASK SET status=%s where id=%s", (val, options["task"])) def task_read_status(task_id): """Read status information in the BibSched task table.""" res = run_sql("SELECT status FROM schTASK where id=%s", (task_id,), 1) try: out = res[0][0] except: out = 'UNKNOWN' return out def task_get_options(id): """Returns options for the task 'id' read from the BibSched task queue table.""" out = {} res = run_sql("SELECT arguments FROM schTASK WHERE id=%s AND proc='bibtaskex'", (id,)) try: out = marshal.loads(res[0][0]) except: write_message("Error: BibTaskEx task %d does not seem to exist." % id, sys.stderr) sys.exit(1) return out def task_run(task_id): """Runs the task by fetching arguments from the BibSched task queue. This is what BibSched will be invoking via daemon call. The task prints Fibonacci numbers for up to NUM on the stdout, and some messages on stderr. Return 1 in case of success and 0 in case of failure.""" global options options = task_get_options(task_id) # get options from BibSched task table ## check task id: if not options.has_key("task"): write_message("Error: The task #%d does not seem to be a BibTaskEx task." % task_id, sys.stderr) return 0 ## check task status: task_status = task_read_status(task_id) if task_status != "WAITING": write_message("Error: The task #%d is %s. I expected WAITING." % (task_id, task_status), sys.stderr) return 0 ## we can run the task now: if options["verbose"]: write_message("Task #%d started." % task_id) task_update_status("RUNNING") ## initialize signal handler: signal.signal(signal.SIGUSR1, task_sig_sleep) signal.signal(signal.SIGTERM, task_sig_stop) signal.signal(signal.SIGABRT, task_sig_suicide) signal.signal(signal.SIGCONT, task_sig_wakeup) signal.signal(signal.SIGINT, task_sig_unknown) ## run the task: if options.has_key("number"): n = options["number"] else: n = cfg_n_default if options["verbose"] >= 9: write_message("Printing %d Fibonacci numbers." % n) - for i in range(0,n): + for i in range(0, n): if i > 0 and i % 4 == 0: if options["verbose"] >= 3: write_message("Error: water in the CPU. Ignoring and continuing.", sys.stderr) elif i > 0 and i % 5 == 0: if options["verbose"]: write_message("Error: floppy drive dropped on the floor. Ignoring and continuing.", sys.stderr) if options["verbose"]: write_message("fib(%d)=%d" % (i, fib(i))) - task_update_progress("Done %d out of %d." % (i,n)) + task_update_progress("Done %d out of %d." % (i, n)) time.sleep(1) ## we are done: - task_update_progress("Done %d out of %d." % (n,n)) + task_update_progress("Done %d out of %d." % (n, n)) task_update_status("DONE") if options["verbose"]: write_message("Task #%d finished." % task_id) return 1 def usage(exitcode=1, msg=""): """Prints usage info.""" if msg: sys.stderr.write("Error: %s.\n" % msg) sys.stderr.write("Usage: %s [options]\n" % sys.argv[0]) sys.stderr.write("Command options:\n") sys.stderr.write(" -n, --number=NUM\t Print Fibonacci numbers for up to NUM. [default=%d]\n" % cfg_n_default) sys.stderr.write("Scheduling options:\n") sys.stderr.write(" -u, --user=USER \t User name to submit the task as, password needed.\n") sys.stderr.write(" -t, --runtime=TIME \t Time to execute the task (now), e.g.: +15s, 5m, 3h, 2002-10-27 13:57:26\n") sys.stderr.write(" -s, --sleeptime=SLEEP \t Sleeping frequency after which to repeat task (no), e.g.: 30m, 2h, 1d\n") sys.stderr.write("General options:\n") sys.stderr.write(" -h, --help \t\t Print this help.\n") sys.stderr.write(" -V, --version \t\t Print version information.\n") sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n") sys.exit(exitcode) def main(): """Main function that analyzes command line input and calls whatever is appropriate. Useful for learning on how to write BibSched tasks.""" global options ## parse command line: if len(sys.argv) == 2 and sys.argv[1].isdigit(): ## A - run the task task_id = int(sys.argv[1]) try: if not task_run(task_id): write_message("Error occurred. Exiting.", sys.stderr) except StandardError, e: write_message("Unexpected error occurred: %s." % e, sys.stderr) write_message("Traceback is:", sys.stderr) traceback.print_tb(sys.exc_info()[2]) write_message("Exiting.", sys.stderr) task_update_status("ERROR") else: ## B - submit the task # set default values: options = {} options["runtime"] = time.strftime("%Y-%m-%d %H:%M:%S") options["sleeptime"] = "" options["verbose"] = 1 # set user-defined options: try: - opts, args = getopt.getopt(sys.argv[1:], "hVv:n:u:s:t:", ["help", "version", "verbose=","number=","user=","sleep=","time="]) + opts, args = getopt.getopt(sys.argv[1:], "hVv:n:u:s:t:", ["help", "version", "verbose=", "number=", "user=", "sleep=", "time="]) except getopt.GetoptError, err: usage(1, err) try: for opt in opts: if opt[0] in ["-h", "--help"]: usage(0) elif opt[0] in ["-V", "--version"]: print __revision__ sys.exit(0) elif opt[0] in [ "-u", "--user"]: options["user"] = opt[1] elif opt[0] in ["-v", "--verbose"]: options["verbose"] = int(opt[1]) elif opt[0] in ["-n", "--number"]: - options["number"]=int(opt[1]) + options["number"] = int(opt[1]) elif opt[0] in [ "-s", "--sleeptime" ]: get_datetime(opt[1]) # see if it is a valid shift options["sleeptime"] = opt[1] elif opt[0] in [ "-t", "--runtime" ]: options["runtime"] = get_datetime(opt[1]) else: usage(1) except StandardError, e: usage(e) task_submit() return ### okay, here we go: if __name__ == '__main__': main()