diff --git a/configure.ac b/configure.ac
index 968b6b901..0d4d32287 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,703 +1,703 @@
## $Id$
## Purpose: CDS Invenio main configure.ac file.
## Note: If you change this file, please run "autoreconf" to regenerate the "configure" script.
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
+## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
## Initialize autoconf and automake:
AC_INIT(cds-invenio, 0.93.50.20070904, cds.support@cern.ch)
AM_INIT_AUTOMAKE(cds-invenio, 0.93.50.20070904)
## Check for install:
AC_PROG_INSTALL
## Define WEBURL URL:
AC_ARG_WITH(weburl, AC_HELP_STRING([--with-weburl], [specify URL where web files will be reached (e.g. http://webserver.domain.com)]), WEBURL=${withval})
if test -z "$WEBURL"; then
AC_MSG_ERROR([
- You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached.
+ You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached.
Example: "--with-weburl=http://webserver.domain.com".])
-fi
+fi
AC_ARG_WITH(sweburl, AC_HELP_STRING([--with-sweburl], [specify URL where HTTPS files will be reached (e.g. https://webserver.domain.com)]), SWEBURL=${withval})
if test -z "${SWEBURL}"; then
SWEBURL="${WEBURL}"
AC_MSG_WARN([
- You have not specified secure URL for accessing the site (--with-sweburl).
+ You have not specified secure URL for accessing the site (--with-sweburl).
User authentication will be done in clear text across the network.
])
-fi
+fi
## Define DBHOST database host:
AC_ARG_WITH(dbhost, AC_HELP_STRING([--with-dbhost], [specify DB server (e.g. sqlserver.domain.com)]), DBHOST=${withval})
if test -z "$DBHOST"; then
AC_MSG_ERROR([
- You have not used the "--with-dbhost" argument that should define which DB server to use.
+ You have not used the "--with-dbhost" argument that should define which DB server to use.
Example: "--with-dbhost=sqlserver.domain.com".])
-fi
+fi
## Define DBNAME database name:
AC_ARG_WITH(dbname, AC_HELP_STRING([--with-dbname], [specify DB name (e.g. cdsinvenio)]), DBNAME=${withval})
if test -z "$DBNAME"; then
AC_MSG_ERROR([
- You have not used the "--with-dbname" argument that should define DB name.
+ You have not used the "--with-dbname" argument that should define DB name.
Example: "--with-dbname=cdsinvenio".])
-fi
+fi
## Define DBUSER user name:
AC_ARG_WITH(dbuser, AC_HELP_STRING([--with-dbuser], [specify DB user name (e.g. cdsinvenio)]), DBUSER=${withval})
if test -z "$DBUSER"; then
AC_MSG_ERROR([
- You have not used the "--with-dbuser" argument that should define DB user name.
+ You have not used the "--with-dbuser" argument that should define DB user name.
Example: "--with-dbuser=cdsinvenio".])
-fi
+fi
## Define DBPASS user password:
AC_ARG_WITH(dbpass, AC_HELP_STRING([--with-dbpass], [specify DB user password (e.g. myp1ss)]), DBPASS=${withval})
if test -z "$DBPASS"; then
AC_MSG_ERROR([
- You have not used the "--with-dbpass" argument that should define DB user password.
+ You have not used the "--with-dbpass" argument that should define DB user password.
Example: "--with-dbpass=myp1ss".])
-fi
+fi
## Check for gettext support
AM_GNU_GETTEXT(external)
AM_GNU_GETTEXT_VERSION(0.14.4)
## Check for WML:
AC_PATH_PROG(WML, wml)
if test -z "$WML"; then
AC_MSG_ERROR([
- WML (Website META Language) was not found in your PATH. Please install it first.
+ WML (Website META Language) was not found in your PATH. Please install it first.
Available from .])
fi
## Set option -c in WML so that it can work from a separate build tree
WML="${WML} -c"
## Check for MySQL client:
AC_PATH_PROG(MYSQL, mysql)
if test -z "$MYSQL"; then
AC_MSG_ERROR([
MySQL command-line client was not found in your PATH. Please install it first.
Available from .])
fi
## Check for Python:
AC_MSG_CHECKING(for python)
AC_ARG_WITH(python, AC_HELP_STRING([--with-python], [path to a specific Python binary (optional)]), PYTHON=${withval})
if test -n "$PYTHON"; then
AC_MSG_RESULT($PYTHON)
else
AC_PATH_PROG(PYTHON, python)
if test -z "$PYTHON"; then
AC_MSG_ERROR([
- Python was not found in your PATH. Please either install it
+ Python was not found in your PATH. Please either install it
in your PATH or specify --with-python configure option.
Python is available from .])
fi
fi
## Check for Python version and modules:
AC_MSG_CHECKING(for required Python modules)
$PYTHON ${srcdir}/configure-tests.py
if test $? -ne 0; then
AC_MSG_ERROR([Please fix the above Python problem before continuing.])
fi
AC_MSG_RESULT(found)
## Check for PHP
AC_PATH_PROG(PHP, php)
## Check for Acrobat Reader:
AC_PATH_PROG(ACROREAD, acroread)
if test -z "$ACROREAD"; then
AC_MSG_WARN([
- Acrobat Reader was not found in your PATH. It is used in
- the WebSubmit module for automatic conversion of submitted documents.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ Acrobat Reader was not found in your PATH. It is used in
+ the WebSubmit module for automatic conversion of submitted documents.
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Acrobat Reader is available from
.])
fi
## Check for gzip:
AC_PATH_PROG(GZIP, gzip)
if test -z "$GZIP"; then
AC_MSG_WARN([
- Gzip was not found in your PATH. It is used in
+ Gzip was not found in your PATH. It is used in
the WebSubmit module to compress the data submitted in an archive.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Gzip is available from
.])
fi
## Check for gunzip:
AC_PATH_PROG(GUNZIP, gunzip)
if test -z "$GUNZIP"; then
AC_MSG_WARN([
- Gunzip was not found in your PATH. It is used in
+ Gunzip was not found in your PATH. It is used in
the WebSubmit module to correctly deal with submitted compressed
files.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Gunzip is available from
.])
fi
## Check for tar:
AC_PATH_PROG(TAR, tar)
if test -z "$TAR"; then
AC_MSG_WARN([
- Tar was not found in your PATH. It is used in
+ Tar was not found in your PATH. It is used in
the WebSubmit module to pack the submitted data into an archive.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Tar is available from
.])
fi
## Check for ps2pdf:
AC_PATH_PROG(PS2PDF, ps2pdf)
if test -z "$PS2PDF"; then
AC_MSG_WARN([
- ps2pdf was not found in your PATH. It is used in
+ ps2pdf was not found in your PATH. It is used in
the WebSubmit module to convert submitted PostScripts into PDF.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. PS2PDF is available from
.])
fi
## Check for pdftotext:
AC_PATH_PROG(PDFTOTEXT, pdftotext)
if test -z "$PDFTOTEXT"; then
AC_MSG_WARN([
- pdftotext was not found in your PATH. It is used for the fulltext indexation
+ pdftotext was not found in your PATH. It is used for the fulltext indexation
of PDF files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
- script. pdftotext is available from .
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
+ script. pdftotext is available from .
])
fi
## Check for pstotext:
AC_PATH_PROG(PSTOTEXT, pstotext)
if test -z "$PSTOTEXT"; then
AC_MSG_WARN([
- pstotext was not found in your PATH. It is used for the fulltext indexation
+ pstotext was not found in your PATH. It is used for the fulltext indexation
of PDF and PostScript files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. pstotext is available from .
])
fi
## Check for ps2ascii:
AC_PATH_PROG(PSTOASCII, ps2ascii)
if test -z "$PSTOASCII"; then
AC_MSG_WARN([
- ps2ascii was not found in your PATH. It is used for the fulltext indexation
+ ps2ascii was not found in your PATH. It is used for the fulltext indexation
of PostScript files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. ps2ascii is available from .
])
fi
## Check for antiword:
AC_PATH_PROG(ANTIWORD, antiword)
if test -z "$ANTIWORD"; then
AC_MSG_WARN([
- antiword was not found in your PATH. It is used for the fulltext indexation
+ antiword was not found in your PATH. It is used for the fulltext indexation
of Microsoft Word files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
- script. antiword is available from .
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
+ script. antiword is available from .
])
fi
## Check for catdoc:
AC_PATH_PROG(CATDOC, catdoc)
if test -z "$CATDOC"; then
AC_MSG_WARN([
- catdoc was not found in your PATH. It is used for the fulltext indexation
+ catdoc was not found in your PATH. It is used for the fulltext indexation
of Microsoft Word files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. catdoc is available from .
])
fi
## Check for wvText:
AC_PATH_PROG(WVTEXT, wvText)
if test -z "$WVTEXT"; then
AC_MSG_WARN([
- wvText was not found in your PATH. It is used for the fulltext indexation
+ wvText was not found in your PATH. It is used for the fulltext indexation
of Microsoft Word files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. wvText is available from .
])
-fi
+fi
## Check for ppthtml:
AC_PATH_PROG(PPTHTML, ppthtml)
if test -z "$PPTHTML"; then
AC_MSG_WARN([
- ppthtml was found in your PATH. It is used for the fulltext indexation
+ ppthtml was found in your PATH. It is used for the fulltext indexation
of Microsoft PowerPoint files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. ppthtml is available from .
])
fi
## Check for xlhtml:
AC_PATH_PROG(XLHTML, xlhtml)
if test -z "$XLHTML"; then
AC_MSG_WARN([
- xlhtml was found in your PATH. It is used for the fulltext indexation
+ xlhtml was found in your PATH. It is used for the fulltext indexation
of Microsoft Excel files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. xlhtml is available from .
])
fi
## Check for html2text:
AC_PATH_PROG(HTMLTOTEXT, html2text)
if test -z "$HTMLTOTEXT"; then
AC_MSG_WARN([
- html2text was found in your PATH. It is used for the fulltext indexation
+ html2text was found in your PATH. It is used for the fulltext indexation
of Microsoft PowerPoint and Excel files.
- You can continue without it but you may miss fulltext searching capability
- of CDS Invenio. We recomend you to install it first and to rerun the configure
+ You can continue without it but you may miss fulltext searching capability
+ of CDS Invenio. We recomend you to install it first and to rerun the configure
script. html2text is available from .
])
fi
## Check for Giftext:
AC_PATH_PROG(GIFTEXT, giftext)
if test -z "$GIFTEXT"; then
AC_MSG_WARN([
- Giftext was not found in your PATH. It is used in
+ Giftext was not found in your PATH. It is used in
the WebSubmit module to create an icon from a submitted picture.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Giftext is available from
.])
fi
## Check for file:
AC_PATH_PROG(FILE, file)
if test -z "$FILE"; then
AC_MSG_WARN([
- File was not found in your PATH. It is used in
+ File was not found in your PATH. It is used in
the WebSubmit module to check the validity of the submitted files.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. File is available from
.])
fi
## Check for convert:
AC_PATH_PROG(CONVERT, convert)
if test -z "$CONVERT"; then
AC_MSG_WARN([
- Convert was not found in your PATH. It is used in
+ Convert was not found in your PATH. It is used in
the WebSubmit module to create an icon from a submitted picture.
- You can continue without it but you will miss some CDS Invenio
- functionality. We recommend you to install it first and to rerun
+ You can continue without it but you will miss some CDS Invenio
+ functionality. We recommend you to install it first and to rerun
the configure script. Convert is available from
.])
fi
## Check for CLISP:
AC_MSG_CHECKING(for clisp)
AC_ARG_WITH(clisp, AC_HELP_STRING([--with-clisp], [path to a specific CLISP binary (optional)]), CLISP=${withval})
if test -n "$CLISP"; then
AC_MSG_RESULT($CLISP)
else
AC_PATH_PROG(CLISP, clisp)
if test -z "$CLISP"; then
AC_MSG_WARN([
GNU CLISP was not found in your PATH. It is used by the WebStat
module to produce statistics about CDS Invenio usage. (Alternatively,
SBCL or CMUCL can be used instead of CLISP.)
- You can continue without it but you will miss this feature. We
+ You can continue without it but you will miss this feature. We
recommend you to install it first (if you don't have neither CMUCL
- nor SBCL) and to rerun the configure script.
+ nor SBCL) and to rerun the configure script.
GNU CLISP is available from .])
fi
fi
## Check for CMUCL:
AC_MSG_CHECKING(for cmucl)
AC_ARG_WITH(cmucl, AC_HELP_STRING([--with-cmucl], [path to a specific CMUCL binary (optional)]), CMUCL=${withval})
if test -n "$CMUCL"; then
AC_MSG_RESULT($CMUCL)
else
AC_PATH_PROG(CMUCL, cmucl)
if test -z "$CMUCL"; then
AC_MSG_CHECKING(for lisp) # CMUCL can also be installed under `lisp' exec name
- AC_PATH_PROG(CMUCL, lisp)
+ AC_PATH_PROG(CMUCL, lisp)
fi
- if test -z "$CMUCL"; then
+ if test -z "$CMUCL"; then
AC_MSG_WARN([
CMUCL was not found in your PATH. It is used by the WebStat
module to produce statistics about CDS Invenio usage. (Alternatively,
CLISP or SBCL can be used instead of CMUCL.)
- You can continue without it but you will miss this feature. We
+ You can continue without it but you will miss this feature. We
recommend you to install it first (if you don't have neither CLISP
- nor SBCL) and to rerun the configure script.
+ nor SBCL) and to rerun the configure script.
CMUCL is available from .])
- fi
+ fi
fi
## Check for SBCL:
AC_MSG_CHECKING(for sbcl)
AC_ARG_WITH(sbcl, AC_HELP_STRING([--with-sbcl], [path to a specific SBCL binary (optional)]), SBCL=${withval})
if test -n "$SBCL"; then
AC_MSG_RESULT($SBCL)
else
AC_PATH_PROG(SBCL, sbcl)
if test -z "$SBCL"; then
AC_MSG_WARN([
SBCL was not found in your PATH. It is used by the WebStat
module to produce statistics about CDS Invenio usage. (Alternatively,
CLISP or CMUCL can be used instead of SBCL.)
- You can continue without it but you will miss this feature. We
+ You can continue without it but you will miss this feature. We
recommend you to install it first (if you don't have neither CLISP
- nor CMUCL) and to rerun the configure script.
+ nor CMUCL) and to rerun the configure script.
SBCL is available from .])
- fi
+ fi
fi
## Check for gnuplot:
AC_PATH_PROG(GNUPLOT, gnuplot)
if test -z "$GNUPLOT"; then
AC_MSG_WARN([
Gnuplot was not found in your PATH. It is used by the BibRank
module to produce graphs about download and citation history.
- You can continue without it but you will miss these graphs. We
- recommend you to install it first and to rerun the configure script.
+ You can continue without it but you will miss these graphs. We
+ recommend you to install it first and to rerun the configure script.
Gnuplot is available from .])
fi
## Substitute variables:
AC_SUBST(VERSION)
AC_SUBST(WML)
AC_SUBST(MYSQL)
AC_SUBST(PHP)
AC_SUBST(PYTHON)
AC_SUBST(CLIDIR)
AC_SUBST(WEBURL)
AC_SUBST(SWEBURL)
AC_SUBST(DBHOST)
AC_SUBST(DBNAME)
AC_SUBST(DBUSER)
AC_SUBST(DBPASS)
AC_SUBST(PDFTOTEXT)
AC_SUBST(PSTOTEXT)
AC_SUBST(PSTOASCII)
AC_SUBST(ANTIWORD)
AC_SUBST(CATDOC)
AC_SUBST(WVTEXT)
AC_SUBST(PPTHTML)
AC_SUBST(XLHTML)
AC_SUBST(HTMLTOTEXT)
AC_SUBST(localstatedir, `eval echo "${localstatedir}"`)
AC_SUBST(CACHEDIR)
AC_SUBST(CLISP)
AC_SUBST(CMUCL)
AC_SUBST(SBCL)
AC_SUBST(GNUPLOT)
AC_SUBST(DJPEG)
AC_SUBST(CONVERT)
AC_SUBST(GIFTEXT)
AC_SUBST(JPEGSIZE)
AC_SUBST(PNMSCALE)
AC_SUBST(PPMQUANT)
AC_SUBST(PPMTOGIF)
AC_SUBST(GIFINTER)
AC_SUBST(GIFRSIZE)
## Define output files:
AC_CONFIG_FILES([config.nice \
Makefile \
po/Makefile.in \
config/Makefile \
config/configbis.wml \
modules/Makefile \
modules/bibconvert/Makefile \
modules/bibconvert/bin/Makefile \
modules/bibconvert/bin/bibconvert \
modules/bibconvert/doc/Makefile \
modules/bibconvert/doc/admin/Makefile \
modules/bibconvert/doc/hacking/Makefile \
modules/bibconvert/etc/Makefile \
modules/bibconvert/lib/Makefile \
modules/bibmatch/Makefile \
modules/bibmatch/bin/Makefile \
modules/bibmatch/bin/bibmatch \
modules/bibmatch/doc/Makefile \
modules/bibmatch/doc/admin/Makefile \
modules/bibmatch/etc/Makefile \
modules/bibmatch/lib/Makefile \
modules/bibedit/Makefile \
modules/bibedit/bin/Makefile \
modules/bibedit/bin/refextract \
modules/bibedit/bin/xmlmarclint \
modules/bibedit/bin/xmlmarc2textmarc \
modules/bibedit/doc/Makefile \
modules/bibedit/doc/admin/Makefile \
modules/bibedit/doc/hacking/Makefile \
modules/bibedit/etc/Makefile \
modules/bibedit/lib/Makefile \
modules/bibedit/web/Makefile \
modules/bibedit/web/admin/Makefile \
modules/bibformat/Makefile \
modules/bibformat/bin/Makefile \
modules/bibformat/bin/bibformat \
modules/bibformat/bin/bibreformat \
modules/bibformat/doc/Makefile \
modules/bibformat/doc/admin/Makefile \
modules/bibformat/doc/hacking/Makefile \
modules/bibformat/etc/Makefile \
modules/bibformat/etc/output_formats/Makefile \
modules/bibformat/etc/format_templates/Makefile \
modules/bibformat/lib/Makefile \
modules/bibformat/lib/elements/Makefile \
modules/bibformat/lib/core/Makefile \
modules/bibformat/lib/common/Makefile \
modules/bibformat/web/Makefile \
modules/bibformat/web/admin/Makefile \
modules/bibharvest/Makefile \
modules/bibharvest/bin/Makefile \
modules/bibharvest/bin/bibharvest \
modules/bibharvest/bin/oaiharvest \
modules/bibharvest/bin/oaiarchive \
modules/bibharvest/doc/Makefile \
modules/bibharvest/doc/admin/Makefile \
modules/bibharvest/doc/hacking/Makefile \
modules/bibharvest/lib/Makefile \
modules/bibharvest/web/Makefile \
modules/bibharvest/web/admin/Makefile \
modules/bibclassify/Makefile \
modules/bibclassify/bin/Makefile \
modules/bibclassify/bin/bibclassify \
modules/bibclassify/bin/bibclassifyd \
modules/bibclassify/doc/Makefile \
modules/bibclassify/doc/admin/Makefile \
modules/bibclassify/doc/hacking/Makefile \
modules/bibclassify/etc/Makefile
modules/bibclassify/lib/Makefile
modules/bibindex/Makefile \
modules/bibindex/bin/Makefile \
modules/bibindex/bin/bibindex \
modules/bibindex/bin/bibstat \
modules/bibindex/doc/Makefile \
modules/bibindex/doc/admin/Makefile \
modules/bibindex/doc/hacking/Makefile \
modules/bibindex/lib/Makefile \
modules/bibindex/web/Makefile \
modules/bibindex/web/admin/Makefile \
modules/bibrank/Makefile \
modules/bibrank/bin/Makefile \
modules/bibrank/bin/bibrank \
modules/bibrank/bin/bibrankgkb \
modules/bibrank/doc/Makefile \
modules/bibrank/doc/admin/Makefile \
modules/bibrank/doc/hacking/Makefile \
modules/bibrank/etc/Makefile \
modules/bibrank/etc/bibrankgkb.cfg \
modules/bibrank/etc/demo_jif.cfg \
modules/bibrank/etc/template_single_tag_rank_method.cfg \
modules/bibrank/lib/Makefile \
modules/bibrank/web/Makefile \
modules/bibrank/web/admin/Makefile \
modules/bibsched/Makefile \
modules/bibsched/bin/Makefile \
modules/bibsched/bin/bibsched \
modules/bibsched/bin/bibtaskex \
modules/bibsched/doc/Makefile \
modules/bibsched/doc/admin/Makefile \
modules/bibsched/doc/hacking/Makefile \
modules/bibsched/lib/Makefile \
modules/bibupload/Makefile \
modules/bibupload/bin/Makefile \
modules/bibupload/bin/bibupload \
modules/bibupload/doc/Makefile \
modules/bibupload/doc/admin/Makefile \
modules/bibupload/doc/hacking/Makefile \
modules/bibupload/lib/Makefile \
modules/elmsubmit/Makefile \
modules/elmsubmit/bin/Makefile \
modules/elmsubmit/bin/elmsubmit \
modules/elmsubmit/doc/Makefile \
modules/elmsubmit/doc/admin/Makefile \
modules/elmsubmit/doc/hacking/Makefile \
modules/elmsubmit/etc/Makefile \
modules/elmsubmit/etc/elmsubmit.cfg \
modules/elmsubmit/lib/Makefile \
modules/elmsubmit/lib/magic/Makefile \
modules/miscutil/Makefile \
modules/miscutil/bin/Makefile \
modules/miscutil/bin/dbexec \
modules/miscutil/bin/dbtest \
modules/miscutil/bin/testsuite \
modules/miscutil/bin/regressiontestsuite \
modules/miscutil/lib/Makefile \
modules/miscutil/demo/Makefile \
modules/miscutil/sql/Makefile \
modules/miscutil/web/Makefile \
modules/miscutil/doc/Makefile \
modules/miscutil/doc/hacking/Makefile \
modules/webaccess/Makefile \
modules/webaccess/bin/Makefile \
modules/webaccess/bin/authaction \
modules/webaccess/bin/webaccessadmin \
modules/webaccess/doc/Makefile \
modules/webaccess/doc/admin/Makefile \
modules/webaccess/doc/hacking/Makefile \
modules/webaccess/lib/Makefile \
modules/webaccess/web/Makefile \
modules/webaccess/web/admin/Makefile \
modules/webalert/Makefile \
modules/webalert/bin/Makefile \
modules/webalert/bin/alertengine \
modules/webalert/doc/Makefile \
modules/webalert/doc/admin/Makefile \
modules/webalert/doc/hacking/Makefile \
modules/webalert/lib/Makefile \
modules/webalert/web/Makefile \
modules/webhelp/Makefile \
modules/webhelp/web/Makefile \
modules/webhelp/web/hacking/Makefile \
modules/webhelp/web/admin/Makefile \
modules/webhelp/web/admin/howto/Makefile \
modules/websearch/Makefile \
modules/websearch/bin/Makefile \
modules/websearch/bin/webcoll \
modules/websearch/doc/Makefile \
modules/websearch/doc/admin/Makefile \
modules/websearch/doc/hacking/Makefile \
modules/websearch/lib/Makefile \
modules/websearch/web/Makefile \
modules/websearch/web/admin/Makefile \
modules/websession/Makefile \
modules/websession/bin/Makefile \
- modules/websession/bin/sessiongc \
+ modules/websession/bin/inveniogc \
modules/websession/doc/Makefile \
modules/websession/doc/admin/Makefile \
modules/websession/doc/hacking/Makefile \
modules/websession/lib/Makefile \
modules/websession/web/Makefile \
modules/webstat/Makefile \
modules/webstat/bin/Makefile \
modules/webstat/bin/webstat \
modules/webstat/bin/webstatadmin \
modules/webstat/doc/Makefile \
modules/webstat/doc/admin/Makefile \
modules/webstat/doc/hacking/Makefile \
modules/webstat/etc/Makefile \
modules/webstat/lib/Makefile \
modules/webstyle/Makefile \
modules/webstyle/css/Makefile \
modules/webstyle/doc/Makefile \
modules/webstyle/doc/admin/Makefile \
modules/webstyle/doc/hacking/Makefile \
modules/webstyle/img/Makefile \
modules/webstyle/lib/Makefile \
modules/webcomment/Makefile \
modules/webcomment/doc/Makefile \
modules/webcomment/doc/admin/Makefile \
modules/webcomment/doc/hacking/Makefile \
modules/webcomment/lib/Makefile \
modules/webcomment/web/Makefile \
modules/webcomment/web/admin/Makefile \
modules/webbasket/Makefile \
modules/webbasket/doc/Makefile \
modules/webbasket/doc/admin/Makefile \
modules/webbasket/doc/hacking/Makefile \
modules/webbasket/lib/Makefile \
modules/webbasket/web/Makefile \
modules/webmessage/Makefile \
modules/webmessage/bin/Makefile \
modules/webmessage/bin/webmessageadmin \
modules/webmessage/doc/Makefile \
modules/webmessage/doc/admin/Makefile \
modules/webmessage/doc/hacking/Makefile \
modules/webmessage/lib/Makefile \
modules/webmessage/web/Makefile \
modules/websubmit/Makefile \
modules/websubmit/bin/Makefile \
modules/websubmit/bin/thumbmaker \
modules/websubmit/etc/Makefile \
modules/websubmit/etc/bibconvert/Makefile \
modules/websubmit/etc/bibconvert/KB/Makefile \
modules/websubmit/etc/bibconvert/config/Makefile \
modules/websubmit/etc/bibconvert/config/DEMOPICcreate.tpl \
modules/websubmit/etc/bibconvert/config/EDSTEXTcreate.tpl \
modules/websubmit/etc/bibconvert/config/EDSRTEXTcreate.tpl \
modules/websubmit/doc/Makefile \
modules/websubmit/doc/admin/Makefile \
modules/websubmit/doc/hacking/Makefile \
modules/websubmit/lib/Makefile \
modules/websubmit/lib/functions/Makefile \
modules/websubmit/web/Makefile \
modules/websubmit/web/admin/Makefile \
])
AC_CONFIG_COMMANDS(.wmlrc, [
cat<.wmlrc
-Iconfig -I${srcdir}/config
EOF
])
## Finally, write output files:
AC_OUTPUT
## Write help:
AC_MSG_RESULT([****************************************************************************])
-AC_MSG_RESULT([** Your CDS Invenio installation is now ready for building. **])
-AC_MSG_RESULT([** You have entered the following parameters: **])
+AC_MSG_RESULT([** Your CDS Invenio installation is now ready for building. **])
+AC_MSG_RESULT([** You have entered the following parameters: **])
AC_MSG_RESULT([** - CDS Invenio main install directory: ${prefix}])
AC_MSG_RESULT([** - Web home URL: $WEBURL])
AC_MSG_RESULT([** - Secure web home URL: $SWEBURL])
AC_MSG_RESULT([** - DB server: $DBHOST])
AC_MSG_RESULT([** - DB name: $DBNAME])
AC_MSG_RESULT([** - DB username: $DBUSER])
AC_MSG_RESULT([** - DB password: $DBPASS])
AC_MSG_RESULT([** - Python executable: $PYTHON])
AC_MSG_RESULT([** - CLISP executable: $CLISP])
AC_MSG_RESULT([** - CMUCL executable: $CMUCL])
AC_MSG_RESULT([** - SBCL executable: $SBCL])
AC_MSG_RESULT([** Here are the steps to continue the building process: **])
AC_MSG_RESULT([** 1) Customize self-explanatory config file './config/config.wml'. **])
AC_MSG_RESULT([** 2) Type 'make' to build your customized CDS Invenio installation. **])
AC_MSG_RESULT([** 3) Type 'make create-tables' if you have not created tables before. **])
AC_MSG_RESULT([** 4) Type 'make install' to install the system. **])
AC_MSG_RESULT([** 5) Type 'make create-demo-site' to install and test the demo site. **])
AC_MSG_RESULT([** Good luck, and thanks for choosing CDS Invenio. **])
AC_MSG_RESULT([** -- CDS Development Group **])
AC_MSG_RESULT([****************************************************************************])
## end of file
diff --git a/modules/bibsched/lib/bibsched.py b/modules/bibsched/lib/bibsched.py
index 9c1982651..8534f6892 100644
--- a/modules/bibsched/lib/bibsched.py
+++ b/modules/bibsched/lib/bibsched.py
@@ -1,892 +1,892 @@
# -*- coding: utf-8 -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibSched - task management, scheduling and executing system for CDS Invenio
"""
__revision__ = "$Id$"
### -- local configuration section starts here ---
# which tasks are recognized as valid?
cfg_valid_processes = ["bibindex", "bibupload", "bibreformat",
"webcoll", "bibtaskex", "bibrank",
- "oaiharvest", "oaiarchive", "sessiongc",
+ "oaiharvest", "oaiarchive", "inveniogc",
"webstatadmin", "bibclassifyd"]
### -- local configuration section ends here ---
import os
import string
import sys
import time
import re
import marshal
import getopt
import curses
import curses.panel
from curses.wrapper import wrapper
import signal
from invenio.config import \
CFG_PREFIX, \
CFG_BIBSCHED_REFRESHTIME, \
CFG_BIBSCHED_LOG_PAGER, \
bindir, \
logdir
from invenio.dbquery import run_sql
def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"):
"""Returns a date string according to the format string.
It can handle normal date strings and shifts with respect
to now."""
try:
date = time.time()
shift_re = re.compile("([-\+]{0,1})([\d]+)([dhms])")
factors = {"d":24*3600, "h":3600, "m":60, "s":1}
m = shift_re.match(var)
if m:
sign = m.groups()[0] == "-" and -1 or 1
factor = factors[m.groups()[2]]
value = float(m.groups()[1])
date = time.localtime(date + sign * factor * value)
date = time.strftime(format_string, date)
else:
date = time.strptime(var, format_string)
date = time.strftime(format_string, date)
return date
except:
return None
def get_my_pid(process, args=''):
if sys.platform.startswith('freebsd'):
COMMAND = "ps -o pid,args | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, args)
else:
COMMAND = "ps -C %s o '%%p%%a' | grep '%s %s' | grep -v 'grep' | sed -n 1p" % (process, process, args)
answer = string.strip(os.popen(COMMAND).read())
if answer == '':
answer = 0
else:
answer = answer[:string.find(answer,' ')]
return int(answer)
def get_task_pid(task_name, task_id):
"""Return the pid of task_name/task_id"""
try:
pid = int(open(os.path.join(CFG_PREFIX, 'var', 'run', 'bibsched_task_%d.pid' % task_id)).read())
except IOError:
return get_my_pid(task_name, str(task_id))
try:
os.kill(pid, signal.SIGCONT)
except OSError:
return get_my_pid(task_name, str(task_id))
return pid
def get_output_channelnames(task_id):
"Construct and return filename for stdout and stderr for the task 'task_id'."
filenamebase = "%s/bibsched_task_%d" % (logdir, task_id)
return [filenamebase + ".log", filenamebase + ".err"]
def is_task_scheduled(task_name):
"""Check if a certain task_name is due for execution (WAITING or RUNNING)"""
sql = "SELECT COUNT(proc) FROM schTASK WHERE proc = %s AND (status = 'WAITING' or status = 'RUNNING')"
return run_sql(sql, (task_name,))[0][0] > 0
def get_task_ids_by_descending_date(task_name, statuses=['SCHEDULED']):
"""Returns list of task ids, ordered by descending runtime."""
sql = "SELECT id FROM schTASK WHERE proc=%s AND (" + \
" OR ".join(["status = '%s'" % x for x in statuses]) + ") ORDER BY runtime DESC"
return [x[0] for x in run_sql(sql, (task_name,))]
def get_task_options(task_id):
"""Returns options for task_id read from the BibSched task queue table."""
res = run_sql("SELECT arguments FROM schTASK WHERE id=%s", (task_id,))
try:
return marshal.loads(res[0][0])
except IndexError:
return {}
class Manager:
def __init__(self):
self.helper_modules = cfg_valid_processes
self.running = 1
self.footer_move_mode = "[KeyUp/KeyDown Move] [M Select mode] [Q Quit]"
self.footer_auto_mode = "[A Manual mode] [1/2 Display Type] [P Purge Done] [Q Quit]"
self.footer_select_mode = "[KeyUp/KeyDown/PgUp/PgDown Select] [L View Log] [1/2 Display Type] [M Move mode] [A Auto mode] [Q Quit]"
self.footer_waiting_item = "[R Run] [D Delete]"
self.footer_running_item = "[S Sleep] [T Stop] [K Kill]"
self.footer_stopped_item = "[I Initialise] [D Delete]"
self.footer_sleeping_item = "[W Wake Up]"
self.item_status = ""
self.selected_line = 2
self.rows = []
self.panel = None
self.display = 2
self.first_visible_line = 0
self.move_mode = 0
self.auto_mode = 0
self.currentrow = ["", "", "", "", "", "", ""]
wrapper(self.start)
def handle_keys(self, chr):
if chr == -1:
return
if self.auto_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN,
curses.KEY_PPAGE, curses.KEY_NPAGE,
ord("q"), ord("Q"), ord("a"),
ord("A"), ord("1"), ord("2"),
ord("p"), ord("P"))):
self.display_in_footer("in automatic mode")
self.stdscr.refresh()
elif self.move_mode and (chr not in (curses.KEY_UP, curses.KEY_DOWN,
ord("m"), ord("M"), ord("q"),
ord("Q"))):
self.display_in_footer("in move mode")
self.stdscr.refresh()
else:
if chr == curses.KEY_UP:
if self.move_mode:
self.move_up()
else:
self.selected_line = max(self.selected_line - 1, 2)
self.repaint()
if chr == curses.KEY_PPAGE:
self.selected_line = max(self.selected_line - 10, 2)
self.repaint()
elif chr == curses.KEY_DOWN:
if self.move_mode:
self.move_down()
else:
self.selected_line = min(self.selected_line + 1, len(self.rows) + 1 )
self.repaint()
elif chr == curses.KEY_NPAGE:
self.selected_line = min(self.selected_line + 10, len(self.rows) + 1 )
self.repaint()
elif chr == curses.KEY_HOME:
self.first_visible_line = 0
self.selected_line = 2
elif chr in (ord("a"), ord("A")):
self.change_auto_mode()
elif chr in (ord("l"), ord("L")):
self.openlog()
elif chr in (ord("w"), ord("W")):
self.wakeup()
elif chr in (ord("r"), ord("R")):
self.run()
elif chr in (ord("s"), ord("S")):
self.sleep()
elif chr in (ord("k"), ord("K")):
self.kill()
elif chr in (ord("t"), ord("T")):
self.stop()
elif chr in (ord("d"), ord("D")):
self.delete()
elif chr in (ord("i"), ord("I")):
self.init()
elif chr in (ord("m"), ord("M")):
self.change_select_mode()
elif chr in (ord("p"), ord("P")):
self.purge_done()
elif chr == ord("1"):
self.display = 1
self.first_visible_line = 0
self.selected_line = 2
self.display_in_footer("only done processes are displayed")
elif chr == ord("2"):
self.display = 2
self.first_visible_line = 0
self.selected_line = 2
self.display_in_footer("only not done processes are displayed")
elif chr in (ord("q"), ord("Q")):
if curses.panel.top_panel() == self.panel:
self.panel.bottom()
curses.panel.update_panels()
else:
self.running = 0
return
def set_status(self, task_id, status):
return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id))
def set_progress(self, task_id, progress):
return run_sql("UPDATE schTASK set progress=%s WHERE id=%s", (progress, task_id))
def openlog(self):
task_id = self.currentrow[0]
status = self.currentrow[5]
if status != 'WAITING':
tmpname = os.tmpnam()
tmpfile = open(tmpname, "w")
try:
tmpfile.write(open(os.path.join(logdir, 'bibsched_task_%d.log' % task_id)).read())
except IOError:
pass
try:
tmpfile.write(open(os.path.join(logdir, 'bibsched_task_%d.err' % task_id)).read())
except IOError:
pass
tmpfile.close()
if CFG_BIBSCHED_LOG_PAGER:
pager = CFG_BIBSCHED_LOG_PAGER
else:
pager = os.environ.get('PAGER', '/bin/more')
curses.endwin()
os.spawnlp(os.P_WAIT, pager, pager, tmpname)
os.remove(tmpname)
curses.panel.update_panels()
def count_processes(self, status):
out = 0
res = run_sql("SELECT COUNT(id) FROM schTASK WHERE status=%s GROUP BY status", (status,))
try:
out = res[0][0]
except:
pass
return out
def wakeup(self):
task_id = self.currentrow[0]
process = self.currentrow[1]
status = self.currentrow[5]
if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1:
self.display_in_footer("a process is already running!")
elif status == "SLEEPING":
mypid = get_task_pid(process, task_id)
if mypid != 0:
os.kill(mypid, signal.SIGCONT)
self.display_in_footer("process woken up")
else:
self.display_in_footer("process is not sleeping")
self.stdscr.refresh()
def _display_YN_box(self, msg):
msg += ' (Y/N)'
rows = msg.split('\n')
height = len(rows) + 2
width = max([len(row) for row in rows]) + 4
self.win = curses.newwin(
height,
width,
(self.height - height) / 2 + 1,
(self.width - width) / 2 + 1
)
self.panel = curses.panel.new_panel( self.win )
self.panel.top()
self.win.border()
i = 1
for row in rows:
self.win.addstr(i, 2, row)
i += 1
self.win.refresh()
while 1:
c = self.win.getch()
if c in (ord('y'), ord('Y')):
return True
elif c in (ord('n'), ord('N')):
return False
def purge_done(self):
if self._display_YN_box("You are going to purge all the list of DONE tasks.\n"
"This will definitely alter your task history.\nAre you sure?"):
run_sql("DELETE FROM schTASK WHERE status='DONE'")
curses.panel.update_panels()
self.display_in_footer("DONE processes purged")
else:
curses.panel.update_panels()
def run(self):
task_id = self.currentrow[0]
process = self.currentrow[1]
status = self.currentrow[5]
sleeptime = self.currentrow[4]
if self.count_processes('RUNNING') + self.count_processes('CONTINUING') >= 1:
self.display_in_footer("a process is already running!")
elif status == "STOPPED" or status == "WAITING":
if process in self.helper_modules:
program = os.path.join(bindir, process)
fdout, fderr = get_output_channelnames(task_id)
COMMAND = "%s %s >> %s 2>> %s &" % (program, str(task_id), fdout, fderr)
os.system(COMMAND)
Log("manually running task #%d (%s)" % (task_id, process))
if sleeptime:
new_runtime = get_datetime(sleeptime)
new_task_arguments = marshal.loads(self.currentrow[7])
new_task_arguments["runtime"] = new_runtime
new_task_id = run_sql("INSERT INTO schTASK (proc,user,runtime,sleeptime,arguments,status)"\
" VALUES (%s,%s,%s,%s,%s,'WAITING')",
(process, self.currentrow[2], new_runtime, sleeptime,
self.currentrow[7]))
new_task_arguments["task"] = new_task_id
run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""",
(marshal.dumps(new_task_arguments), new_task_id))
else:
self.display_in_footer("process status should be STOPPED or WAITING!")
self.stdscr.refresh()
def sleep(self):
task_id = self.currentrow[0]
process = self.currentrow[1]
status = self.currentrow[5]
if status != 'RUNNING' and status != 'CONTINUING':
self.display_in_footer("this process is not running!")
else:
mypid = get_task_pid(process, task_id)
if mypid != 0:
os.kill(mypid, signal.SIGUSR1)
self.display_in_footer("USR1 signal sent to process #%s" % mypid)
else:
self.set_status(task_id, 'STOPPED')
self.display_in_footer("cannot find process...")
self.stdscr.refresh()
def kill(self):
task_id = self.currentrow[0]
process = self.currentrow[1]
#status = self.currentrow[5]
mypid = get_task_pid(process, task_id)
if mypid != 0:
os.kill(mypid, signal.SIGKILL)
self.set_status(task_id, 'STOPPED')
self.display_in_footer("KILL signal sent to process #%s" % mypid)
else:
self.set_status(task_id, 'STOPPED')
self.display_in_footer("cannot find process...")
self.stdscr.refresh()
def stop(self):
task_id = self.currentrow[0]
process = self.currentrow[1]
#status = self.currentrow[5]
mypid = get_task_pid(process, task_id)
if mypid != 0:
os.kill(mypid, signal.SIGTERM)
self.display_in_footer("TERM signal sent to process #%s" % mypid)
else:
self.set_status(task_id, 'STOPPED')
self.display_in_footer("cannot find process...")
self.stdscr.refresh()
def delete(self):
task_id = self.currentrow[0]
#process = self.currentrow[1]
status = self.currentrow[5]
if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING':
self.set_status(task_id, "%s_DELETED" % status)
self.display_in_footer("process deleted")
self.selected_line = max(self.selected_line, 2)
else:
self.display_in_footer("cannot delete running processes")
self.stdscr.refresh()
def init(self):
task_id = self.currentrow[0]
#process = self.currentrow[1]
status = self.currentrow[5]
if status != 'RUNNING' and status != 'CONTINUING' and status != 'SLEEPING':
self.set_status(task_id, "WAITING")
self.set_progress(task_id, "None")
self.display_in_footer("process initialised")
else:
self.display_in_footer("cannot initialise running processes")
self.stdscr.refresh()
def change_select_mode(self):
if self.move_mode:
self.move_mode = 0
else:
status = self.currentrow[5]
if status in ( "RUNNING" , "CONTINUING" , "SLEEPING" ):
self.display_in_footer("cannot move running processes!")
else:
self.move_mode = 1
self.stdscr.refresh()
def change_auto_mode(self):
if self.auto_mode:
program = os.path.join(bindir, "bibsched")
COMMAND = "%s -q stop" % program
os.system(COMMAND)
self.auto_mode = 0
else:
program = os.path.join( bindir, "bibsched")
COMMAND = "%s -q start" % program
os.system(COMMAND)
self.auto_mode = 1
self.move_mode = 0
self.stdscr.refresh()
def move_up(self):
self.display_in_footer("not implemented yet")
self.stdscr.refresh()
def move_down(self):
self.display_in_footer("not implemented yet")
self.stdscr.refresh()
def put_line(self, row):
col_w = [ 5 , 11 , 21 , 21 , 7 , 11 , 25 ]
maxx = self.width
if self.y == self.selected_line - self.first_visible_line and self.y > 1:
if self.auto_mode:
attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD
elif self.move_mode:
attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD
else:
attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD
self.item_status = row[5]
self.currentrow = row
elif self.y == 0:
if self.auto_mode:
attr = curses.color_pair(2) + curses.A_STANDOUT + curses.A_BOLD
elif self.move_mode:
attr = curses.color_pair(7) + curses.A_STANDOUT + curses.A_BOLD
else:
attr = curses.color_pair(8) + curses.A_STANDOUT + curses.A_BOLD
elif row[5] == "DONE":
attr = curses.color_pair(5) + curses.A_BOLD
elif row[5] == "STOPPED":
attr = curses.color_pair(6) + curses.A_BOLD
elif row[5].find("ERROR") > -1:
attr = curses.color_pair(4) + curses.A_BOLD
elif row[5] == "WAITING":
attr = curses.color_pair(3) + curses.A_BOLD
elif row[5] in ("RUNNING","CONTINUING") :
attr = curses.color_pair(2) + curses.A_BOLD
else:
attr = curses.A_BOLD
myline = str(row[0]).ljust(col_w[0])
myline += str(row[1]).ljust(col_w[1])
myline += str(row[2]).ljust(col_w[2])
myline += str(row[3])[:19].ljust(col_w[3])
myline += str(row[4]).ljust(col_w[4])
myline += str(row[5]).ljust(col_w[5])
myline += str(row[6]).ljust(col_w[6])
myline = myline.ljust(maxx)
self.stdscr.addnstr(self.y, 0, myline, maxx, attr)
self.y = self.y+1
def display_in_footer(self, footer, i = 0, print_time_p=0):
if print_time_p:
footer = "%s %s" % (footer, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
maxx = self.stdscr.getmaxyx()[1]
footer = footer.ljust(maxx)
if self.auto_mode:
colorpair = 2
elif self.move_mode:
colorpair = 7
else:
colorpair = 1
self.stdscr.addnstr(self.y - i, 0, footer, maxx - 1, curses.A_STANDOUT + curses.color_pair(colorpair) + curses.A_BOLD )
def repaint(self):
self.y = 0
self.stdscr.clear()
self.height, self.width = self.stdscr.getmaxyx()
maxy = self.height - 2
#maxx = self.width
self.put_line( ("ID", "PROC", "USER", "RUNTIME", "SLEEP", "STATUS", "PROGRESS") )
self.put_line( ("---", "----", "----", "-------------------", "-----", "-----", "--------") )
if self.selected_line > maxy + self.first_visible_line - 1:
self.first_visible_line = self.selected_line - maxy + 1
if self.selected_line < self.first_visible_line + 2:
self.first_visible_line = self.selected_line - 2
for row in self.rows[self.first_visible_line:self.first_visible_line+maxy-2]:
task_id, proc, user, runtime, sleeptime, status, progress, arguments = row
self.put_line( row )
self.y = self.stdscr.getmaxyx()[0] - 1
if self.auto_mode:
self.display_in_footer(self.footer_auto_mode, print_time_p=1)
elif self.move_mode:
self.display_in_footer(self.footer_move_mode, print_time_p=1)
else:
self.display_in_footer(self.footer_select_mode, print_time_p=1)
footer2 = ""
if self.item_status.find("DONE") > -1 or self.item_status == "ERROR" or self.item_status == "STOPPED":
footer2 += self.footer_stopped_item
elif self.item_status == "RUNNING" or self.item_status == "CONTINUING":
footer2 += self.footer_running_item
elif self.item_status == "SLEEPING":
footer2 += self.footer_sleeping_item
elif self.item_status == "WAITING":
footer2 += self.footer_waiting_item
self.display_in_footer(footer2, 1)
self.stdscr.refresh()
def start(self, stdscr):
ring = 0
if curses.has_colors():
curses.start_color()
curses.init_pair(8, curses.COLOR_WHITE, curses.COLOR_BLACK)
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_RED)
curses.init_pair(2, curses.COLOR_GREEN, curses.COLOR_BLACK)
curses.init_pair(3, curses.COLOR_MAGENTA, curses.COLOR_BLACK)
curses.init_pair(4, curses.COLOR_RED, curses.COLOR_BLACK)
curses.init_pair(5, curses.COLOR_BLUE, curses.COLOR_BLACK)
curses.init_pair(6, curses.COLOR_CYAN, curses.COLOR_BLACK)
curses.init_pair(7, curses.COLOR_YELLOW, curses.COLOR_BLACK)
self.stdscr = stdscr
self.base_panel = curses.panel.new_panel( self.stdscr )
self.base_panel.bottom()
curses.panel.update_panels()
self.height, self.width = stdscr.getmaxyx()
self.stdscr.clear()
if server_pid (): self.auto_mode = 1
if self.display == 1:
where = "and status='DONE'"
order = "DESC"
else:
where = "and status!='DONE'"
order = "ASC"
self.rows = run_sql("""SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s""" % (where, order))
self.repaint()
ring = 0
while self.running:
ring += 1
char = -1
try:
char = timed_out(self.stdscr.getch, 1)
if char == 27: # escaping sequence
char = self.stdscr.getch()
if char == 79: # arrow
char = self.stdscr.getch()
if char == 65: #arrow up
char = curses.KEY_UP
elif char == 66: #arrow down
char = curses.KEY_DOWN
elif char == 72:
char = curses.KEY_PPAGE
elif char == 70:
char = curses.KEY_NPAGE
elif char == 91:
char = self.stdscr.getch()
if char == 53:
char = self.stdscr.getch()
if char == 126:
char = curses.KEY_HOME
except TimedOutExc:
char = -1
self.handle_keys(char)
if ring == 4:
if self.display == 1:
where = "and status='DONE'"
order = "DESC"
else:
where = "and status!='DONE'"
order = "ASC"
self.rows = run_sql("""SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s""" % (where, order))
ring = 0
self.repaint()
class BibSched:
def __init__(self):
self.helper_modules = cfg_valid_processes
self.running = {}
self.sleep_done = {}
self.sleep_sent = {}
self.stop_sent = {}
self.suicide_sent = {}
def set_status(self, task_id, status):
return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, task_id))
def can_run( self, proc ):
return len( self.running.keys() ) == 0
def get_running_processes(self):
row = None
res = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK "\
" WHERE status='RUNNING' or status='CONTINUING' LIMIT 1")
try:
row = res[0]
except:
pass
return row
def handle_row( self, row ):
task_id, proc, user, runtime, sleeptime, arguments, status = row
if status == "SLEEP":
if task_id in self.running.keys():
self.set_status( task_id, "SLEEP SENT" )
os.kill( self.running[task_id], signal.SIGUSR1 )
self.sleep_sent[task_id] = self.running[task_id]
elif status == "SLEEPING":
if task_id in self.sleep_sent.keys():
self.sleep_done[task_id] = self.sleep_sent[task_id]
del self.sleep_sent[task_id]
if status == "WAKEUP":
if task_id in self.sleep_done.keys():
self.running[task_id] = self.sleep_done[task_id]
del self.sleep_done[task_id]
os.kill( self.running[task_id], signal.SIGCONT )
self.set_status( task_id, "RUNNING" )
elif status == "STOP":
if task_id in self.running.keys():
self.set_status( task_id, "STOP SENT" )
os.kill( self.running[task_id], signal.SIGTERM )
self.stop_sent[task_id] = self.running[task_id]
del self.running[task_id]
elif status == "STOPPED" and task_id in self.stop_sent.keys():
del self.stop_sent[task_id]
elif status == "SUICIDE":
if task_id in self.running.keys():
self.set_status( task_id, "SUICIDE SENT" )
os.kill( self.running[task_id], signal.SIGABRT )
self.suicide_sent[task_id] = self.running[task_id]
del self.running[task_id]
elif status == "SUICIDED" and task_id in self.suicide_sent.keys():
del self.suicide_sent[task_id]
elif status.find("DONE") > -1 and task_id in self.running.keys():
del self.running[task_id]
elif self.can_run(proc) and status == "WAITING" and runtime <= time.time():
if proc in self.helper_modules:
program = os.path.join(bindir, proc)
fdout, fderr = get_output_channelnames(task_id)
COMMAND = "%s %s >> %s 2>> %s" % (program, str(task_id), fdout, fderr)
Log("task #%d (%s) started" % (task_id, proc))
os.system(COMMAND)
Log("task #%d (%s) ended" % (task_id, proc))
self.running[task_id] = get_task_pid(proc, task_id)
if sleeptime:
new_runtime = get_datetime(sleeptime)
new_task_arguments = marshal.loads(arguments)
new_task_arguments["runtime"] = new_runtime
new_task_id = run_sql("INSERT INTO schTASK (proc,user,runtime,sleeptime,arguments,status)"\
" VALUES (%s,%s,%s,%s,%s,'WAITING')",
(proc, user, new_runtime, sleeptime, arguments))
new_task_arguments["task"] = new_task_id
run_sql("""UPDATE schTASK SET arguments=%s WHERE id=%s""",
(marshal.dumps(new_task_arguments), new_task_id))
def watch_loop(self):
running_process = self.get_running_processes()
if running_process:
proc = running_process[ 1 ]
task_id = running_process[ 0 ]
if get_task_pid(proc, task_id):
self.running[task_id] = get_task_pid(proc, task_id)
else:
self.set_status(task_id,"ERROR")
rows = []
while 1:
for row in rows:
self.handle_row( row )
time.sleep(CFG_BIBSCHED_REFRESHTIME)
rows = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK ORDER BY runtime ASC")
class TimedOutExc(Exception):
def __init__(self, value = "Timed Out"):
self.value = value
def __str__(self):
return repr(self.value)
def timed_out(f, timeout, *args, **kwargs):
def handler(signum, frame):
raise TimedOutExc()
old = signal.signal(signal.SIGALRM, handler)
signal.alarm(timeout)
try:
result = f(*args, **kwargs)
finally:
signal.signal(signal.SIGALRM, old)
signal.alarm(0)
return result
def Log(message):
log = open(logdir + "/bibsched.log","a")
log.write(time.strftime("%Y-%m-%d %H:%M:%S --> ", time.localtime()))
log.write(message)
log.write("\n")
log.close()
def redirect_stdout_and_stderr():
"This function redirects stdout and stderr to bibsched.log and bibsched.err file."
sys.stdout = open(logdir + "/bibsched.log", "a")
sys.stderr = open(logdir + "/bibsched.err", "a")
def usage(exitcode=1, msg=""):
"""Prints usage info."""
if msg:
sys.stderr.write("Error: %s.\n" % msg)
sys.stderr.write ("""\
Usage: %s [options] [start|stop|restart|monitor]
The following commands are available for bibsched:
- start: start bibsched in background
- stop: stop a running bibsched
- restart: restart a running bibsched
- monitor: enter the interactive monitor
Command options:
-d, --daemon\t Launch BibSched in the daemon mode (deprecated, use 'start')
General options:
-h, --help \t\t Print this help.
-V, --version \t\t Print version information.
""" % sys.argv [0])
#sys.stderr.write(" -v, --verbose=LEVEL \t Verbose level (0=min, 1=default, 9=max).\n")
sys.exit(exitcode)
pidfile = os.path.join(CFG_PREFIX, 'var', 'run', 'bibsched.pid')
def error (msg):
print >> sys.stderr, "error: " + msg
sys.exit (1)
def server_pid ():
# The pid must be stored on the filesystem
try:
pid = int (open (pidfile).read ())
except IOError:
return None
# Even if the pid is available, we check if it corresponds to an
# actual process, as it might have been killed externally
try:
os.kill (pid, signal.SIGCONT)
except OSError:
return None
return pid
def start (verbose = True):
""" Fork this process in the background and start processing
requests. The process PID is stored in a pid file, so that it can
be stopped later on."""
if verbose:
sys.stdout.write ("starting bibsched: ")
sys.stdout.flush ()
pid = server_pid ()
if pid:
error ("another instance of bibsched (pid %d) is running" % pid)
# start the child process using the "double fork" technique
pid = os.fork ()
if pid > 0: sys.exit (0)
os.setsid ()
os.chdir ('/')
pid = os.fork ()
if pid > 0:
if verbose:
sys.stdout.write ('pid %d\n' % pid)
Log ("daemon started (pid %d)" % pid)
open (pidfile, 'w').write ('%d' % pid)
return
sys.stdin.close ()
redirect_stdout_and_stderr ()
sched = BibSched()
sched.watch_loop ()
return
def stop (verbose = True):
pid = server_pid ()
if not pid:
error ('bibsched seems not to be running.')
try: os.kill (pid, signal.SIGKILL)
except OSError:
print >> sys.stderr, 'no bibsched process found'
Log ("daemon stopped (pid %d)" % pid)
if verbose: print "stopping bibsched: pid %d" % pid
os.unlink (pidfile)
return
def monitor(verbose = True):
redirect_stdout_and_stderr()
manager = Manager()
return
def restart(verbose = True):
stop(verbose)
start(verbose)
return
def main():
verbose = True
try:
opts, args = getopt.getopt(sys.argv[1:], "hVdq", [
"help","version","daemon", "quiet"])
except getopt.GetoptError, err:
Log ("Error: %s" % err)
usage(1, err)
for opt, arg in opts:
if opt in ["-h", "--help"]:
usage (0)
elif opt in ["-V", "--version"]:
print __revision__
sys.exit(0)
elif opt in ["-d", "--daemon"]:
redirect_stdout_and_stderr ()
sched = BibSched()
Log("daemon started")
sched.watch_loop()
elif opt in ['-q', '--quiet']:
verbose = False
else:
usage(1)
try: cmd = args [0]
except IndexError: cmd = 'monitor'
try:
{ 'start': start,
'stop': stop,
'restart': restart,
'monitor': monitor } [cmd] (verbose)
except KeyError:
usage (1, 'unkown command: %s' % `cmd`)
return
if __name__ == '__main__':
main()
diff --git a/modules/websession/bin/Makefile.am b/modules/websession/bin/Makefile.am
index 2a3f6b8b9..68e9fed80 100644
--- a/modules/websession/bin/Makefile.am
+++ b/modules/websession/bin/Makefile.am
@@ -1,24 +1,24 @@
## $Id$
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
+## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-bin_SCRIPTS = sessiongc
+bin_SCRIPTS = inveniogc
-EXTRA_DIST = sessiongc.in
+EXTRA_DIST = inveniogc.in
CLEANFILES = *~ *.tmp
\ No newline at end of file
diff --git a/modules/websession/bin/sessiongc.in b/modules/websession/bin/inveniogc.in
similarity index 96%
rename from modules/websession/bin/sessiongc.in
rename to modules/websession/bin/inveniogc.in
index 589a9a643..47ee91fda 100644
--- a/modules/websession/bin/sessiongc.in
+++ b/modules/websession/bin/inveniogc.in
@@ -1,36 +1,36 @@
#!@PYTHON@
## -*- mode: python; coding: utf-8; -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Guest user sessions garbage collector. To be run via cron once per day. (say)
"""
__revision__ = "$Id$"
try:
- from invenio.sessiongc import main
+ from invenio.inveniogc import main
except ImportError, e:
print "Error: %s" % e
import sys
sys.exit(1)
main()
diff --git a/modules/websession/lib/Makefile.am b/modules/websession/lib/Makefile.am
index 1b267f6f1..14fdec003 100644
--- a/modules/websession/lib/Makefile.am
+++ b/modules/websession/lib/Makefile.am
@@ -1,33 +1,33 @@
## $Id$
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
pylibdir = $(libdir)/python/invenio
pylib_DATA = websession.py session.py webuser.py webuser_tests.py \
websession_templates.py websession_webinterface.py \
webgroup.py webgroup_dblayer.py websession_config.py \
webaccount.py websession_regression_tests.py \
webgroup_regression_tests.py webuser_regression_tests.py \
- webgroup_tests.py sessiongc.py
+ webgroup_tests.py inveniogc.py
noinst_DATA = password_migration_kit.py
EXTRA_DIST = $(pylib_DATA) $(noinst_DATA)
CLEANFILES = *~ *.tmp *.pyc
diff --git a/modules/websession/lib/sessiongc.py b/modules/websession/lib/inveniogc.py
similarity index 65%
rename from modules/websession/lib/sessiongc.py
rename to modules/websession/lib/inveniogc.py
index 79e129ac5..c0b6ea4ec 100644
--- a/modules/websession/lib/sessiongc.py
+++ b/modules/websession/lib/inveniogc.py
@@ -1,506 +1,387 @@
## -*- mode: python; coding: utf-8; -*-
##
## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Guest user sessions garbage collector.
"""
__revision__ = "$Id$"
import sys
try:
from invenio.dbquery import run_sql
from invenio.config import logdir, tmpdir
from invenio.websession_config import CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS
from invenio.bibtask import task_init, task_set_option, task_get_option, \
write_message, write_messages
from invenio.access_control_mailcookie import mail_cookie_gc
+ from invenio.file import BibDoc
import time
import os
except ImportError, e:
print "Error: %s" % (e, )
sys.exit(1)
# configure variables
CFG_MYSQL_ARGUMENTLIST_SIZE = 100
# After how many days to remove obsolete log/err files
CFG_MAX_ATIME_RM_LOG = 28
# After how many days to zip obsolete log/err files
CFG_MAX_ATIME_ZIP_LOG = 7
# After how many days to remove obsolete bibreformat fmt xml files
CFG_MAX_ATIME_RM_FMT = 28
# After how many days to zip obsolete bibreformat fmt xml files
CFG_MAX_ATIME_ZIP_FMT = 7
# After how many days to remove obsolete bibharvest fmt xml files
CFG_MAX_ATIME_RM_OAI = 28
# After how many days to zip obsolete bibharvest fmt xml files
CFG_MAX_ATIME_ZIP_OAI = 7
+# After how many days to remove deleted bibdocs
+CFG_DELETED_BIBDOC_MAXLIFE = 365*10
def gc_exec_command(command):
""" Exec the command logging in appropriate way its output."""
write_message(' %s' % command, verbose=9)
(dontcare, output, errors) = os.popen3(command)
write_messages(errors.read())
write_messages(output.read())
-def clean_filesystem():
- """ Clean the filesystem from obsolete files. """
- write_message("""FILESYSTEM CLEANING STARTED""")
+def clean_logs():
+ """ Clean the logs from obsolete files. """
+ write_message("""LOGS CLEANING STARTED""")
write_message("- deleting/gzipping bibsched empty/old err/log "
"BibSched files")
vstr = task_get_option('verbose') > 1 and '-v' or ''
gc_exec_command('find %s -name "bibsched_task_*"'
' -size 0c -exec rm %s -f {} \;' \
% (logdir, vstr))
gc_exec_command('find %s -name "bibsched_task_*"'
' -atime +%s -exec rm %s -f {} \;' \
% (logdir, CFG_MAX_ATIME_RM_LOG, vstr))
gc_exec_command('find %s -name "bibsched_task_*"'
' -atime +%s -exec gzip %s -9 {} \;' \
% (logdir, CFG_MAX_ATIME_ZIP_LOG, vstr))
write_message("- deleting/gzipping temporary empty/old "
"BibReformat xml files")
gc_exec_command('find %s -name "rec_fmt_*"'
' -size 0c -exec rm %s -f {} \;' \
% (tmpdir, vstr))
gc_exec_command('find %s -name "rec_fmt_*"'
' -atime +%s -exec rm %s -f {} \;' \
% (tmpdir, CFG_MAX_ATIME_RM_FMT, vstr))
gc_exec_command('find %s -name "rec_fmt_*"'
' -atime +%s -exec gzip %s -9 {} \;' \
% (tmpdir, CFG_MAX_ATIME_ZIP_FMT, vstr))
write_message("- deleting/gzipping temporary old "
"BibHarvest xml files")
gc_exec_command('find %s -name "bibharvestadmin.*"'
' -exec rm %s -f {} \;' \
% (tmpdir, vstr))
gc_exec_command('find %s -name "bibconvertrun.*"'
' -exec rm %s -f {} \;' \
% (tmpdir, vstr))
gc_exec_command('find %s -name "oaiharvest*"'
' -atime +%s -exec gzip %s -9 {} \;' \
% (tmpdir, CFG_MAX_ATIME_ZIP_OAI, vstr))
gc_exec_command('find %s -name "oaiharvest*"'
' -atime +%s -exec rm %s -f {} \;' \
% (tmpdir, CFG_MAX_ATIME_RM_OAI, vstr))
gc_exec_command('find %s -name "oai_archive*"'
' -atime +%s -exec rm %s -f {} \;' \
% (tmpdir, CFG_MAX_ATIME_RM_OAI, vstr))
- write_message("""FILESYSTEM CLEANING FINISHED""")
-
+ write_message("""LOGS CLEANING FINISHED""")
+
+def clean_documents():
+ """Delete all the bibdocs that have been set as deleted and have not been
+ modified since CFG_DELETED_BIBDOC_MAXLIFE days. Returns the number of
+ bibdocs involved."""
+ write_message("""OBSOLETED DELETED DOCUMENTS CLEANING STARTED""")
+ write_message("select id from bibdoc where status='DELETED' and NOW()>ADDTIME(modification_date, '%s 0:0:0')" % CFG_DELETED_BIBDOC_MAXLIFE, verbose=9)
+ records = run_sql("select id from bibdoc where status='DELETED' and NOW()>ADDTIME(modification_date, '%s 0:0:0')" % CFG_DELETED_BIBDOC_MAXLIFE)
+ for record in records:
+ bibdoc = BibDoc(record[0])
+ bibdoc.expunge()
+ write_message("DELETE FROM bibdoc WHERE id=%i" % int(record[0]), verbose=9)
+ run_sql("DELETE FROM bibdoc WHERE id=%s", (record[0], ))
+ write_message("""%s Obsoleted deleted documents cleaned""" % len(records))
+ write_message("""OBSOLETED DELETED DOCUMENTS CLEANING FINISHED""")
+ return len(records)
def guest_user_garbage_collector():
"""Session Garbage Collector
program flow/tasks:
1: delete expired sessions
1b:delete guest users without session
2: delete queries not attached to any user
3: delete baskets not attached to any user
4: delete alerts not attached to any user
5: delete expired mailcookies
5b: delete expired not confirmed email address
6: delete expired roles memberships
verbose - level of program output.
0 - nothing
1 - default
9 - max, debug"""
# dictionary used to keep track of number of deleted entries
delcount = {'session': 0,
'user': 0,
'user_query': 0,
'query': 0,
'bskBASKET': 0,
'user_bskBASKET': 0,
'bskREC': 0,
'bskRECORDCOMMENT': 0,
'bskEXTREC': 0,
'bskEXTFMT': 0,
'user_query_basket': 0,
'mail_cookie': 0,
'email_addresses': 0,
'role_membership' : 0}
write_message("GUEST USER SESSIONS GARBAGE"
" COLLECTOR STARTED")
# 1 - DELETE EXPIRED SESSIONS
write_message("- deleting expired sessions")
timelimit = time.time()
write_message(" DELETE FROM session WHERE"
" session_expiry < %d \n" % (timelimit, ), verbose=9)
delcount['session'] += run_sql("DELETE FROM session WHERE"
" session_expiry < %s """ % (timelimit, ))
# 1b - DELETE GUEST USERS WITHOUT SESSION
write_message("- deleting guest users without session")
# get uids
write_message(""" SELECT u.id\n FROM user AS u LEFT JOIN session AS s\n ON u.id = s.uid\n WHERE s.uid IS NULL AND u.email = ''""", verbose=9)
result = run_sql("""SELECT u.id
FROM user AS u LEFT JOIN session AS s
ON u.id = s.uid
WHERE s.uid IS NULL AND u.email = ''""")
write_message(result, verbose=9)
if result:
# work on slices of result list in case of big result
for i in range(0, len(result), CFG_MYSQL_ARGUMENTLIST_SIZE):
# create string of uids
uidstr = ''
for (id_user, ) in result[i:i+CFG_MYSQL_ARGUMENTLIST_SIZE]:
if uidstr: uidstr += ','
uidstr += "%s" % (id_user, )
# delete users
write_message(" DELETE FROM user WHERE"
" id IN (TRAVERSE LAST RESULT) AND email = '' \n", verbose=9)
delcount['user'] += run_sql("DELETE FROM user WHERE"
" id IN (%s) AND email = ''" % (uidstr, ))
# 2 - DELETE QUERIES NOT ATTACHED TO ANY USER
# first step, delete from user_query
write_message("- deleting user_queries referencing"
" non-existent users")
# find user_queries referencing non-existent users
write_message(" SELECT DISTINCT uq.id_user\n"
" FROM user_query AS uq LEFT JOIN user AS u\n"
" ON uq.id_user = u.id\n WHERE u.id IS NULL", verbose=9)
result = run_sql("""SELECT DISTINCT uq.id_user
FROM user_query AS uq LEFT JOIN user AS u
ON uq.id_user = u.id
WHERE u.id IS NULL""")
write_message(result, verbose=9)
# delete in user_query one by one
write_message(" DELETE FROM user_query WHERE"
" id_user = 'TRAVERSE LAST RESULT' \n", verbose=9)
for (id_user, ) in result:
delcount['user_query'] += run_sql("""DELETE FROM user_query
WHERE id_user = %s""" % (id_user, ))
# delete the actual queries
write_message("- deleting queries not attached to any user")
# select queries that must be deleted
write_message(""" SELECT DISTINCT q.id\n FROM query AS q LEFT JOIN user_query AS uq\n ON uq.id_query = q.id\n WHERE uq.id_query IS NULL AND\n q.type <> 'p' """, verbose=9)
result = run_sql("""SELECT DISTINCT q.id
FROM query AS q LEFT JOIN user_query AS uq
ON uq.id_query = q.id
WHERE uq.id_query IS NULL AND
q.type <> 'p'""")
write_message(result, verbose=9)
# delete queries one by one
write_message(""" DELETE FROM query WHERE id = 'TRAVERSE LAST RESULT \n""", verbose=9)
for (id_user, ) in result:
delcount['query'] += run_sql("""DELETE FROM query WHERE id = %s""", (id_user, ))
# 3 - DELETE BASKETS NOT OWNED BY ANY USER
write_message("- deleting baskets not owned by any user")
# select basket ids
write_message(""" SELECT ub.id_bskBASKET\n FROM user_bskBASKET AS ub LEFT JOIN user AS u\n ON u.id = ub.id_user\n WHERE u.id IS NULL""", verbose=9)
try:
result = run_sql("""SELECT ub.id_bskBASKET
FROM user_bskBASKET AS ub LEFT JOIN user AS u
ON u.id = ub.id_user
WHERE u.id IS NULL""")
except:
result = []
write_message(result, verbose=9)
# delete from user_basket and basket one by one
write_message(""" DELETE FROM user_bskBASKET WHERE id_bskBASKET = 'TRAVERSE LAST RESULT' """, verbose=9)
write_message(""" DELETE FROM bskBASKET WHERE id = 'TRAVERSE LAST RESULT' """, verbose=9)
write_message(""" DELETE FROM bskREC WHERE id_bskBASKET = 'TRAVERSE LAST RESULT'""", verbose=9)
write_message(""" DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET = 'TRAVERSE LAST RESULT' \n""", verbose=9)
for (id_basket, ) in result:
delcount['user_bskBASKET'] += run_sql("""DELETE FROM user_bskBASKET WHERE id_bskBASKET = %s""", (id_basket, ))
delcount['bskBASKET'] += run_sql("""DELETE FROM bskBASKET WHERE id = %s""", (id_basket, ))
delcount['bskREC'] += run_sql("""DELETE FROM bskREC WHERE id_bskBASKET = %s""", (id_basket, ))
delcount['bskRECORDCOMMENT'] += run_sql("""DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET = %s""", (id_basket, ))
write_message(""" SELECT DISTINCT ext.id, rec.id_bibrec_or_bskEXTREC FROM bskEXTREC AS ext \nLEFT JOIN bskREC AS rec ON ext.id=-rec.id_bibrec_or_bskEXTREC WHERE id_bibrec_or_bskEXTREC is NULL""", verbose=9)
try:
result = run_sql("""SELECT DISTINCT ext.id FROM bskEXTREC AS ext
LEFT JOIN bskREC AS rec ON ext.id=-rec.id_bibrec_or_bskEXTREC
WHERE id_bibrec_or_bskEXTREC is NULL""")
except:
result = []
write_message(result, verbose=9)
write_message(""" DELETE FROM bskEXTREC WHERE id = 'TRAVERSE LAST RESULT' """, verbose=9)
write_message(""" DELETE FROM bskEXTFMT WHERE id_bskEXTREC = 'TRAVERSE LAST RESULT' \n""", verbose=9)
for (id_basket, ) in result:
delcount['bskEXTREC'] += run_sql("""DELETE FROM bskEXTREC WHERE id=%s""", (id_basket,))
delcount['bskEXTFMT'] += run_sql("""DELETE FROM bskEXTFMT WHERE id_bskEXTREC=%s""", (id_basket,))
# 4 - DELETE ALERTS NOT OWNED BY ANY USER
write_message('- deleting alerts not owned by any user')
# select user ids in uqb that reference non-existent users
write_message("""SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL""", verbose=9)
result = run_sql("""SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL""")
write_message(result, verbose=9)
# delete all these entries
for (id_user, ) in result:
write_message("""DELETE FROM user_query_basket WHERE id_user = 'TRAVERSE LAST RESULT """, verbose=9)
delcount['user_query_basket'] += run_sql("""DELETE FROM user_query_basket WHERE id_user = %s """, (id_user, ))
# 5 - delete expired mailcookies
write_message("""mail_cookie_gc()""", verbose=9)
delcount['mail_cookie'] = mail_cookie_gc()
- # 5b - delete expired not confirmed email address
+ ## 5b - delete expired not confirmed email address
write_message("""DELETE FROM user WHERE note='2' AND NOW()>ADDTIME(last_login, '%s 0:0:0')""" % CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS, verbose=9)
delcount['email_addresses'] = run_sql("""DELETE FROM user WHERE note='2' AND NOW()>ADDTIME(last_login, '%s 0:0:0')""" % CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS)
# 6 - delete expired roles memberships
write_message("""DELETE FROM user_accROLE WHERE expiration 0:
types = listTypesFromArray(bibdocs)
fulltypes = []
for mytype in types:
fulltype = {
'name' : mytype,
'content' : [],
}
for bibdoc in bibdocs:
if mytype == bibdoc.getType():
fulltype['content'].append(bibdoc.display(version,
ln = ln))
fulltypes.append(fulltype)
t = websubmit_templates.tmpl_bibrecdoc_filelist(
ln = ln,
types = fulltypes,
)
return t
class BibDoc:
"""this class represents one file attached to a record
there is a one to one mapping between an instance of this class and
an entry in the bibdoc db table"""
def __init__ (self, bibdocid="", recid="", docname="file", type="Main"):
# bibdocid is known, the document already exists
if bibdocid != "":
if recid == "":
recid = None
self.type = ""
res = run_sql("select id_bibrec,type from bibrec_bibdoc "
"where id_bibdoc=%s", (bibdocid,))
if len(res) > 0:
recid = res[0][0]
self.type = res[0][1]
else:
res = run_sql("select id_bibdoc1 from bibdoc_bibdoc "
"where id_bibdoc2=%s", (bibdocid,))
if len(res) > 0 :
main_bibdoc = res[0][0]
res = run_sql("select id_bibrec,type from bibrec_bibdoc "
"where id_bibdoc=%s", (main_bibdoc,))
if len(res) > 0:
recid = res[0][0]
self.type = res[0][1]
else:
res = run_sql("select type from bibrec_bibdoc "
"where id_bibrec=%s and id_bibdoc=%s", (recid, bibdocid,))
if len(res) > 0:
self.type = res[0][0]
else:
#this bibdoc isn't associated with the corresponding bibrec.
raise StandardError, "No docid associated with the recid %s" % recid
# gather the other information
res = run_sql("select id,status,docname,creation_date,"
"modification_date from bibdoc where id=%s", (bibdocid,))
if len(res) > 0:
self.cd = res[0][3]
self.md = res[0][4]
self.recid = recid
self.docname = res[0][2]
self.id = bibdocid
self.status = res[0][1]
group = "g" + str(int(int(self.id) / filedirsize))
self.basedir = "%s/%s/%s" % (filedir, group, self.id)
else:
# this bibdoc doesn't exist
raise StandardError, "The docid %s does not exist." % bibdocid
# else it is a new document
else:
if docname == "" or type == "":
raise StandardError, "Argument missing for creating a new bibdoc"
else:
self.recid = recid
self.type = type
self.docname = docname
self.status = ''
res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (recid, docname))
if res:
raise StandardError, "A bibdoc called %s already exists for recid %s" % (docname, recid)
self.id = run_sql("insert into bibdoc "
"(status,docname,creation_date,modification_date) "
"values(%s,%s,NOW(),NOW())", (self.status, docname,))
if self.id is not None:
# we link the document to the record if a recid was
# specified
if self.recid != "":
run_sql("insert into bibrec_bibdoc values(%s,%s,%s)",
(recid, self.id, self.type,))
else:
raise StandardError, "New docid cannot be created"
group = "g" + str(int(int(self.id) / filedirsize))
self.basedir = "%s/%s/%s" % (filedir, group, self.id)
# we create the corresponding storage directory
if not os.path.exists(self.basedir):
old_umask = os.umask(022)
os.makedirs(self.basedir)
# and save the father record id if it exists
if self.recid != "":
fp = open("%s/.recid" % self.basedir, "w")
fp.write(str(self.recid))
fp.close()
if self.type != "":
fp = open("%s/.type" % self.basedir, "w")
fp.write(str(self.type))
fp.close()
os.umask(old_umask)
# build list of attached files
self.BuildFileList()
# link with relatedFiles
self.BuildRelatedFileList()
def getStatus(self):
"""Retrieve the status."""
return self.status
+ def touch(self):
+ """Update the modification time of the bibdoc."""
+ run_sql('UPDATE bibdoc SET modification_date=NOW() WHERE id=%s', self.id)
+
def setStatus(self, new_status):
"""Set a new status."""
- run_sql('UPDATE bibdoc SET status=%s WHERE id=%s', (new_status, id))
+ run_sql('UPDATE bibdoc SET status=%s WHERE id=%s', (new_status, self.id))
self.BuildFileList()
self.BuildRelatedFileList()
def addFilesNewVersion(self, files=[]):
"""add a new version of a file to an archive"""
latestVersion = self.getLatestVersion()
if latestVersion == "0":
myversion = "1"
else:
myversion = str(int(latestVersion) + 1)
for file in files:
if os.path.exists(file):
dummy, basename, extension = decompose_file(file)
if extension:
extension = '.' + extension
#self.changeName(basename)
#destination = propose_unique_name("%s/%s%s;%s" %
#(self.basedir, basename, extension, myversion), True)
destination = "%s/%s%s;%s" % (self.basedir, self.docname, extension, myversion)
shutil.copyfile(file, destination)
os.chmod(destination, 0644)
+ self.touch()
self.BuildFileList()
def purge(self):
"""Phisically Remove all the previous version of the given bibdoc"""
version = self.getLatestVersion()
if version > 1:
for file in self.docfiles:
if file.getVersion() < version:
os.remove(file.getFullPath())
Md5Folder(self.basedir).update()
+ self.touch()
self.BuildFileList()
self.BuildRelatedFileList()
def expunge(self):
"""Phisically remove all the traces of a given bibdoc"""
for file in self.docfiles:
os.remove(file.getFullPath())
Md5Folder(self.basedir).update()
+ self.touch()
self.BuildFileList()
self.BuildRelatedFileList()
def addFilesNewFormat(self, files=[], version=""):
"""add a new format of a file to an archive"""
if version == "":
version = self.getLatestVersion()
for file in files:
if os.path.exists(file):
dummy, basename, extension = decompose_file(file)
if extension:
extension = '.' + extension
destination = "%s/%s%s;%s" % (self.basedir, self.docname, extension, version)
if os.path.exists(destination):
raise StandardError, "A file for docname '%s' for the recid '%s' already exists for the format '%s'" % (self.docname, self.recid, extension)
#destination = propose_unique_name("%s/%s%s;%s" %
#(self.basedir, basename, extension, version), True)
shutil.copyfile(file, destination)
os.chmod(destination, 0644)
+ self.touch()
self.BuildFileList()
def getIcon(self):
if self.relatedFiles.has_key('Icon'):
return self.relatedFiles['Icon'][0]
else:
return None
def addIcon(self, file, basename=''):
"""link an icon with the bibdoc object"""
#first check if an icon already exists
existingIcon = self.getIcon()
if existingIcon is not None:
existingIcon.delete()
#then add the new one
if not basename:
basename = decompose_file(file)[1]
try:
newicon = BibDoc(type='Icon', docname=basename)
except StandardError:
pass
else:
newicon.addFilesNewVersion([file])
run_sql("insert into bibdoc_bibdoc values(%s,%s,'Icon')",
(self.id, newicon.getId(),))
if os.path.exists(newicon.getBaseDir()):
old_umask = os.umask(022)
fp = open("%s/.docid" % newicon.getBaseDir(), "w")
fp.write(str(self.id))
fp.close()
fp = open("%s/.type" % newicon.getBaseDir(), "w")
fp.write(str(self.type))
fp.close()
os.umask(old_umask)
+ self.touch()
self.BuildRelatedFileList()
def deleteIcon(self):
existingIcon = self.getIcon()
if existingIcon is not None:
existingIcon.delete()
+ self.touch()
self.BuildRelatedFileList()
def display(self, version="", ln = cdslang):
t = ""
if version == "all":
docfiles = self.listAllFiles()
elif version != "":
docfiles = self.listVersionFiles(version)
else:
docfiles = self.listLatestFiles()
existingIcon = self.getIcon()
if existingIcon is not None:
imagepath = "%s/getfile.py?docid=%s&name=%s&format=gif" % \
(weburl, existingIcon.getId(),
urllib.quote(existingIcon.getDocName()))
else:
imagepath = "%s/smallfiles.gif" % images
versions = []
for version in listVersionsFromArray(docfiles):
currversion = {
'version' : version,
'previous' : 0,
'content' : []
}
if version == self.getLatestVersion() and version != "1":
currversion['previous'] = 1
for docfile in docfiles:
if docfile.getVersion() == version:
currversion['content'].append(docfile.display(ln = ln))
versions.append(currversion)
t = websubmit_templates.tmpl_bibdoc_filelist(
ln = ln,
weburl = weburl,
versions = versions,
imagepath = imagepath,
docname = self.docname,
id = self.id,
recid = self.recid
)
return t
def changeName(self, newname):
"""Rename the bibdoc name. New name must not be already used by the linked
bibrecs."""
- res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (recid, docname))
+ res = run_sql("SELECT b.id FROM bibrec_bibdoc bb JOIN bibdoc b on bb.id_bibdoc=b.id WHERE bb.id_bibrec=%s AND b.docname=%s", (self.recid, self.docname))
if res:
- raise StandardError, "A bibdoc called %s already exists for recid %s" % (docname, recid)
+ raise StandardError, "A bibdoc called %s already exists for recid %s" % (self.docname, self.recid)
run_sql("update bibdoc set docname=%s where id=%s", (newname, self.id,))
for f in os.listdir(self.basedir):
if f.startswith(self.docname):
- sh.move('%s/%s' % (self.basedir, f), '%s/%s' % (self.basedir, f.replace(self.docname, newname, 1)))
+ shutil.move('%s/%s' % (self.basedir, f), '%s/%s' % (self.basedir, f.replace(self.docname, newname, 1)))
self.docname = newname
Md5Folder(self.basedir).update()
+ self.touch()
self.BuildFileList()
self.BuildRelatedFileList()
def getDocName(self):
"""retrieve bibdoc name"""
return self.docname
def getBaseDir(self):
"""retrieve bibdoc base directory"""
return self.basedir
def getType(self):
"""retrieve bibdoc type"""
return self.type
def getRecid(self):
"""retrieve bibdoc recid"""
return self.recid
def getId(self):
"""retrieve bibdoc id"""
return self.id
def getFile(self, name, format, version):
if version == "":
docfiles = self.listLatestFiles()
else:
docfiles = self.listVersionFiles(version)
for docfile in docfiles:
if docfile.getName()==name and (docfile.getFormat()==format or not format):
return docfile
raise StandardError, "No file called '%s' of format '%s', version '%s'" % (name, format, version)
def listVersions(self):
versions = []
for docfile in self.docfiles:
if not docfile.getVersion() in versions:
versions.append(docfile.getVersion())
return versions
def delete(self):
"""delete the current bibdoc instance"""
run_sql("update bibdoc set status='DELETED' where id=%s", (self.id,))
def BuildFileList(self):
"""lists all files attached to the bibdoc"""
self.docfiles = []
if os.path.exists(self.basedir):
self.md5s = Md5Folder(self.basedir)
for fil in os.listdir(self.basedir):
if fil not in (".recid", ".docid", ".", "..",
'.doc_checksum', '.type'):
filepath = "%s/%s" % (self.basedir, fil)
fileversion = re.sub(".*;", "", fil)
fullname = fil.replace(";%s" % fileversion, "")
checksum = self.md5s.get_checksum(fil)
(dirname, basename, extension) = decompose_file(fullname)
# we can append file:
self.docfiles.append(BibDocFile(filepath, self.type,
fileversion, basename, extension,
self.id, self.status, checksum))
def BuildRelatedFileList(self):
self.relatedFiles = {}
res = run_sql("select ln.id_bibdoc2,ln.type,bibdoc.status from "
"bibdoc_bibdoc as ln,bibdoc where id=ln.id_bibdoc2 and "
"ln.id_bibdoc1=%s", (self.id,))
for row in res:
bibdocid = row[0]
type = row[1]
if row[2] != 'DELETED':
if not self.relatedFiles.has_key(type):
self.relatedFiles[type] = []
try:
cur_doc = BibDoc(bibdocid=bibdocid)
self.relatedFiles[type].append(cur_doc)
except StandardError:
pass
def listAllFiles(self):
return self.docfiles
def listLatestFiles(self):
return self.listVersionFiles(self.getLatestVersion())
def listVersionFiles(self, version):
tmp = []
for docfile in self.docfiles:
if docfile.getVersion() == str(version):
tmp.append(docfile)
return tmp
def getLatestVersion(self):
if len(self.docfiles) > 0:
self.docfiles.sort(orderFilesWithVersion)
return self.docfiles[0].getVersion()
else:
return 0
def getFileNumber(self):
return len(self.docfiles)
def registerDownload(self, addressIp, version, format, userid=0):
return run_sql("INSERT INTO rnkDOWNLOADS "
"(id_bibrec,id_bibdoc,file_version,file_format,"
"id_user,client_host,download_time) VALUES "
"(%s,%s,%s,%s,%s,INET_ATON(%s),NOW())",
(self.recid, self.id, version, format.upper(),
userid, addressIp,))
class BibDocFile:
"""this class represents a physical file in the CDS Invenio filesystem"""
def __init__(self, fullpath, type, version, name, format, bibdocid, status, checksum):
self.fullpath = fullpath
self.type = type
self.bibdocid = bibdocid
self.version = version
self.status = status
self.checksum = checksum
self.size = os.path.getsize(fullpath)
self.md = os.path.getmtime(fullpath)
try:
self.cd = os.path.getctime(fullpath)
except:
self.cd = self.md
self.name = name
self.format = format
self.dir = os.path.dirname(fullpath)
if format == "":
self.mime = "text/plain"
self.encoding = ""
self.fullname = name
else:
self.fullname = "%s.%s" % (name, format)
(self.mime, self.encoding) = _mimes.guess_type(self.fullname)
if self.mime is None:
self.mime = "text/plain"
def display(self, ln = cdslang):
if self.format != "":
format = ".%s" % self.format
else:
format = ""
return websubmit_templates.tmpl_bibdocfile_filelist(
ln = ln,
weburl = weburl,
id = self.bibdocid,
selfformat = self.format,
version = self.version,
name = self.name,
format = format,
size = self.size,
)
def isRestricted(self):
"""return restriction state"""
return self.status != ''
def getType(self):
return self.type
def getPath(self):
return self.fullpath
def getBibDocId(self):
return self.bibdocid
def getName(self):
return self.name
def getFullName(self):
return self.fullname
def getFullPath(self):
return self.fullpath
def getFormat(self):
return self.format
def getSize(self):
return self.size
def getVersion(self):
return self.version
def getChecksum(self):
return self.checksum
def getRecid(self):
return run_sql("select id_bibrec from bibrec_bibdoc where "
"id_bibdoc=%s",(self.bibdocid,))[0][0]
def stream(self, req):
if self.status:
(auth_code, auth_message) = acc_authorize_action(req, 'viewrestrdoc', status=self.status)
else:
auth_code = 0
if auth_code == 0:
if os.path.exists(self.fullpath):
req.content_type = self.mime
req.encoding = self.encoding
req.filename = self.fullname
req.headers_out["Content-Disposition"] = \
"attachment; filename=%s" % quoteattr(self.fullname)
req.send_http_header()
fp = file(self.fullpath, "r")
content = fp.read()
fp.close()
return content
else:
raise StandardError, "%s does not exists!" % self.fullpath
else:
raise StandardError, "You are not authorized to download %s: %s" % (self.fullname, auth_message)
def readfile(path):
"""Read the contents of a text file and return them as a string.
Return an empty string if unable to access the file for reading.
@param path: (string) - path to the file
@return: (string) contents of file or empty string
"""
content = ""
if os.access(path, os.F_OK|os.R_OK):
try:
fp = open(path, "r")
except IOError:
pass
else:
content = fp.read()
fp.close()
return content
def listTypesFromArray(bibdocs):
types = []
for bibdoc in bibdocs:
if not bibdoc.getType() in types:
types.append(bibdoc.getType())
return types
def listVersionsFromArray(docfiles):
versions = []
for docfile in docfiles:
if not docfile.getVersion() in versions:
versions.append(docfile.getVersion())
return versions
def orderFilesWithVersion(docfile1, docfile2):
"""order docfile objects according to their version"""
version1 = int(docfile1.getVersion())
version2 = int(docfile2.getVersion())
return cmp(version2, version1)
class Md5Folder:
"""Manage all the Md5 checksum about a folder"""
def __init__(self, folder):
"""Initialize the class from the md5 checksum of a given path"""
self.folder = folder
self.load()
def update(self, only_new = True):
"""Update the .doc_checksum file with the current files. If only_new
is specified then only not already calculated file are calculated."""
self.md5s = {}
for filename in os.listdir(self.folder):
if not only_new or self.md5s.get(filename, None) is None and \
not filename.startswith('.'):
self.md5s[filename] = md5.new(open("%s/%s" %
(self.folder, filename), "rb").read()).hexdigest()
self.store()
def store(self):
"""Store the current md5 dictionary into .doc_checksum"""
old_umask = os.umask(022)
md5file = open("%s/.doc_checksum" % self.folder, "w")
for key, value in self.md5s.items():
md5file.write('%s *%s\n' % (value, key))
os.umask(old_umask)
def load(self):
"""Load .doc_checksum into the md5 dictionary"""
self.md5s = {}
try:
for row in open("%s/.doc_checksum" % self.folder, "r"):
md5hash = row[:32]
filename = row[34:].strip()
self.md5s[filename] = md5hash
except IOError:
pass
def check(self, filename = ''):
"""Check the specified file or all the files for which it exists a hash
for being coherent with the stored hash."""
if filename and filename in self.md5s.keys():
return self.md5s[filename] == md5.new(open("%s/%s" %
(self.folder, filename), "rb").read()).hexdigest()
else:
for filename, md5hash in self.md5s.items():
if md5.new(open("%s/%s" % (self.folder, filename),
"rb").read()).hexdigest() != md5hash:
return False
return True
def get_checksum(self, filename):
md5hash = self.md5s.get(filename, None)
if md5hash is None:
self.update()
# Now it should not fail!
md5hash = self.md5s[filename]
return md5hash