get value of a given option from conf files
--conf-dir path to directory where invenio*.conf files are [optional]
--detect-system-details print system details such as Apache/Python/MySQL versions
"""
__revision__ = "$Id$"
from ConfigParser import ConfigParser
import os
import re
import shutil
import socket
import sys
def print_usage():
"""Print help."""
print __doc__
def print_version():
"""Print version information."""
print __revision__
def convert_conf_option(option_name, option_value):
"""
Convert conf option into Python config.py line, converting
values to ints or strings as appropriate.
"""
## 1) convert option name to uppercase:
option_name = option_name.upper()
## 2) convert option value to int or string:
if option_name in ['CFG_BIBUPLOAD_REFERENCE_TAG',
'CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS',]:
# some options are supposed be string even when they look like
# numeric
option_value = '"' + option_value + '"'
else:
try:
option_value = int(option_value)
except ValueError:
option_value = '"' + option_value + '"'
## 3a) special cases: chars regexps
if option_name in ['CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS',
'CFG_BIBINDEX_CHARS_PUNCTUATION']:
option_value = 'r"[' + option_value[1:-1] + ']"'
## 3abis) special cases: real regexps
if option_name in ['CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES',
'CFG_BIBINDEX_SPLASH_PAGES']:
option_value = 'r"' + option_value[1:-1] + '"'
## 3b) special cases: True, False, None
if option_value in ['"True"', '"False"', '"None"']:
option_value = option_value[1:-1]
- ## 3c) special cases: dicts
+ ## 3c) special cases: dicts and real pythonic lists
if option_name in ['CFG_WEBSEARCH_FIELDS_CONVERT',
'CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS',
- 'CFG_SITE_EMERGENCY_EMAIL_ADDRESSES']:
+ 'CFG_SITE_EMERGENCY_EMAIL_ADDRESSES',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS']:
option_value = option_value[1:-1]
## 3d) special cases: comma-separated lists
if option_name in ['CFG_SITE_LANGS',
'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS',
'CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS',
'CFG_BIBSCHED_GC_TASKS_TO_REMOVE',
'CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE',
'CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS',
'CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS',
'CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES',
'CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST',
'CFG_WEBSEARCH_RSS_I18N_COLLECTIONS',
'CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY',
'CFG_BATCHUPLOADER_WEB_ROBOT_AGENT']:
out = "["
for elem in option_value[1:-1].split(","):
if elem:
if option_name in ['CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES']:
# 3d1) integer values
out += "%i, " % int(elem)
else:
# 3d2) string values
out += "'%s', " % elem
out += "]"
option_value = out
## 3e) special cases: multiline
if option_name == 'CFG_OAI_IDENTIFY_DESCRIPTION':
# make triple quotes
option_value = '""' + option_value + '""'
## 3f) ignore some options:
if option_name.startswith('CFG_SITE_NAME_INTL'):
# treated elsewhere
return
## 3g) special cases: float
if option_name == 'CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY':
option_value = float(option_value[1:-1])
## 4) finally, return output line:
return '%s = %s' % (option_name, option_value)
def cli_cmd_update_config_py(conf):
"""
Update new config.py from conf options, keeping previous
config.py in a backup copy.
"""
print ">>> Going to update config.py..."
## location where config.py is:
configpyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'config.py'
## backup current config.py file:
if os.path.exists(configpyfile):
shutil.copy(configpyfile, configpyfile + '.OLD')
## here we go:
fdesc = open(configpyfile, 'w')
## generate preamble:
fdesc.write("# -*- coding: utf-8 -*-\n")
fdesc.write("# DO NOT EDIT THIS FILE! IT WAS AUTOMATICALLY GENERATED\n")
fdesc.write("# FROM INVENIO.CONF BY EXECUTING:\n")
fdesc.write("# " + " ".join(sys.argv) + "\n")
## special treatment for CFG_SITE_NAME_INTL options:
fdesc.write("CFG_SITE_NAME_INTL = {}\n")
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
fdesc.write("CFG_SITE_NAME_INTL['%s'] = \"%s\"\n" % (lang, conf.get("Invenio",
"CFG_SITE_NAME_INTL_" + lang)))
## special treatment for CFG_SITE_SECURE_URL that may be empty, in
## which case it should be put equal to CFG_SITE_URL:
if not conf.get("Invenio", "CFG_SITE_SECURE_URL"):
conf.set("Invenio", "CFG_SITE_SECURE_URL",
conf.get("Invenio", "CFG_SITE_URL"))
## process all the options normally:
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
if not option.startswith('CFG_DATABASE_'):
# put all options except for db credentials into config.py
line_out = convert_conf_option(option, conf.get(section, option))
if line_out:
fdesc.write(line_out + "\n")
## FIXME: special treatment for experimental variables
## CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES and CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE
## (not offering them in invenio.conf since they will be refactored)
fdesc.write("CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE = 0\n")
fdesc.write("CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES = [0, 1,]\n")
## generate postamble:
fdesc.write("")
fdesc.write("# END OF GENERATED FILE")
## we are done:
fdesc.close()
print "You may want to restart Apache now."
print ">>> config.py updated successfully."
def cli_cmd_update_dbquery_py(conf):
"""
Update lib/dbquery.py file with DB parameters read from conf file.
Note: this edits dbquery.py in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbquery.py..."
## location where dbquery.py is:
dbquerypyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'dbquery.py'
## backup current dbquery.py file:
if os.path.exists(dbquerypyfile):
shutil.copy(dbquerypyfile, dbquerypyfile + '.OLD')
## replace db parameters:
out = ''
for line in open(dbquerypyfile, 'r').readlines():
match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS)(\s*=\s*)\'.*\'$', line)
if match:
dbparam = 'CFG_DATABASE_' + match.group(1)
out += "%s%s'%s'\n" % (dbparam, match.group(2),
conf.get('Invenio', dbparam))
else:
out += line
fdesc = open(dbquerypyfile, 'w')
fdesc.write(out)
fdesc.close()
print "You may want to restart Apache now."
print ">>> dbquery.py updated successfully."
def cli_cmd_update_dbexec(conf):
"""
Update bin/dbexec file with DB parameters read from conf file.
Note: this edits dbexec in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbexec..."
## location where dbexec is:
dbexecfile = conf.get("Invenio", "CFG_BINDIR") + \
os.sep + 'dbexec'
## backup current dbexec file:
if os.path.exists(dbexecfile):
shutil.copy(dbexecfile, dbexecfile + '.OLD')
## replace db parameters via sed:
out = ''
for line in open(dbexecfile, 'r').readlines():
match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS)(\s*=\s*)\'.*\'$', line)
if match:
dbparam = 'CFG_DATABASE_' + match.group(1)
out += "%s%s'%s'\n" % (dbparam, match.group(2),
conf.get("Invenio", dbparam))
else:
out += line
fdesc = open(dbexecfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> dbexec updated successfully."
def cli_cmd_update_bibconvert_tpl(conf):
"""
Update bibconvert/config/*.tpl files looking for 856
http://.../record/ lines, replacing URL with CFG_SITE_URL taken
from conf file. Note: this edits tpl files in situ, taking a
backup first. Use only when you know what you are doing.
"""
print ">>> Going to update bibconvert templates..."
## location where bibconvert/config/*.tpl are:
tpldir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'bibconvert' + os.sep + 'config'
## find all *.tpl files:
for tplfilename in os.listdir(tpldir):
if tplfilename.endswith(".tpl"):
## change tpl file:
tplfile = tpldir + os.sep + tplfilename
shutil.copy(tplfile, tplfile + '.OLD')
out = ''
for line in open(tplfile, 'r').readlines():
match = re.search(r'^(.*)http://.*?/record/(.*)$', line)
if match:
out += "%s%s/record/%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_SITE_URL'),
match.group(2))
else:
out += line
fdesc = open(tplfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> bibconvert templates updated successfully."
def cli_cmd_update_web_tests(conf):
"""
Update web test cases lib/webtest/test_*.html looking for
http://.+?[] strings and replacing them with CFG_SITE_URL
taken from conf file. Note: this edits test files in situ, taking
a backup first. Use only when you know what you are doing.
"""
print ">>> Going to update web tests..."
## location where test_*.html files are:
testdir = conf.get("Invenio", 'CFG_PREFIX') + os.sep + \
'lib' + os.sep + 'webtest' + os.sep + 'invenio'
## find all test_*.html files:
for testfilename in os.listdir(testdir):
if testfilename.startswith("test_") and \
testfilename.endswith(".html"):
## change test file:
testfile = testdir + os.sep + testfilename
shutil.copy(testfile, testfile + '.OLD')
out = ''
for line in open(testfile, 'r').readlines():
match = re.search(r'^(.* | )http://.+?([].*)$', line)
if match:
out += "%s%s%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_SITE_URL'),
match.group(2))
else:
match = re.search(r'^(.* | )/opt/invenio(.*)$', line)
if match:
out += "%s%s%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_PREFIX'),
match.group(2))
else:
out += line
fdesc = open(testfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> web tests updated successfully."
def cli_cmd_reset_sitename(conf):
"""
Reset collection-related tables with new CFG_SITE_NAME and
CFG_SITE_NAME_INTL* read from conf files.
"""
print ">>> Going to reset CFG_SITE_NAME and CFG_SITE_NAME_INTL..."
from invenio.dbquery import run_sql, IntegrityError
# reset CFG_SITE_NAME:
sitename = conf.get("Invenio", "CFG_SITE_NAME")
try:
run_sql("""INSERT INTO collection (id, name, dbquery, reclist) VALUES
(1,%s,NULL,NULL)""", (sitename,))
except IntegrityError:
run_sql("""UPDATE collection SET name=%s WHERE id=1""", (sitename,))
# reset CFG_SITE_NAME_INTL:
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
sitename_lang = conf.get("Invenio", "CFG_SITE_NAME_INTL_" + lang)
try:
run_sql("""INSERT INTO collectionname (id_collection, ln, type, value) VALUES
(%s,%s,%s,%s)""", (1, lang, 'ln', sitename_lang))
except IntegrityError:
run_sql("""UPDATE collectionname SET value=%s
WHERE ln=%s AND id_collection=1 AND type='ln'""",
(sitename_lang, lang))
print "You may want to restart Apache now."
print ">>> CFG_SITE_NAME and CFG_SITE_NAME_INTL* reset successfully."
def cli_cmd_reset_recstruct_cache(conf):
"""If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the recstruct
format."""
from invenio.intbitset import intbitset
from invenio.dbquery import run_sql, serialize_via_marshal
from invenio.search_engine import get_record
from invenio.bibsched import server_pid, pidfile
enable_recstruct_cache = conf.get("Invenio", "CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
enable_recstruct_cache = enable_recstruct_cache in ('True', '1')
pid = server_pid(ping_the_process=False)
if pid:
print >> sys.stderr, "ERROR: bibsched seems to run with pid %d, according to %s." % (pid, pidfile)
print >> sys.stderr, " Please stop bibsched before running this procedure."
sys.exit(1)
if enable_recstruct_cache:
print ">>> Searching records which need recstruct cache resetting; this may take a while..."
all_recids = intbitset(run_sql("SELECT id FROM bibrec"))
good_recids = intbitset(run_sql("SELECT bibrec.id FROM bibrec JOIN bibfmt ON bibrec.id = bibfmt.id_bibrec WHERE format='recstruct' AND modification_date < last_updated"))
recids = all_recids - good_recids
print ">>> Generating recstruct cache..."
tot = len(recids)
count = 0
for recid in recids:
value = serialize_via_marshal(get_record(recid))
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='recstruct'", (recid, ))
run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'recstruct', NOW(), %s)", (recid, value))
count += 1
if count % 1000 == 0:
print " ... done records %s/%s" % (count, tot)
if count % 1000 != 0:
print " ... done records %s/%s" % (count, tot)
print ">>> recstruct cache generated successfully."
else:
print ">>> Cleaning recstruct cache..."
run_sql("DELETE FROM bibfmt WHERE format='recstruct'")
def cli_cmd_reset_siteadminemail(conf):
"""
Reset user-related tables with new CFG_SITE_ADMIN_EMAIL read from conf files.
"""
print ">>> Going to reset CFG_SITE_ADMIN_EMAIL..."
from invenio.dbquery import run_sql
siteadminemail = conf.get("Invenio", "CFG_SITE_ADMIN_EMAIL")
run_sql("DELETE FROM user WHERE id=1")
run_sql("""INSERT INTO user (id, email, password, note, nickname) VALUES
(1, %s, AES_ENCRYPT(email, ''), 1, 'admin')""",
(siteadminemail,))
print "You may want to restart Apache now."
print ">>> CFG_SITE_ADMIN_EMAIL reset successfully."
def cli_cmd_reset_fieldnames(conf):
"""
Reset I18N field names such as author, title, etc and other I18N
ranking method names such as word similarity. Their translations
are taken from the PO files.
"""
print ">>> Going to reset I18N field names..."
from invenio.messages import gettext_set_language, language_list_long
from invenio.dbquery import run_sql, IntegrityError
## get field id and name list:
field_id_name_list = run_sql("SELECT id, name FROM field")
## get rankmethod id and name list:
rankmethod_id_name_list = run_sql("SELECT id, name FROM rnkMETHOD")
## update names for every language:
for lang, dummy in language_list_long():
_ = gettext_set_language(lang)
## this list is put here in order for PO system to pick names
## suitable for translation
field_name_names = {"any field": _("any field"),
"title": _("title"),
"author": _("author"),
"abstract": _("abstract"),
"keyword": _("keyword"),
"report number": _("report number"),
"subject": _("subject"),
"reference": _("reference"),
"fulltext": _("fulltext"),
"collection": _("collection"),
"division": _("division"),
"year": _("year"),
"journal": _("journal"),
"experiment": _("experiment"),
"record ID": _("record ID")}
## update I18N names for every language:
for (field_id, field_name) in field_id_name_list:
if field_name_names.has_key(field_name):
try:
run_sql("""INSERT INTO fieldname (id_field,ln,type,value) VALUES
(%s,%s,%s,%s)""", (field_id, lang, 'ln',
field_name_names[field_name]))
except IntegrityError:
run_sql("""UPDATE fieldname SET value=%s
WHERE id_field=%s AND ln=%s AND type=%s""",
(field_name_names[field_name], field_id, lang, 'ln',))
## ditto for rank methods:
rankmethod_name_names = {"wrd": _("word similarity"),
"demo_jif": _("journal impact factor"),
"citation": _("times cited"),
"citerank_citation_t": _("time-decay cite count"),
"citerank_pagerank_c": _("all-time-best cite rank"),
"citerank_pagerank_t": _("time-decay cite rank"),}
for (rankmethod_id, rankmethod_name) in rankmethod_id_name_list:
if rankmethod_name_names.has_key(rankmethod_name):
try:
run_sql("""INSERT INTO rnkMETHODNAME (id_rnkMETHOD,ln,type,value) VALUES
(%s,%s,%s,%s)""", (rankmethod_id, lang, 'ln',
rankmethod_name_names[rankmethod_name]))
except IntegrityError:
run_sql("""UPDATE rnkMETHODNAME SET value=%s
WHERE id_rnkMETHOD=%s AND ln=%s AND type=%s""",
(rankmethod_name_names[rankmethod_name], rankmethod_id, lang, 'ln',))
print ">>> I18N field names reset successfully."
def cli_check_openoffice_dir(conf):
"""
If OpenOffice.org integration is enabled, checks whether the system is
properly configured.
"""
from invenio.textutils import wrap_text_in_a_box
from invenio.websubmit_file_converter import check_openoffice_tmpdir, \
InvenioWebSubmitFileConverterError, CFG_OPENOFFICE_TMPDIR
from invenio.config import CFG_OPENOFFICE_USER, \
CFG_PATH_OPENOFFICE_PYTHON, \
CFG_OPENOFFICE_SERVER_HOST, \
CFG_BIBSCHED_PROCESS_USER
from invenio.bibtask import guess_apache_process_user, \
check_running_process_user
check_running_process_user()
print ">>> Checking if OpenOffice is correctly integrated...",
if CFG_OPENOFFICE_SERVER_HOST:
try:
check_openoffice_tmpdir()
except InvenioWebSubmitFileConverterError, err:
print wrap_text_in_a_box("""\
OpenOffice.org can't properly create files in the OpenOffice.org temporary
directory %(tmpdir)s, as the user %(nobody)s (as configured in
CFG_OPENOFFICE_USER invenio(-local).conf variable): %(err)s.
In your /etc/sudoers file, you should authorize the %(apache)s user to run
%(python)s as %(nobody)s user as in:
%(apache)s localhost=(%(nobody)s) NOPASSWD: %(python)s
You should then run the following commands:
$ sudo mkdir -p %(tmpdir)s
$ sudo chown %(nobody)s %(tmpdir)s
$ sudo chmod 755 %(tmpdir)s""" % {
'tmpdir' : CFG_OPENOFFICE_TMPDIR,
'nobody' : CFG_OPENOFFICE_USER,
'err' : err,
'apache' : CFG_BIBSCHED_PROCESS_USER or guess_apache_process_user(),
'python' : CFG_PATH_OPENOFFICE_PYTHON
})
sys.exit(1)
print "ok"
else:
print "OpenOffice.org integration not enabled"
def test_db_connection():
"""
Test DB connection, and if fails, advise user how to set it up.
Useful to be called during table creation.
"""
print "Testing DB connection...",
from invenio.textutils import wrap_text_in_a_box
from invenio.dbquery import run_sql, Error
## first, test connection to the DB server:
try:
run_sql("SHOW TABLES")
except Error, err:
from invenio.dbquery import CFG_DATABASE_HOST, CFG_DATABASE_PORT, \
CFG_DATABASE_NAME, CFG_DATABASE_USER, CFG_DATABASE_PASS
print wrap_text_in_a_box("""\
DATABASE CONNECTIVITY ERROR %(errno)d: %(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please login as MySQL admin user and run the
following commands now:
$ mysql -h %(dbhost)s -P %(dbport)s -u root -p mysql
mysql> CREATE DATABASE %(dbname)s DEFAULT CHARACTER SET utf8;
mysql> GRANT ALL PRIVILEGES ON %(dbname)s.*
TO %(dbuser)s@%(webhost)s IDENTIFIED BY '%(dbpass)s';
mysql> QUIT
The values printed above were detected from your
configuration. If they are not right, then please edit your
invenio-local.conf file and rerun 'inveniocfg --update-all' first.
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing.""" % \
{'errno': err.args[0],
'errmsg': err.args[1],
'dbname': CFG_DATABASE_NAME,
'dbhost': CFG_DATABASE_HOST,
'dbport': CFG_DATABASE_PORT,
'dbuser': CFG_DATABASE_USER,
'dbpass': CFG_DATABASE_PASS,
'webhost': CFG_DATABASE_HOST == 'localhost' and 'localhost' or os.popen('hostname -f', 'r').read().strip(),
})
sys.exit(1)
print "ok"
## second, test insert/select of a Unicode string to detect
## possible Python/MySQL/MySQLdb mis-setup:
print "Testing Python/MySQL/MySQLdb UTF-8 chain...",
try:
beta_in_utf8 = "β" # Greek beta in UTF-8 is 0xCEB2
run_sql("CREATE TEMPORARY TABLE test__invenio__utf8 (x char(1), y varbinary(2)) DEFAULT CHARACTER SET utf8")
run_sql("INSERT INTO test__invenio__utf8 (x, y) VALUES (%s, %s)", (beta_in_utf8, beta_in_utf8))
res = run_sql("SELECT x,y,HEX(x),HEX(y),LENGTH(x),LENGTH(y),CHAR_LENGTH(x),CHAR_LENGTH(y) FROM test__invenio__utf8")
assert res[0] == ('\xce\xb2', '\xce\xb2', 'CEB2', 'CEB2', 2L, 2L, 1L, 2L)
run_sql("DROP TEMPORARY TABLE test__invenio__utf8")
except Exception, err:
print wrap_text_in_a_box("""\
DATABASE RELATED ERROR %s\n
A problem was detected with the UTF-8 treatment in the chain
between the Python application, the MySQLdb connector, and
the MySQL database. You may perhaps have installed older
versions of some prerequisite packages?\n
Please check the INSTALL file and please fix this problem
before continuing.""" % err)
sys.exit(1)
print "ok"
def cli_cmd_create_tables(conf):
"""Create and fill Invenio DB tables. Useful for the installation process."""
print ">>> Going to create and fill tables..."
from invenio.config import CFG_PREFIX
test_db_connection()
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabcreate.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/dbexec < %s/lib/sql/invenio/tabfill.sql" % (CFG_PREFIX, CFG_PREFIX)]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
for cmd in ["%s/bin/webaccessadmin -u admin -c -a" % CFG_PREFIX]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables created and filled successfully."
def cli_cmd_load_webstat_conf(conf):
print ">>> Going to load WebStat config..."
from invenio.config import CFG_PREFIX
cmd = "%s/bin/webstatadmin --load-config" % CFG_PREFIX
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> WebStat config load successfully."
def cli_cmd_drop_tables(conf):
"""Drop Invenio DB tables. Useful for the uninstallation process."""
print ">>> Going to drop tables..."
from invenio.config import CFG_PREFIX
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your database tables!"""))
cmd = "%s/bin/dbexec < %s/lib/sql/invenio/tabdrop.sql" % (CFG_PREFIX, CFG_PREFIX)
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables dropped successfully."
def cli_cmd_create_demo_site(conf):
"""Create demo site. Useful for testing purposes."""
print ">>> Going to create demo site..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
run_sql("TRUNCATE schTASK")
run_sql("TRUNCATE session")
run_sql("DELETE FROM user WHERE email=''")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/democfgdata.sql" % \
(CFG_PREFIX, CFG_PREFIX),]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_fieldnames(conf) # needed for I18N demo ranking method names
for cmd in ["%s/bin/webaccessadmin -u admin -c -r -D" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo site created successfully."
def cli_cmd_load_demo_records(conf):
"""Load demo records. Useful for testing purposes."""
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
print ">>> Going to load demo records..."
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/bibupload -u admin -i %s/var/tmp/demobibdata.xml" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/bibupload 1" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --with-ocr --recid 97" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --all" % CFG_PREFIX,
"%s/bin/bibindex -u admin" % CFG_PREFIX,
"%s/bin/bibindex 2" % CFG_PREFIX,
"%s/bin/bibreformat -u admin -o HB" % CFG_PREFIX,
"%s/bin/bibreformat 3" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 4" % CFG_PREFIX,
"%s/bin/bibrank -u admin" % CFG_PREFIX,
"%s/bin/bibrank 5" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records loaded successfully."
def cli_cmd_remove_demo_records(conf):
"""Remove demo records. Useful when you are finished testing."""
print ">>> Going to remove demo records..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your records and documents!"""))
if os.path.exists(CFG_PREFIX + os.sep + 'var' + os.sep + 'data'):
shutil.rmtree(CFG_PREFIX + os.sep + 'var' + os.sep + 'data')
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabbibclean.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records removed successfully."
def cli_cmd_drop_demo_site(conf):
"""Drop demo site completely. Useful when you are finished testing."""
print ">>> Going to drop demo site..."
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your site and documents!"""))
cli_cmd_drop_tables(conf)
cli_cmd_create_tables(conf)
cli_cmd_remove_demo_records(conf)
print ">>> Demo site dropped successfully."
def cli_cmd_run_unit_tests(conf):
"""Run unit tests, usually on the working demo site."""
from invenio.testutils import build_and_run_unit_test_suite
build_and_run_unit_test_suite()
def cli_cmd_run_regression_tests(conf):
"""Run regression tests, usually on the working demo site."""
from invenio.testutils import build_and_run_regression_test_suite
build_and_run_regression_test_suite()
def cli_cmd_run_web_tests(conf):
"""Run web tests in a browser. Requires Firefox with
Selenium IDE extension."""
from invenio.testutils import build_and_run_web_test_suite
build_and_run_web_test_suite()
def _detect_ip_address():
"""Detect IP address of this computer. Useful for creating Apache
vhost conf snippet on RHEL like machines.
@return: IP address, or '*' if cannot detect
@rtype: string
@note: creates socket for real in order to detect real IP address,
not the loopback one.
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('invenio-software.org', 0))
return s.getsockname()[0]
except:
return '*'
def cli_cmd_create_apache_conf(conf):
"""
Create Apache conf files for this site, keeping previous
files in a backup copy.
"""
print ">>> Going to create Apache conf files..."
from invenio.textutils import wrap_text_in_a_box
apache_conf_dir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'apache'
xsendfile_directive_needed = int(conf.get("Invenio", 'CFG_BIBDOCFILE_USE_XSENDFILE')) != 0
## Apache vhost conf file is distro specific, so analyze needs:
# Gentoo (and generic defaults):
listen_directive_needed = True
ssl_pem_directive_needed = False
ssl_pem_path = '/etc/apache2/ssl/apache.pem'
ssl_crt_path = '/etc/apache2/ssl/server.crt'
ssl_key_path = '/etc/apache2/ssl/server.key'
vhost_ip_address_needed = False
wsgi_socket_directive_needed = False
# Debian:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'debian_version'):
listen_directive_needed = False
ssl_pem_directive_needed = True
# RHEL/SLC:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'redhat-release'):
listen_directive_needed = False
ssl_crt_path = '/etc/pki/tls/certs/localhost.crt'
ssl_key_path = '/etc/pki/tls/private/localhost.key'
vhost_ip_address_needed = True
wsgi_socket_directive_needed = True
## okay, let's create Apache vhost files:
if not os.path.exists(apache_conf_dir):
os.mkdir(apache_conf_dir)
apache_vhost_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost.conf'
apache_vhost_ssl_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost-ssl.conf'
apache_vhost_body = """\
AddDefaultCharset UTF-8
ServerSignature Off
ServerTokens Prod
NameVirtualHost %(vhost_ip_address)s:80
%(listen_directive)s
%(wsgi_socket_directive)s
%(xsendfile_directive)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
DocumentRoot %(webdir)s
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
ErrorLog %(logdir)s/apache.err
LogLevel warn
CustomLog %(logdir)s/apache.log combined
DirectoryIndex index.en.html index.html
Alias /img/ %(webdir)s/img/
Alias /js/ %(webdir)s/js/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /fckeditor/ %(webdir)s/fckeditor/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
WSGIDaemonProcess invenio processes=5 threads=1 display-name=%%{GROUP} inactivity-timeout=3600 maximum-requests=10000
WSGIImportScript %(wsgidir)s/invenio.wsgi process-group=invenio application-group=%%{GLOBAL}
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
""" % {'servername': conf.get('Invenio', 'CFG_SITE_URL').replace("http://", ""),
'serveralias': conf.get('Invenio', 'CFG_SITE_URL').replace("http://", "").split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'),
'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address() or '*',
'listen_directive': listen_directive_needed and 'Listen 80' or '#Listen 80',
'wsgi_socket_directive': (wsgi_socket_directive_needed and \
'WSGISocketPrefix ' or '#WSGISocketPrefix ') + \
conf.get('Invenio', 'CFG_PREFIX') + os.sep + 'var' + os.sep + 'run',
'xsendfile_directive' : xsendfile_directive_needed and \
"XSendFile On\nXSendFileAllowAbove On" or \
"#XSendFile On\n#XSendFileAllowAbove On",
}
apache_vhost_ssl_body = """\
ServerSignature Off
ServerTokens Prod
%(listen_directive)s
NameVirtualHost %(vhost_ip_address)s:443
%(ssl_pem_directive)s
%(ssl_crt_directive)s
%(ssl_key_directive)s
%(xsendfile_directive)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
SSLEngine on
DocumentRoot %(webdir)s
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
ErrorLog %(logdir)s/apache-ssl.err
LogLevel warn
CustomLog %(logdir)s/apache-ssl.log combined
DirectoryIndex index.en.html index.html
Alias /img/ %(webdir)s/img/
Alias /js/ %(webdir)s/js/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /fckeditor/ %(webdir)s/fckeditor/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
""" % {'servername': conf.get('Invenio', 'CFG_SITE_SECURE_URL').replace("https://", ""),
'serveralias': conf.get('Invenio', 'CFG_SITE_SECURE_URL').replace("https://", "").split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'),
'wsgidir' : os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address() or '*',
'listen_directive' : listen_directive_needed and 'Listen 443' or '#Listen 443',
'ssl_pem_directive': ssl_pem_directive_needed and \
'SSLCertificateFile %s' % ssl_pem_path or \
'#SSLCertificateFile %s' % ssl_pem_path,
'ssl_crt_directive': ssl_pem_directive_needed and \
'#SSLCertificateFile %s' % ssl_crt_path or \
'SSLCertificateFile %s' % ssl_crt_path,
'ssl_key_directive': ssl_pem_directive_needed and \
'#SSLCertificateKeyFile %s' % ssl_key_path or \
'SSLCertificateKeyFile %s' % ssl_key_path,
'xsendfile_directive' : xsendfile_directive_needed and \
"XSendFile On\nXSendFileAllowAbove On" or \
"#XSendFile On\n#XSendFileAllowAbove On",
}
# write HTTP vhost snippet:
if os.path.exists(apache_vhost_file):
shutil.copy(apache_vhost_file,
apache_vhost_file + '.OLD')
fdesc = open(apache_vhost_file, 'w')
fdesc.write(apache_vhost_body)
fdesc.close()
print
print "Created file", apache_vhost_file
# write HTTPS vhost snippet:
vhost_ssl_created = False
if conf.get('Invenio', 'CFG_SITE_SECURE_URL') != \
conf.get('Invenio', 'CFG_SITE_URL'):
if os.path.exists(apache_vhost_ssl_file):
shutil.copy(apache_vhost_ssl_file,
apache_vhost_ssl_file + '.OLD')
fdesc = open(apache_vhost_ssl_file, 'w')
fdesc.write(apache_vhost_ssl_body)
fdesc.close()
vhost_ssl_created = True
print "Created file", apache_vhost_ssl_file
print wrap_text_in_a_box("""\
Apache virtual host configuration file(s) for your Invenio site
was(were) created. Please check created file(s) and activate virtual
host(s). For example, you can put the following include statements in
your httpd.conf:\n
Include %s
%s
Please see the INSTALL file for more details.
""" % (apache_vhost_file, (vhost_ssl_created and 'Include ' or '#Include ') + apache_vhost_ssl_file))
print ">>> Apache conf files created."
def cli_cmd_get(conf, varname):
"""
Return value of VARNAME read from CONF files. Useful for
third-party programs to access values of conf options such as
CFG_PREFIX. Return None if VARNAME is not found.
"""
# do not pay attention to upper/lower case:
varname = varname.lower()
# do not pay attention to section names yet:
all_options = {}
for section in conf.sections():
for option in conf.options(section):
all_options[option] = conf.get(section, option)
return all_options.get(varname, None)
def cli_cmd_list(conf):
"""
Print a list of all conf options and values from CONF.
"""
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
print option.upper(), '=', conf.get(section, option)
def _grep_version_from_executable(path_to_exec, version_regexp):
"""
Try to detect a program version by digging into its binary
PATH_TO_EXEC and looking for VERSION_REGEXP. Return program
version as a string. Return empty string if not succeeded.
"""
from invenio.shellutils import run_shell_command
exec_version = ""
if os.path.exists(path_to_exec):
dummy1, cmd2_out, dummy2 = run_shell_command("strings %s | grep %s",
(path_to_exec, version_regexp))
if cmd2_out:
for cmd2_out_line in cmd2_out.split("\n"):
if len(cmd2_out_line) > len(exec_version):
# the longest the better
exec_version = cmd2_out_line
return exec_version
def detect_apache_version():
"""
Try to detect Apache version by localizing httpd or apache
executables and grepping inside binaries. Return list of all
found Apache versions and paths. (For a given executable, the
returned format is 'apache_version [apache_path]'.) Return empty
list if no success.
"""
from invenio.shellutils import run_shell_command
out = []
dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache")
for apache in cmd_out.split("\n"):
apache_version = _grep_version_from_executable(apache, '^Apache\/')
if apache_version:
out.append("%s [%s]" % (apache_version, apache))
return out
def cli_cmd_detect_system_details(conf):
"""
Detect and print system details such as Apache/Python/MySQL
versions etc. Useful for debugging problems on various OS.
"""
import MySQLdb
print ">>> Going to detect system details..."
print "* Hostname: " + socket.gethostname()
print "* Invenio version: " + conf.get("Invenio", "CFG_VERSION")
print "* Python version: " + sys.version.replace("\n", " ")
print "* Apache version: " + ";\n ".join(detect_apache_version())
print "* MySQLdb version: " + MySQLdb.__version__
try:
from invenio.dbquery import run_sql
print "* MySQL version:"
for key, val in run_sql("SHOW VARIABLES LIKE 'version%'") + \
run_sql("SHOW VARIABLES LIKE 'charact%'") + \
run_sql("SHOW VARIABLES LIKE 'collat%'"):
if False:
print " - %s: %s" % (key, val)
elif key in ['version',
'character_set_client',
'character_set_connection',
'character_set_database',
'character_set_results',
'character_set_server',
'character_set_system',
'collation_connection',
'collation_database',
'collation_server']:
print " - %s: %s" % (key, val)
except ImportError:
print "* ERROR: cannot import dbquery"
print ">>> System details detected successfully."
def main():
"""Main entry point."""
conf = ConfigParser()
if '--help' in sys.argv or \
'-h' in sys.argv:
print_usage()
elif '--version' in sys.argv or \
'-V' in sys.argv:
print_version()
else:
confdir = None
if '--conf-dir' in sys.argv:
try:
confdir = sys.argv[sys.argv.index('--conf-dir') + 1]
except IndexError:
pass # missing --conf-dir argument value
if not os.path.exists(confdir):
print "ERROR: bad or missing --conf-dir option value."
sys.exit(1)
else:
## try to detect path to conf dir (relative to this bin dir):
confdir = re.sub(r'/bin$', '/etc', sys.path[0])
## read conf files:
for conffile in [confdir + os.sep + 'invenio.conf',
confdir + os.sep + 'invenio-autotools.conf',
confdir + os.sep + 'invenio-local.conf',]:
if os.path.exists(conffile):
conf.read(conffile)
else:
if not conffile.endswith("invenio-local.conf"):
# invenio-local.conf is optional, otherwise stop
print "ERROR: Badly guessed conf file location", conffile
print "(Please use --conf-dir option.)"
sys.exit(1)
## decide what to do:
done = False
for opt_idx in range(0, len(sys.argv)):
opt = sys.argv[opt_idx]
if opt == '--conf-dir':
# already treated before, so skip silently:
pass
elif opt == '--get':
try:
varname = sys.argv[opt_idx + 1]
except IndexError:
print "ERROR: bad or missing --get option value."
sys.exit(1)
if varname.startswith('-'):
print "ERROR: bad or missing --get option value."
sys.exit(1)
varvalue = cli_cmd_get(conf, varname)
if varvalue is not None:
print varvalue
else:
sys.exit(1)
done = True
elif opt == '--list':
cli_cmd_list(conf)
done = True
elif opt == '--detect-system-details':
cli_cmd_detect_system_details(conf)
done = True
elif opt == '--create-tables':
cli_cmd_create_tables(conf)
done = True
elif opt == '--load-webstat-conf':
cli_cmd_load_webstat_conf(conf)
done = True
elif opt == '--drop-tables':
cli_cmd_drop_tables(conf)
done = True
elif opt == '--check-openoffice-dir':
cli_check_openoffice_dir(conf)
done = True
elif opt == '--create-demo-site':
cli_cmd_create_demo_site(conf)
done = True
elif opt == '--load-demo-records':
cli_cmd_load_demo_records(conf)
done = True
elif opt == '--remove-demo-records':
cli_cmd_remove_demo_records(conf)
done = True
elif opt == '--drop-demo-site':
cli_cmd_drop_demo_site(conf)
done = True
elif opt == '--run-unit-tests':
cli_cmd_run_unit_tests(conf)
done = True
elif opt == '--run-regression-tests':
cli_cmd_run_regression_tests(conf)
done = True
elif opt == '--run-web-tests':
cli_cmd_run_web_tests(conf)
done = True
elif opt == '--update-all':
cli_cmd_update_config_py(conf)
cli_cmd_update_dbquery_py(conf)
cli_cmd_update_dbexec(conf)
cli_cmd_update_bibconvert_tpl(conf)
cli_cmd_update_web_tests(conf)
done = True
elif opt == '--update-config-py':
cli_cmd_update_config_py(conf)
done = True
elif opt == '--update-dbquery-py':
cli_cmd_update_dbquery_py(conf)
done = True
elif opt == '--update-dbexec':
cli_cmd_update_dbexec(conf)
done = True
elif opt == '--update-bibconvert-tpl':
cli_cmd_update_bibconvert_tpl(conf)
done = True
elif opt == '--update-web-tests':
cli_cmd_update_web_tests(conf)
done = True
elif opt == '--reset-all':
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
cli_cmd_reset_recstruct_cache(conf)
done = True
elif opt == '--reset-sitename':
cli_cmd_reset_sitename(conf)
done = True
elif opt == '--reset-siteadminemail':
cli_cmd_reset_siteadminemail(conf)
done = True
elif opt == '--reset-fieldnames':
cli_cmd_reset_fieldnames(conf)
done = True
elif opt == '--reset-recstruct-cache':
cli_cmd_reset_recstruct_cache(conf)
done = True
elif opt == '--create-apache-conf':
cli_cmd_create_apache_conf(conf)
done = True
elif opt.startswith("-") and opt != '--yes-i-know':
print "ERROR: unknown option", opt
sys.exit(1)
if not done:
print """ERROR: Please specify a command. Please see '--help'."""
sys.exit(1)
if __name__ == '__main__':
main()
diff --git a/modules/websubmit/lib/websubmit_config.py b/modules/websubmit/lib/websubmit_config.py
index 95bf3809c..b04d40730 100644
--- a/modules/websubmit/lib/websubmit_config.py
+++ b/modules/websubmit/lib/websubmit_config.py
@@ -1,248 +1,287 @@
## This file is part of Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Invenio Submission Web Interface config file."""
__revision__ = "$Id$"
import re
## test:
test = "FALSE"
## CC all action confirmation mails to administrator? (0 == NO; 1 == YES)
CFG_WEBSUBMIT_COPY_MAILS_TO_ADMIN = 0
## During submission, warn user if she is going to leave the
## submission by following some link on the page?
## Does not work with Opera and Konqueror.
## This requires all submission functions to set Javascript variable
## 'user_must_confirm_before_leaving_page' to 'false' before
## programmatically submitting a form , or else users will be asked
## confirmation after each submission step.
## (0 == NO; 1 == YES)
CFG_WEBSUBMIT_CHECK_USER_LEAVES_SUBMISSION = 0
## List of keywords/format parameters that should not write by default
## corresponding files in submission directory (`curdir'). Some other
## filenames not included here are reserved too, such as those
## containing non-alphanumeric chars (excepted underscores '_'), for
## eg all names containing a dot ('bibdocactions.log',
## 'performed_actions.log', etc.)
CFG_RESERVED_SUBMISSION_FILENAMES = ['SuE',
'files',
'lastuploadedfile',
'curdir',
'function_log',
'SN']
## CFG_WEBSUBMIT_BEST_FORMATS_TO_EXTRACT_TEXT_FROM -- a comma-separated
## list of document extensions in decrescent order of preference
## to suggest what is considered the best format to extract text from.
CFG_WEBSUBMIT_BEST_FORMATS_TO_EXTRACT_TEXT_FROM = ('txt', 'html', 'xml', 'odt', 'doc', 'docx', 'djvu', 'pdf', 'ps', 'ps.gz')
## CFG_WEBSUBMIT_DESIRED_CONVERSIONS -- a dictionary having as keys
## a format and as values the corresponding list of desired converted
## formats.
CFG_WEBSUBMIT_DESIRED_CONVERSIONS = {
'pdf' : ('ps.gz', ),
'ps.gz' : ('pdf', ),
'djvu' : ('ps.gz', 'pdf'),
'docx' : ('doc', 'odt', 'pdf', 'ps.gz'),
'doc' : ('odt', 'pdf', 'ps.gz'),
'rtf' : ('pdf', 'odt', 'ps.gz'),
'odt' : ('pdf', 'doc', 'ps.gz'),
'pptx' : ('ppt', 'odp', 'pdf', 'ps.gz'),
'ppt' : ('odp', 'pdf', 'ps.gz'),
'odp' : ('pdf', 'ppt', 'ps.gz'),
'xlsx' : ('xls', 'ods', 'csv'),
'xls' : ('ods', 'csv'),
'ods' : ('xls', 'csv'),
'tiff' : ('pdf', 'ps.gz'),
'tif' : ('pdf', 'ps.gz')
}
## CFG_WEBSUBMIT_ICON_SUBFORMAT_RE -- a subformat is an Invenio concept to give
## file formats more semantic. For example "foo.gif;icon" has ".gif;icon"
## 'format', ".gif" 'superformat' and "icon" 'subformat'. That means that this
## particular format/instance of the "foo" document, not only is a ".gif" but
## is in the shape of an "icon", i.e. most probably it will be low-resolution.
## This configuration variable let the administrator to decide which implicit
## convention will be used to know which formats will be meant to be used
## as an icon.
CFG_WEBSUBMIT_ICON_SUBFORMAT_RE = re.compile(r"icon.*")
## CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT -- this is the default subformat used
## when creating new icons.
CFG_WEBSUBMIT_DEFAULT_ICON_SUBFORMAT = "icon"
+try:
+ from invenio.config import CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC
+except ImportError:
+ CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC = {
+ 'can_revise_doctypes': ['*'],
+ 'can_comment_doctypes': ['*'],
+ 'can_describe_doctypes': ['*'],
+ 'can_delete_doctypes': ['*'],
+ 'can_keep_doctypes': ['*'],
+ 'can_rename_doctypes': ['*'],
+ 'can_add_format_to_doctypes': ['*'],
+ 'can_restrict_doctypes': ['*']}
+
+try:
+ from invenio.config import CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES
+except ImportError:
+ CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES = [
+ ('Main', 'Main document'),
+ ('LaTeX', 'LaTeX'),
+ ('Source', 'Source'),
+ ('Additional', 'Additional File'),
+ ('Audio', 'Audio file'),
+ ('Video', 'Video file'),
+ ('Script', 'Script'),
+ ('Data', 'Data'),
+ ('Figure', 'Figure'),
+ ('Schema', 'Schema'),
+ ('Graph', 'Graph'),
+ ('Image', 'Image'),
+ ('Drawing', 'Drawing'),
+ ('Slides', 'Slides')]
+
+try:
+ from invenio.config import CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS
+except ImportError:
+ CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS = [
+ ('', 'Public'),
+ ('restricted', 'Restricted')]
+
class InvenioWebSubmitFunctionError(Exception):
"""This exception should only ever be raised by WebSubmit functions.
It will be caught and handled by the WebSubmit core itself.
It is used to signal to WebSubmit core that one of the functions
encountered a FATAL ERROR situation that should all further execution
of the submission.
The exception will carry an error message in its "value" string. This
message will probably be displayed on the user's browser in an Invenio
"error" box, and may be logged for the admin to examine.
Again: If this exception is raised by a WebSubmit function, an error
message will displayed and the submission ends in failure.
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - an error string to display to the user.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
class InvenioWebSubmitFunctionStop(Exception):
"""This exception should only ever be raised by WebSubmit functions.
It will be caught and handled by the WebSubmit core itself.
It is used to signal to WebSubmit core that one of the functions
encountered a situation that should prevent the functions that follow
it from being executed, and that WebSubmit core should display some sort
of message to the user. This message will be stored in the "value"
attribute of the object.
***
NOTE: In the current WebSubmit, this "value" is ususally a JavaScript
string that redirects the user's browser back to the Web form
phase of the submission. The use of JavaScript, however is going
to be removed in the future, so the mechanism may change.
***
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - a string to display to the user.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
class InvenioWebSubmitFunctionWarning(Exception):
"""This exception should be raised by a WebSubmit function
when unexpected behaviour is encountered during the execution
of the function. The unexpected behaviour should not have been
so serious that execution had to be halted, but since the
function was unable to perform its task, the event must be
logged.
Logging of the exception will be performed by WebSubmit.
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - a string to write to the log.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
class InvenioWebSubmitFileStamperError(Exception):
"""This exception should be raised by websubmit_file_stamper when an
error is encoutered that prevents a file from being stamped.
When caught, this exception should be used to stop processing with a
failure signal.
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - a string to write to the log.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
class InvenioWebSubmitIconCreatorError(Exception):
"""This exception should be raised by websubmit_icon_creator when an
error is encoutered that prevents an icon from being created.
When caught, this exception should be used to stop processing with a
failure signal.
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - a string to write to the log.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
class InvenioWebSubmitFileMetadataRuntimeError(Exception):
"""This exception should be raised by websubmit_file_metadata plugins when an
error is encoutered that prevents a extracting/updating a file.
When caught, this exception should be used to stop processing with a
failure signal.
Extends: Exception.
"""
def __init__(self, value):
"""Set the internal "value" attribute to that of the passed "value"
parameter.
@param value: (string) - a string to write to the log.
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""Return oneself as a string (actually, return the contents of
self.value).
@return: (string)
"""
return str(self.value)
diff --git a/modules/websubmit/lib/websubmit_webinterface.py b/modules/websubmit/lib/websubmit_webinterface.py
index 0babe7514..c68fd0ea3 100644
--- a/modules/websubmit/lib/websubmit_webinterface.py
+++ b/modules/websubmit/lib/websubmit_webinterface.py
@@ -1,1189 +1,1165 @@
## This file is part of Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__lastupdated__ = """$Date$"""
__revision__ = "$Id$"
import os
import time
import cgi
import sys
import shutil
from urllib import urlencode
from invenio.config import \
CFG_ACCESS_CONTROL_LEVEL_SITE, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_TMPDIR, \
CFG_SITE_NAME_INTL, \
CFG_SITE_URL, \
CFG_SITE_SECURE_URL, \
CFG_WEBSUBMIT_STORAGEDIR, \
CFG_PREFIX, \
CFG_CERN_SITE
from invenio import webinterface_handler_config as apache
from invenio.dbquery import run_sql
from invenio.access_control_config import VIEWRESTRCOLL
from invenio.access_control_mailcookie import mail_cookie_create_authorize_action
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_admin import acc_is_role
from invenio.webpage import page, create_error_box, pageheaderonly, \
pagefooteronly
from invenio.webuser import getUid, page_not_authorized, collect_user_info, isGuestUser, isUserSuperAdmin
from invenio.websubmit_config import *
from invenio.webinterface_handler import wash_urlargd, WebInterfaceDirectory
from invenio.urlutils import make_canonical_urlargd, redirect_to_url
from invenio.messages import gettext_set_language
from invenio.search_engine import \
guess_primary_collection_of_a_record, get_colID, record_exists, \
create_navtrail_links, check_user_can_view_record, record_empty
from invenio.bibdocfile import BibRecDocs, normalize_format, file_strip_ext, \
stream_restricted_icon, BibDoc, InvenioWebSubmitFileError, stream_file, \
decompose_file, propose_next_docname, get_subformat_from_format
from invenio.errorlib import register_exception
from invenio.websubmit_icon_creator import create_icon, InvenioWebSubmitIconCreatorError
import invenio.template
websubmit_templates = invenio.template.load('websubmit')
from invenio.websearchadminlib import get_detailed_page_tabs
from invenio.session import get_session
import invenio.template
webstyle_templates = invenio.template.load('webstyle')
websearch_templates = invenio.template.load('websearch')
try:
from invenio.fckeditor_invenio_connector import FCKeditorConnectorInvenio
fckeditor_available = True
except ImportError, e:
fckeditor_available = False
from invenio.websubmit_managedocfiles import \
create_file_upload_interface, \
get_upload_file_interface_javascript, \
get_upload_file_interface_css, \
move_uploaded_files_to_storage
class WebInterfaceFilesPages(WebInterfaceDirectory):
def __init__(self,recid):
self.recid = recid
def _lookup(self, component, path):
# after /record//files/ every part is used as the file
# name
filename = component
def getfile(req, form):
args = wash_urlargd(form, websubmit_templates.files_default_urlargd)
ln = args['ln']
_ = gettext_set_language(ln)
uid = getUid(req)
user_info = collect_user_info(req)
verbose = args['verbose']
if verbose >= 1 and not isUserSuperAdmin(user_info):
# Only SuperUser can see all the details!
verbose = 0
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE > 1:
return page_not_authorized(req, "/record/%s" % self.recid,
navmenuid='submit')
if record_exists(self.recid) < 1:
msg = " %s " % _("Requested record does not seem to exist.")
return warningMsg(msg, req, CFG_SITE_NAME, ln)
if record_empty(self.recid):
msg = "%s " % _("Requested record does not seem to have been integrated.")
return warningMsg(msg, req, CFG_SITE_NAME, ln)
(auth_code, auth_message) = check_user_can_view_record(user_info, self.recid)
if auth_code and user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action(VIEWRESTRCOLL, {'collection' : guess_primary_collection_of_a_record(self.recid)})
target = '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_URL + user_info['uri']}, {})
return redirect_to_url(req, target, norobot=True)
elif auth_code:
return page_not_authorized(req, "../", \
text = auth_message)
readonly = CFG_ACCESS_CONTROL_LEVEL_SITE == 1
# From now on: either the user provided a specific file
# name (and a possible version), or we return a list of
# all the available files. In no case are the docids
# visible.
try:
bibarchive = BibRecDocs(self.recid)
except InvenioWebSubmitFileError, e:
register_exception(req=req, alert_admin=True)
msg = "%s %s " % (
_("The system has encountered an error in retrieving the list of files for this document."),
_("The error has been logged and will be taken in consideration as soon as possible."))
return warningMsg(msg, req, CFG_SITE_NAME, ln)
if bibarchive.deleted_p():
return print_warning(req, _("Requested record does not seem to exist."))
docname = ''
format = ''
version = ''
warn = ''
if filename:
# We know the complete file name, guess which docid it
# refers to
## TODO: Change the extension system according to ext.py from setlink
## and have a uniform extension mechanism...
docname = file_strip_ext(filename)
format = filename[len(docname):]
if format and format[0] != '.':
format = '.' + format
if args['subformat']:
format += ';%s' % args['subformat']
else:
docname = args['docname']
if not format:
format = args['format']
if args['subformat']:
format += ';%s' % args['subformat']
if not version:
version = args['version']
# version could be either empty, or all or an integer
try:
int(version)
except ValueError:
if version != 'all':
version = ''
display_hidden = isUserSuperAdmin(user_info)
if version != 'all':
# search this filename in the complete list of files
for doc in bibarchive.list_bibdocs():
if docname == doc.get_docname():
try:
docfile = doc.get_file(format, version)
(auth_code, auth_message) = docfile.is_restricted(user_info)
if auth_code != 0:
if CFG_WEBSUBMIT_ICON_SUBFORMAT_RE.match(get_subformat_from_format(format)):
return stream_restricted_icon(req)
if user_info['email'] == 'guest':
cookie = mail_cookie_create_authorize_action('viewrestrdoc', {'status' : docfile.get_status()})
target = '/youraccount/login' + \
make_canonical_urlargd({'action': cookie, 'ln' : ln, 'referer' : \
CFG_SITE_URL + user_info['uri']}, {})
redirect_to_url(req, target)
else:
req.status = apache.HTTP_UNAUTHORIZED
warn += print_warning(_("This file is restricted: ") + auth_message)
break
if not docfile.hidden_p():
if not readonly:
ip = str(req.remote_ip)
res = doc.register_download(ip, version, format, uid)
try:
return docfile.stream(req)
except InvenioWebSubmitFileError, msg:
register_exception(req=req, alert_admin=True)
req.status = apache.HTTP_INTERNAL_SERVER_ERROR
return warningMsg(_("An error has happened in trying to stream the request file."), req, CFG_SITE_NAME, ln)
else:
req.status = apache.HTTP_UNAUTHORIZED
warn = print_warning(_("The requested file is hidden and can not be accessed."))
except InvenioWebSubmitFileError, msg:
register_exception(req=req, alert_admin=True)
if docname and format and not warn:
req.status = apache.HTTP_NOT_FOUND
warn += print_warning(_("Requested file does not seem to exist."))
filelist = bibarchive.display("", version, ln=ln, verbose=verbose, display_hidden=display_hidden)
t = warn + websubmit_templates.tmpl_filelist(
ln=ln,
recid=self.recid,
docname=args['docname'],
version=version,
filelist=filelist)
cc = guess_primary_collection_of_a_record(self.recid)
unordered_tabs = get_detailed_page_tabs(get_colID(cc), self.recid, ln)
ordered_tabs_id = [(tab_id, values['order']) for (tab_id, values) in unordered_tabs.iteritems()]
ordered_tabs_id.sort(lambda x,y: cmp(x[1],y[1]))
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
tabs = [(unordered_tabs[tab_id]['label'], \
'%s/record/%s/%s%s' % (CFG_SITE_URL, self.recid, tab_id, link_ln), \
tab_id == 'files',
unordered_tabs[tab_id]['enabled']) \
for (tab_id, order) in ordered_tabs_id
if unordered_tabs[tab_id]['visible'] == True]
top = webstyle_templates.detailed_record_container_top(self.recid,
tabs,
args['ln'])
bottom = webstyle_templates.detailed_record_container_bottom(self.recid,
tabs,
args['ln'])
title, description, keywords = websearch_templates.tmpl_record_page_header_content(req, self.recid, args['ln'])
return pageheaderonly(title=title,
navtrail=create_navtrail_links(cc=cc, aas=0, ln=ln) + \
''' > %s
> %s''' % \
(CFG_SITE_URL, self.recid, title, _("Access to Fulltext")),
description="",
keywords="keywords",
uid=uid,
language=ln,
req=req,
navmenuid='search',
navtrail_append_title_p=0) + \
websearch_templates.tmpl_search_pagestart(ln) + \
top + t + bottom + \
websearch_templates.tmpl_search_pageend(ln) + \
pagefooteronly(lastupdated=__lastupdated__, language=ln, req=req)
return getfile, []
def __call__(self, req, form):
"""Called in case of URLs like /record/123/files without
trailing slash.
"""
args = wash_urlargd(form, websubmit_templates.files_default_urlargd)
ln = args['ln']
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
return redirect_to_url(req, '%s/record/%s/files/%s' % (CFG_SITE_URL, self.recid, link_ln))
def websubmit_legacy_getfile(req, form):
""" Handle legacy /getfile.py URLs """
args = wash_urlargd(form, {
'recid': (int, 0),
'docid': (int, 0),
'version': (str, ''),
'name': (str, ''),
'format': (str, ''),
'ln' : (str, CFG_SITE_LANG)
})
_ = gettext_set_language(args['ln'])
def _getfile_py(req, recid=0, docid=0, version="", name="", format="", ln=CFG_SITE_LANG):
if not recid:
## Let's obtain the recid from the docid
if docid:
try:
bibdoc = BibDoc(docid=docid)
recid = bibdoc.get_recid()
except InvenioWebSubmitFileError, e:
return warningMsg(_("An error has happened in trying to retrieve the requested file."), req, CFG_SITE_NAME, ln)
else:
return warningMsg(_('Not enough information to retrieve the document'), req, CFG_SITE_NAME, ln)
else:
if not name and docid:
## Let's obtain the name from the docid
try:
bibdoc = BibDoc(docid)
name = bibdoc.get_docname()
except InvenioWebSubmitFileError, e:
return warningMsg(_("An error has happened in trying to retrieving the requested file."), req, CFG_SITE_NAME, ln)
format = normalize_format(format)
redirect_to_url(req, '%s/record/%s/files/%s%s?ln=%s%s' % (CFG_SITE_URL, recid, name, format, ln, version and 'version=%s' % version or ''), apache.HTTP_MOVED_PERMANENTLY)
return _getfile_py(req, **args)
# --------------------------------------------------
from invenio.websubmit_engine import home, action, interface, endaction
class WebInterfaceSubmitPages(WebInterfaceDirectory):
_exports = ['summary', 'sub', 'direct', '', 'attachfile', 'uploadfile', \
'getuploadedfile', 'managedocfiles', 'managedocfilesasync']
def managedocfiles(self, req, form):
"""
Display admin interface to manage files of a record
"""
argd = wash_urlargd(form, {
'ln': (str, ''),
'access': (str, ''),
'recid': (int, None),
'do': (int, 0),
'cancel': (str, None),
})
_ = gettext_set_language(argd['ln'])
uid = getUid(req)
user_info = collect_user_info(req)
# Check authorization
(auth_code, auth_msg) = acc_authorize_action(req,
'runbibdocfile')
if auth_code and user_info['email'] == 'guest':
# Ask to login
target = '/youraccount/login' + \
make_canonical_urlargd({'ln' : argd['ln'],
'referer' : CFG_SITE_URL + user_info['uri']}, {})
return redirect_to_url(req, target)
elif auth_code:
return page_not_authorized(req, referer="/submit/managedocfiles",
uid=uid, text=auth_msg,
ln=argd['ln'],
navmenuid="admin")
# Prepare navtrail
navtrail = '''Admin Area > %(manage_files)s''' \
% {'CFG_SITE_URL': CFG_SITE_URL,
'manage_files': _("Manage Document Files")}
body = ''
if argd['do'] != 0 and not argd['cancel']:
# Apply modifications
working_dir = os.path.join(CFG_TMPDIR,
'websubmit_upload_interface_config_' + str(uid),
argd['access'])
move_uploaded_files_to_storage(working_dir=working_dir,
recid=argd['recid'],
icon_sizes=['180>','700>'],
create_icon_doctypes=['*'],
force_file_revision=False)
# Clean temporary directory
shutil.rmtree(working_dir)
# Confirm modifications
body += '%s ' % \
(_('Your modifications to record #%i have been submitted') % argd['recid'])
elif argd['cancel']:
# Clean temporary directory
working_dir = os.path.join(CFG_TMPDIR,
'websubmit_upload_interface_config_' + str(uid),
argd['access'])
shutil.rmtree(working_dir)
body += '%s ' % \
(_('Your modifications to record #%i have been cancelled') % argd['recid'])
if not argd['recid'] or argd['do'] != 0:
body += '''
''' % {'edit': _('Edit'),
'edit_record': _('Edit record'),
'CFG_SITE_URL': CFG_SITE_URL}
access = time.strftime('%Y%m%d_%H%M%S')
if argd['recid'] and argd['do'] == 0:
# Displaying interface to manage files
# Prepare navtrail
title, description, keywords = websearch_templates.tmpl_record_page_header_content(req, argd['recid'],
argd['ln'])
navtrail = '''Admin Area >
%(manage_files)s >
%(record)s: %(title)s
''' \
% {'CFG_SITE_URL': CFG_SITE_URL,
'title': title,
'manage_files': _("Document File Manager"),
'record': _("Record #%i") % argd['recid']}
- # FIXME: add parameters to `runbibdocfile' in order to
- # configure the file editor based on role, or at least
- # move configuration below to some config file.
body += create_file_upload_interface(\
recid=argd['recid'],
ln=argd['ln'],
- doctypes_and_desc=[('Main', 'Main document'),
- ('LaTeX', 'LaTeX'),
- ('Source', 'Source'),
- ('Additional', 'Additional File'),
- ('Audio', 'Audio file'),
- ('Video', 'Video file'),
- ('Script', 'Script'),
- ('Data', 'Data'),
- ('Figure', 'Figure'),
- ('Schema', 'Schema'),
- ('Graph', 'Graph'),
- ('Image', 'Image'),
- ('Drawing', 'Drawing'),
- ('Slides', 'Slides')],
- can_revise_doctypes=['*'],
- can_comment_doctypes=['*'],
- can_describe_doctypes=['*'],
- can_delete_doctypes=['*'],
- can_keep_doctypes=['*'],
- can_rename_doctypes=['*'],
- can_add_format_to_doctypes=['*'],
- can_restrict_doctypes=['*'],
- restrictions_and_desc=[('', 'Public'),
- ('restricted', 'Restricted')],
uid=uid,
- sbm_access=access)[1]
+ sbm_access=access,
+ restrictions_and_desc=CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS,
+ doctypes_and_desc=CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES,
+ **CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC)[1]
body += '''
''' % \
{'apply_changes': _("Apply changes"),
'cancel_changes': _("Cancel all changes"),
'recid': argd['recid'],
'access': access,
'ln': argd['ln'],
'CFG_SITE_URL': CFG_SITE_URL}
body += websubmit_templates.tmpl_page_do_not_leave_submission_js(argd['ln'], enabled=True)
return page(title = _("Document File Manager") + (argd['recid'] and (': ' + _("Record #%i") % argd['recid']) or ''),
navtrail=navtrail,
navtrail_append_title_p=0,
metaheaderadd = get_upload_file_interface_javascript(form_url_params='?access='+access) + \
get_upload_file_interface_css(),
body = body,
uid = uid,
language=argd['ln'],
req=req,
navmenuid='admin')
def managedocfilesasync(self, req, form):
"Upload file and returns upload interface"
argd = wash_urlargd(form, {
'ln': (str, ''),
'recid': (int, 1),
'doctype': (str, ''),
'access': (str, ''),
'indir': (str, ''),
})
user_info = collect_user_info(req)
include_headers = False
# User submitted either through WebSubmit, or admin interface.
if form.has_key('doctype') and form.has_key('indir') \
and form.has_key('access'):
# Submitted through WebSubmit. Check rights
include_headers = True
working_dir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR,
argd['indir'], argd['doctype'],
argd['access'])
try:
assert(working_dir == os.path.abspath(working_dir))
except AssertionError:
return apache.HTTP_UNAUTHORIZED
try:
# Retrieve recid from working_dir, safer.
recid_fd = file(os.path.join(working_dir, 'SN'))
recid = int(recid_fd.read())
recid_fd.close()
except:
recid = ""
try:
act_fd = file(os.path.join(working_dir, 'act'))
action = act_fd.read()
act_fd.close()
except:
action = ""
# Is user authorized to perform this action?
(auth_code, auth_msg) = acc_authorize_action(user_info,
"submit",
doctype=argd['doctype'],
act=action)
else:
# User must be allowed to attach files
(auth_code, auth_msg) = acc_authorize_action(user_info,
'runbibdocfile')
recid = argd['recid']
if auth_code:
return apache.HTTP_UNAUTHORIZED
return create_file_upload_interface(recid=recid,
ln=argd['ln'],
print_outside_form_tag=False,
print_envelope=False,
form=form,
include_headers=include_headers,
sbm_indir=argd['indir'],
sbm_access=argd['access'],
sbm_doctype=argd['doctype'],
uid=user_info['uid'])[1]
def uploadfile(self, req, form):
"""
Similar to /submit, but only consider files. Nice for
asynchronous Javascript uploads. Should be used to upload a
single file.
Also try to create an icon, and return URL to file(s) + icon(s)
Authentication is performed based on session ID passed as
parameter instead of cookie-based authentication, due to the
use of this URL by the Flash plugin (to upload multiple files
at once), which does not route cookies.
FIXME: consider adding /deletefile and /modifyfile functions +
parsing of additional parameters to rename files, add
comments, restrictions, etc.
"""
if sys.hexversion < 0x2060000:
try:
import simplejson as json
simplejson_available = True
except ImportError:
# Okay, no Ajax app will be possible, but continue anyway,
# since this package is only recommended, not mandatory.
simplejson_available = False
else:
import json
simplejson_available = True
argd = wash_urlargd(form, {
'doctype': (str, ''),
'access': (str, ''),
'indir': (str, ''),
'session_id': (str, ''),
'rename': (str, ''),
})
curdir = None
if not form.has_key("indir") or \
not form.has_key("doctype") or \
not form.has_key("access"):
return apache.HTTP_BAD_REQUEST
else:
curdir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR,
argd['indir'],
argd['doctype'],
argd['access'])
user_info = collect_user_info(req)
if form.has_key("session_id"):
# Are we uploading using Flash, which does not transmit
# cookie? The expect to receive session_id as a form
# parameter. First check that IP addresses do not
# mismatch. A ValueError will be raises if there is
# something wrong
session = get_session(req=req, sid=argd['session_id'])
try:
session = get_session(req=req, sid=argd['session_id'])
except ValueError, e:
return apache.HTTP_BAD_REQUEST
# Retrieve user information. We cannot rely on the session here.
res = run_sql("SELECT uid FROM session WHERE session_key=%s", (argd['session_id'],))
if len(res):
uid = res[0][0]
user_info = collect_user_info(uid)
try:
act_fd = file(os.path.join(curdir, 'act'))
action = act_fd.read()
act_fd.close()
except:
act = ""
# Is user authorized to perform this action?
(auth_code, auth_message) = acc_authorize_action(uid, "submit",
verbose=0,
doctype=argd['doctype'],
act=action)
if acc_is_role("submit", doctype=argd['doctype'], act=action) and auth_code != 0:
# User cannot submit
return apache.HTTP_UNAUTHORIZED
else:
# Process the upload and get the response
added_files = {}
for key, formfields in form.items():
filename = key.replace("[]", "")
file_to_open = os.path.join(curdir, filename)
if hasattr(formfields, "filename") and formfields.filename:
dir_to_open = os.path.abspath(os.path.join(curdir,
'files',
str(user_info['uid']),
key))
try:
assert(dir_to_open.startswith(CFG_WEBSUBMIT_STORAGEDIR))
except AssertionError:
register_exception(req=req, prefix='curdir="%s", key="%s"' % (curdir, key))
return apache.HTTP_FORBIDDEN
if not os.path.exists(dir_to_open):
try:
os.makedirs(dir_to_open)
except:
register_exception(req=req, alert_admin=True)
return apache.HTTP_FORBIDDEN
filename = formfields.filename
## Before saving the file to disc, wash the filename (in particular
## washing away UNIX and Windows (e.g. DFS) paths):
filename = os.path.basename(filename.split('\\')[-1])
filename = filename.strip()
if filename != "":
# Check that file does not already exist
n = 1
while os.path.exists(os.path.join(dir_to_open, filename)):
#dirname, basename, extension = decompose_file(new_destination_path)
basedir, name, extension = decompose_file(filename)
new_name = propose_next_docname(name)
filename = new_name + extension
# This may be dangerous if the file size is bigger than the available memory
fp = open(os.path.join(dir_to_open, filename), "w")
fp.write(formfields.file.read())
fp.close()
fp = open(os.path.join(curdir, "lastuploadedfile"), "w")
fp.write(filename)
fp.close()
fp = open(file_to_open, "w")
fp.write(filename)
fp.close()
try:
# Create icon
(icon_path, icon_name) = create_icon(
{ 'input-file' : os.path.join(dir_to_open, filename),
'icon-name' : filename, # extension stripped automatically
'icon-file-format' : 'gif',
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'icon-scale' : "300>", # Resize only if width > 300
'verbosity' : 0,
})
icons_dir = os.path.join(os.path.join(curdir,
'icons',
str(user_info['uid']),
key))
if not os.path.exists(icons_dir):
# Create uid/icons dir if needed
os.makedirs(icons_dir)
os.rename(os.path.join(icon_path, icon_name),
os.path.join(icons_dir, icon_name))
added_files[key] = {'name': filename,
'iconName': icon_name}
except InvenioWebSubmitIconCreatorError, e:
# We could not create the icon
added_files[key] = {'name': filename}
continue
else:
return apache.HTTP_BAD_REQUEST
# Send our response
if simplejson_available:
return json.dumps(added_files)
def getuploadedfile(self, req, form):
"""
Stream uploaded files.
For the moment, restrict to files in ./curdir/files/uid or
./curdir/icons/uid directory, so that we are sure we stream
files only to the user who uploaded them.
"""
argd = wash_urlargd(form, {'indir': (str, None),
'doctype': (str, None),
'access': (str, None),
'icon': (int, 0),
'key': (str, None),
'filename': (str, None)})
if None in argd.values():
return apache.HTTP_BAD_REQUEST
uid = getUid(req)
if argd['icon']:
file_path = os.path.join(CFG_WEBSUBMIT_STORAGEDIR,
argd['indir'],
argd['doctype'],
argd['access'],
'icons',
str(uid),
argd['key'],
argd['filename']
)
else:
file_path = os.path.join(CFG_WEBSUBMIT_STORAGEDIR,
argd['indir'],
argd['doctype'],
argd['access'],
'files',
str(uid),
argd['key'],
argd['filename']
)
abs_file_path = os.path.abspath(file_path)
if abs_file_path.startswith(CFG_WEBSUBMIT_STORAGEDIR):
# Check if file exist. Note that icon might not yet have
# been created.
for i in range(5):
if os.path.exists(abs_file_path):
return stream_file(req, abs_file_path)
time.sleep(1)
# Send error 404 in all other cases
return apache.HTTP_NOT_FOUND
def attachfile(self, req, form):
"""
Process requests received from FCKeditor to upload files.
If the uploaded file is an image, create an icon version
"""
if not fckeditor_available:
return apache.HTTP_NOT_FOUND
if not form.has_key('type'):
form['type'] = 'File'
if not form.has_key('NewFile') or \
not form['type'] in \
['File', 'Image', 'Flash', 'Media']:
return apache.HTTP_NOT_FOUND
uid = getUid(req)
# URL where the file can be fetched after upload
user_files_path = '%(CFG_SITE_URL)s/submit/getattachedfile/%(uid)s' % \
{'uid': uid,
'CFG_SITE_URL': CFG_SITE_URL}
# Path to directory where uploaded files are saved
user_files_absolute_path = '%(CFG_PREFIX)s/var/tmp/attachfile/%(uid)s' % \
{'uid': uid,
'CFG_PREFIX': CFG_PREFIX}
try:
os.makedirs(user_files_absolute_path)
except:
pass
# Create a Connector instance to handle the request
conn = FCKeditorConnectorInvenio(form, recid=-1, uid=uid,
allowed_commands=['QuickUpload'],
allowed_types = ['File', 'Image', 'Flash', 'Media'],
user_files_path = user_files_path,
user_files_absolute_path = user_files_absolute_path)
user_info = collect_user_info(req)
(auth_code, auth_message) = acc_authorize_action(user_info, 'attachsubmissionfile')
if user_info['email'] == 'guest':
# User is guest: must login prior to upload
data = conn.sendUploadResults(1, '', '', 'Please login before uploading file.')
elif auth_code:
# User cannot submit
data = conn.sendUploadResults(1, '', '', 'Sorry, you are not allowed to submit files.')
else:
# Process the upload and get the response
data = conn.doResponse()
# At this point, the file has been uploaded. The FCKeditor
# submit the image in form['NewFile']. However, the image
# might have been renamed in between by the FCK connector on
# the server side, by appending (%04d) at the end of the base
# name. Retrieve that file
uploaded_file_path = os.path.join(user_files_absolute_path,
form['type'].lower(),
form['NewFile'].filename)
uploaded_file_path = retrieve_most_recent_attached_file(uploaded_file_path)
uploaded_file_name = os.path.basename(uploaded_file_path)
# Create an icon
if form.get('type','') == 'Image':
try:
(icon_path, icon_name) = create_icon(
{ 'input-file' : uploaded_file_path,
'icon-name' : os.path.splitext(uploaded_file_name)[0],
'icon-file-format' : os.path.splitext(uploaded_file_name)[1][1:] or 'gif',
'multipage-icon' : False,
'multipage-icon-delay' : 100,
'icon-scale' : "300>", # Resize only if width > 300
'verbosity' : 0,
})
# Move original file to /original dir, and replace it with icon file
original_user_files_absolute_path = os.path.join(user_files_absolute_path,
'image', 'original')
if not os.path.exists(original_user_files_absolute_path):
# Create /original dir if needed
os.mkdir(original_user_files_absolute_path)
os.rename(uploaded_file_path,
original_user_files_absolute_path + os.sep + uploaded_file_name)
os.rename(icon_path + os.sep + icon_name,
uploaded_file_path)
except InvenioWebSubmitIconCreatorError, e:
pass
# Transform the headers into something ok for mod_python
for header in conn.headers:
if not header is None:
if header[0] == 'Content-Type':
req.content_type = header[1]
else:
req.headers_out[header[0]] = header[1]
# Send our response
req.send_http_header()
req.write(data)
def _lookup(self, component, path):
""" This handler is invoked for the dynamic URLs (for getting
and putting attachments) Eg:
/submit/getattachedfile/41336978/image/myfigure.png
/submit/attachfile/41336978/image/myfigure.png
"""
if component == 'getattachedfile' and len(path) > 2:
uid = path[0] # uid of the submitter
file_type = path[1] # file, image, flash or media (as
# defined by FCKeditor)
if file_type in ['file', 'image', 'flash', 'media']:
file_name = '/'.join(path[2:]) # the filename
def answer_get(req, form):
"""Accessing files attached to submission."""
form['file'] = file_name
form['type'] = file_type
form['uid'] = uid
return self.getattachedfile(req, form)
return answer_get, []
# All other cases: file not found
return None, []
def getattachedfile(self, req, form):
"""
Returns a file uploaded to the submission 'drop box' by the
FCKeditor.
"""
argd = wash_urlargd(form, {'file': (str, None),
'type': (str, None),
'uid': (int, 0)})
# Can user view this record, i.e. can user access its
# attachments?
uid = getUid(req)
user_info = collect_user_info(req)
if not argd['file'] is None:
# Prepare path to file on disk. Normalize the path so that
# ../ and other dangerous components are removed.
path = os.path.abspath(CFG_PREFIX + '/var/tmp/attachfile/' + \
'/' + str(argd['uid']) + \
'/' + argd['type'] + '/' + argd['file'])
# Check that we are really accessing attachements
# directory, for the declared record.
if path.startswith(CFG_PREFIX + '/var/tmp/attachfile/') and os.path.exists(path):
return stream_file(req, path)
# Send error 404 in all other cases
return(apache.HTTP_NOT_FOUND)
def direct(self, req, form):
"""Directly redirected to an initialized submission."""
args = wash_urlargd(form, {'sub': (str, ''),
'access' : (str, '')})
sub = args['sub']
access = args['access']
ln = args['ln']
_ = gettext_set_language(ln)
uid = getUid(req)
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
return page_not_authorized(req, "direct",
navmenuid='submit')
myQuery = req.args
if not sub:
return warningMsg(_("Sorry, 'sub' parameter missing..."), req, ln=ln)
res = run_sql("SELECT docname,actname FROM sbmIMPLEMENT WHERE subname=%s", (sub,))
if not res:
return warningMsg(_("Sorry. Cannot analyse parameter"), req, ln=ln)
else:
# get document type
doctype = res[0][0]
# get action name
action = res[0][1]
# retrieve other parameter values
params = dict(form)
# find existing access number
if not access:
# create 'unique' access number
pid = os.getpid()
now = time.time()
access = "%i_%s" % (now,pid)
# retrieve 'dir' value
res = run_sql ("SELECT dir FROM sbmACTION WHERE sactname=%s", (action,))
dir = res[0][0]
mainmenu = req.headers_in.get('referer')
params['access'] = access
params['act'] = action
params['doctype'] = doctype
params['startPg'] = '1'
params['mainmenu'] = mainmenu
params['ln'] = ln
params['indir'] = dir
url = "%s/submit?%s" % (CFG_SITE_URL, urlencode(params))
redirect_to_url(req, url)
def sub(self, req, form):
"""DEPRECATED: /submit/sub is deprecated now, so raise email to the admin (but allow submission to continue anyway)"""
args = wash_urlargd(form, {'password': (str, '')})
uid = getUid(req)
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
return page_not_authorized(req, "../sub/",
navmenuid='submit')
try:
raise DeprecationWarning, 'submit/sub handler has been used. Please use submit/direct. e.g. "submit/sub?RN=123@SBIFOO" -> "submit/direct?RN=123&sub=SBIFOO"'
except DeprecationWarning:
register_exception(req=req, alert_admin=True)
ln = args['ln']
_ = gettext_set_language(ln)
#DEMOBOO_RN=DEMO-BOOK-2008-001&ln=en&password=1223993532.26572%40APPDEMOBOO
params = dict(form)
password = args['password']
if password:
del params['password']
if "@" in password:
params['access'], params['sub'] = password.split('@', 1)
else:
params['sub'] = password
else:
args = str(req.args).split('@')
if len(args) > 1:
params = {'sub' : args[-1]}
args = '@'.join(args[:-1])
params.update(cgi.parse_qs(args))
else:
return warningMsg(_("Sorry, invalid URL..."), req, ln=ln)
url = "%s/submit/direct?%s" % (CFG_SITE_URL, urlencode(params, doseq=True))
redirect_to_url(req, url)
def summary(self, req, form):
args = wash_urlargd(form, {
'doctype': (str, ''),
'act': (str, ''),
'access': (str, ''),
'indir': (str, '')})
uid = getUid(req)
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
return page_not_authorized(req, "../summary",
navmenuid='submit')
t=""
curdir = os.path.join(CFG_WEBSUBMIT_STORAGEDIR, args['indir'], args['doctype'], args['access'])
try:
assert(curdir == os.path.abspath(curdir))
except AssertionError:
register_exception(req=req, alert_admin=True, prefix='Possible cracking tentative: indir="%s", doctype="%s", access="%s"' % (args['indir'], args['doctype'], args['access']))
return warningMsg("Invalid parameters", req)
subname = "%s%s" % (args['act'], args['doctype'])
res = run_sql("select sdesc,fidesc,pagenb,level from sbmFIELD where subname=%s "
"order by pagenb,fieldnb", (subname,))
nbFields = 0
values = []
for arr in res:
if arr[0] != "":
val = {
'mandatory' : (arr[3] == 'M'),
'value' : '',
'page' : arr[2],
'name' : arr[0],
}
if os.path.exists(os.path.join(curdir, curdir,arr[1])):
fd = open(os.path.join(curdir, arr[1]),"r")
value = fd.read()
fd.close()
value = value.replace("\n"," ")
value = value.replace("Select:","")
else:
value = ""
val['value'] = value
values.append(val)
return websubmit_templates.tmpl_submit_summary(
ln = args['ln'],
values = values,
)
def index(self, req, form):
args = wash_urlargd(form, {
'c': (str, CFG_SITE_NAME),
'doctype': (str, ''),
'act': (str, ''),
'startPg': (str, "1"),
'access': (str, ''),
'mainmenu': (str, ''),
'fromdir': (str, ''),
'nextPg': (str, ''),
'nbPg': (str, ''),
'curpage': (str, '1'),
'step': (str, '0'),
'mode': (str, 'U'),
})
## Strip whitespace from beginning and end of doctype and action:
args["doctype"] = args["doctype"].strip()
args["act"] = args["act"].strip()
def _index(req, c, ln, doctype, act, startPg, access,
mainmenu, fromdir, nextPg, nbPg, curpage, step,
mode):
uid = getUid(req)
if isGuestUser(uid):
return redirect_to_url(req, "%s/youraccount/login%s" % (
CFG_SITE_SECURE_URL,
make_canonical_urlargd({
'referer' : CFG_SITE_URL + req.unparsed_uri, 'ln' : args['ln']}, {})), norobot=True)
if uid == -1 or CFG_ACCESS_CONTROL_LEVEL_SITE >= 1:
return page_not_authorized(req, "../submit",
navmenuid='submit')
if CFG_CERN_SITE:
## HACK BEGIN: this is a hack for CMS and ATLAS draft
from invenio.webuser import collect_user_info
user_info = collect_user_info(req)
if doctype == 'CMSPUB' and act == "" and 'cds-admin [CERN]' not in user_info['group'] and not user_info['email'].lower() == 'cds.support@cern.ch':
if 'cms-publication-committee-chair [CERN]' not in user_info['group']:
return page_not_authorized(req, "../submit", text="In order to access this submission interface you need to be member of the CMS Publication Committee Chair.",
navmenuid='submit')
elif doctype == 'ATLPUB' and 'cds-admin [CERN]' not in user_info['group'] and not user_info['email'].lower() == 'cds.support@cern.ch':
if 'atlas-gen [CERN]' not in user_info['group']:
return page_not_authorized(req, "../submit", text="In order to access this submission interface you need to be member of ATLAS.",
navmenuid='submit')
## HACK END
if doctype=="":
return home(req,c,ln)
elif act=="":
return action(req,c,ln,doctype)
elif int(step)==0:
return interface(req, c, ln, doctype, act, startPg, access, mainmenu, fromdir, nextPg, nbPg, curpage)
else:
return endaction(req, c, ln, doctype, act, startPg, access,mainmenu, fromdir, nextPg, nbPg, curpage, step, mode)
return _index(req, **args)
# Answer to both /submit/ and /submit
__call__ = index
def errorMsg(title, req, c=None, ln=CFG_SITE_LANG):
# load the right message language
_ = gettext_set_language(ln)
if c is None:
c = CFG_SITE_NAME_INTL.get(ln, CFG_SITE_NAME)
return page(title = _("Error"),
body = create_error_box(req, title=str(title), verbose=0, ln=ln),
description="%s - Internal Error" % c,
keywords="%s, Internal Error" % c,
uid = getUid(req),
language=ln,
req=req,
navmenuid='submit')
def warningMsg(title, req, c=None, ln=CFG_SITE_LANG):
# load the right message language
_ = gettext_set_language(ln)
if c is None:
c = CFG_SITE_NAME_INTL.get(ln, CFG_SITE_NAME)
return page(title = _("Warning"),
body = title,
description="%s - Internal Error" % c,
keywords="%s, Internal Error" % c,
uid = getUid(req),
language=ln,
req=req,
navmenuid='submit')
def print_warning(msg, type='', prologue=' ', epilogue=' '):
"""Prints warning message and flushes output."""
if msg:
return websubmit_templates.tmpl_print_warning(
msg = msg,
type = type,
prologue = prologue,
epilogue = epilogue,
)
else:
return ''
def retrieve_most_recent_attached_file(file_path):
"""
Retrieve the latest file that has been uploaded with the
FCKeditor. This is the only way to retrieve files that the
FCKeditor has renamed after the upload.
Eg: 'prefix/image.jpg' was uploaded but did already
exist. FCKeditor silently renamed it to 'prefix/image(1).jpg':
>>> retrieve_most_recent_attached_file('prefix/image.jpg')
'prefix/image(1).jpg'
"""
(base_path, filename) = os.path.split(file_path)
base_name = os.path.splitext(filename)[0]
file_ext = os.path.splitext(filename)[1][1:]
most_recent_filename = filename
i = 0
while True:
i += 1
possible_filename = "%s(%d).%s" % \
(base_name, i, file_ext)
if os.path.exists(base_path + os.sep + possible_filename):
most_recent_filename = possible_filename
else:
break
return os.path.join(base_path, most_recent_filename)
|