diff --git a/config/invenio.conf b/config/invenio.conf
index cd107e53f..86504a333 100644
--- a/config/invenio.conf
+++ b/config/invenio.conf
@@ -1,1138 +1,1133 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
###################################################
## About 'invenio.conf' and 'invenio-local.conf' ##
###################################################
## The 'invenio.conf' file contains the vanilla default configuration
## parameters of a CDS Invenio installation, as coming from the
## distribution. The file should be self-explanatory. Once installed
## in its usual location (usually /opt/cds-invenio/etc), you could in
## principle go ahead and change the values according to your local
## needs.
##
## However, you can also create a file named 'invenio-local.conf' in
## the same directory where 'invenio.conf' lives and put there only
## the localizations you need to have different from the default ones.
## For example:
##
## $ cat /opt/cds-invenio/etc/invenio-local.conf
## [Invenio]
## CFG_SITE_URL = http://your.site.com
## CFG_SITE_SECURE_URL = https://your.site.com
## CFG_SITE_ADMIN_EMAIL = john.doe@your.site.com
## CFG_SITE_SUPPORT_EMAIL = john.doe@your.site.com
##
## The Invenio system will then read both the default invenio.conf
## file and your customized invenio-local.conf file and it will
## override any default options with the ones you have set in your
## local file. This cascading of configuration parameters will ease
## you future upgrades.
[Invenio]
###################################
## Part 1: Essential parameters ##
###################################
## This part defines essential CDS Invenio internal parameters that
## everybody should override, like the name of the server or the email
## address of the local CDS Invenio administrator.
## CFG_DATABASE_* - specify which MySQL server to use, the name of the
## database to use, and the database access credentials.
CFG_DATABASE_HOST = localhost
CFG_DATABASE_PORT = 3306
CFG_DATABASE_NAME = cdsinvenio
CFG_DATABASE_USER = cdsinvenio
CFG_DATABASE_PASS = my123p$ss
## CFG_SITE_URL - specify URL under which your installation will be
## visible. For example, use "http://your.site.com". Do not leave
## trailing slash.
CFG_SITE_URL = http://localhost
## CFG_SITE_SECURE_URL - specify secure URL under which your
## installation secure pages such as login or registration will be
## visible. For example, use "https://your.site.com". Do not leave
## trailing slash. If you don't plan on using HTTPS, then you may
## leave this empty.
CFG_SITE_SECURE_URL = https://localhost
## CFG_SITE_NAME -- the visible name of your CDS Invenio installation.
CFG_SITE_NAME = Atlantis Institute of Fictive Science
## CFG_SITE_NAME_INTL -- the international versions of CFG_SITE_NAME
## in various languages. (See also CFG_SITE_LANGS below.)
CFG_SITE_NAME_INTL_en = Atlantis Institute of Fictive Science
CFG_SITE_NAME_INTL_fr = Atlantis Institut des Sciences Fictives
CFG_SITE_NAME_INTL_de = Atlantis Institut der fiktiven Wissenschaft
CFG_SITE_NAME_INTL_es = Atlantis Instituto de la Ciencia Fictive
CFG_SITE_NAME_INTL_ca = Institut Atlantis de Ciència Fictícia
CFG_SITE_NAME_INTL_pt = Instituto Atlantis de Ciência Fictícia
CFG_SITE_NAME_INTL_it = Atlantis Istituto di Scienza Fittizia
CFG_SITE_NAME_INTL_ru = Атлантис Институт фиктивных Наук
CFG_SITE_NAME_INTL_sk = Atlantis Inštitút Fiktívnych Vied
CFG_SITE_NAME_INTL_cs = Atlantis Institut Fiktivních Věd
CFG_SITE_NAME_INTL_no = Atlantis Institutt for Fiktiv Vitenskap
CFG_SITE_NAME_INTL_sv = Atlantis Institut för Fiktiv Vetenskap
CFG_SITE_NAME_INTL_el = Ινστιτούτο Φανταστικών Επιστημών Ατλαντίδος
CFG_SITE_NAME_INTL_uk = Інститут вигаданих наук в Атлантісі
CFG_SITE_NAME_INTL_ja = Fictive 科学のAtlantis の協会
CFG_SITE_NAME_INTL_pl = Instytut Fikcyjnej Nauki Atlantis
CFG_SITE_NAME_INTL_bg = Институт за фиктивни науки Атлантис
CFG_SITE_NAME_INTL_hr = Institut Fiktivnih Znanosti Atlantis
CFG_SITE_NAME_INTL_zh_CN = 阿特兰提斯虚拟科学学院
CFG_SITE_NAME_INTL_zh_TW = 阿特蘭提斯虛擬科學學院
CFG_SITE_NAME_INTL_hu = Kitalált Tudományok Atlantiszi Intézete
CFG_SITE_NAME_INTL_af = Atlantis Instituut van Fiktiewe Wetenskap
CFG_SITE_NAME_INTL_gl = Instituto Atlantis de Ciencia Fictive
CFG_SITE_NAME_INTL_ro = Institutul Atlantis al Ştiinţelor Fictive
CFG_SITE_NAME_INTL_rw = Atlantis Ishuri Rikuru Ry'ubuhanga
CFG_SITE_NAME_INTL_ka = ატლანტიდის ფიქტიური მეცნიერების ინსტიტუტი
## CFG_SITE_LANG -- the default language of the interface: '
CFG_SITE_LANG = en
## CFG_SITE_LANGS -- list of all languages the user interface should
## be available in, separated by commas. The order specified below
## will be respected on the interface pages. A good default would be
## to use the alphabetical order. Currently supported languages
## include Afrikaans, Bulgarian, Catalan, Czech, German, Georgian,
## Greek, English, Spanish, French, Croatian, Hungarian, Galician,
## Italian, Japanese, Kinyarwanda, Norwegian, Polish, Portuguese,
## Romanian, Russian, Slovak, Swedish, Ukrainian, Chinese (China),
## Chinese (Taiwan), so that the eventual maximum you can currently
## select is "af,bg,ca,cs,de,el,en,es,fr,hr,gl,it,ka,rw,hu,ja,no,pl,pt,ro,ru,sk,sv,uk,zh_CN,zh_TW".
CFG_SITE_LANGS = af,bg,ca,cs,de,el,en,es,fr,hr,gl,it,ka,rw,hu,ja,no,pl,pt,ro,ru,sk,sv,uk,zh_CN,zh_TW
## CFG_SITE_SUPPORT_EMAIL -- the email address of the support team for
## this installation:
CFG_SITE_SUPPORT_EMAIL = cds.support@cern.ch
## CFG_SITE_ADMIN_EMAIL -- the email address of the 'superuser' for
## this installation. Enter your email address below and login with
## this address when using CDS Invenio inistration modules. You
## will then be automatically recognized as superuser of the system.
CFG_SITE_ADMIN_EMAIL = cds.support@cern.ch
## CFG_SITE_EMERGENCY_PHONE_NUMBERS -- list of mobile phone numbers to
## which an sms should be sent in case of emergency (e.g. bibsched queue
## has been stopped because of an error).
## Note that in order to use this function, if CFG_CERN_SITE is set to 0,
## the function send_sms in errorlib should be reimplemented.
CFG_SITE_EMERGENCY_PHONE_NUMBERS =
## CFG_SITE_ADMIN_EMAIL_EXCEPTIONS -- set this to 0 if you do not want
## to receive any captured exception via email to CFG_SITE_ADMIN_EMAIL
## address. Captured exceptions will still be available in
## var/log/invenio.err file. Set this to 1 if you want to receive
## some of the captured exceptions (this depends on the actual place
## where the exception is captured). Set this to 2 if you want to
## receive all captured exceptions.
CFG_SITE_ADMIN_EMAIL_EXCEPTIONS = 1
## CFG_CERN_SITE -- do we want to enable CERN-specific code?
## Put "1" for "yes" and "0" for "no".
CFG_CERN_SITE = 0
## CFG_INSPIRE_SITE -- do we want to enable INSPIRE-specific code?
## Put "1" for "yes" and "0" for "no".
CFG_INSPIRE_SITE = 0
## CFG_ADS_SITE -- do we want to enable ADS-specific code?
## Put "1" for "yes" and "0" for "no".
CFG_ADS_SITE = 0
## CFG_DEVEL_SITE -- is this a development site? If it is, you might
## prefer that it does not do certain things. For example, you might
## not want WebSubmit to send certain emails or trigger certain
## processes on a development site.
## Put "1" for "yes" (this is a development site) or "0" for "no"
## (this isn't a development site.)
CFG_DEVEL_SITE = 0
################################
## Part 2: Web page style ##
################################
## The variables affecting the page style. The most important one is
## the 'template skin' you would like to use and the obfuscation mode
## for your email addresses. Please refer to the WebStyle Admin Guide
## for more explanation. The other variables are listed here mostly
## for backwards compatibility purposes only.
## CFG_WEBSTYLE_TEMPLATE_SKIN -- what template skin do you want to
## use?
CFG_WEBSTYLE_TEMPLATE_SKIN = default
## CFG_WEBSTYLE_EMAIL_ADDRESSES_OBFUSCATION_MODE. How do we "protect"
## email addresses from undesired automated email harvesters? This
## setting will not affect 'support' and 'admin' emails.
## NOTE: there is no ultimate solution to protect against email
## harvesting. All have drawbacks and can more or less be
## circumvented. Choose you preferred mode ([t] means "transparent"
## for the user):
## -1: hide all emails.
## [t] 0 : no protection, email returned as is.
## foo@example.com => foo@example.com
## 1 : basic email munging: replaces @ by [at] and . by [dot]
## foo@example.com => foo [at] example [dot] com
## [t] 2 : transparent name mangling: characters are replaced by
## equivalent HTML entities.
## foo@example.com => foo@example.com
## [t] 3 : javascript insertion. Requires Javascript enabled on client
## side.
## 4 : replaces @ and . characters by gif equivalents.
## foo@example.com => foo![[at]](at.gif)
example![[dot]](dot.gif)
com
CFG_WEBSTYLE_EMAIL_ADDRESSES_OBFUSCATION_MODE = 2
## CFG_WEBSTYLE_INSPECT_TEMPLATES -- Do we want to debug all template
## functions so that they would return HTML results wrapped in
## comments indicating which part of HTML page was created by which
## template function? Useful only for debugging Pythonic HTML
## template. See WebStyle Admin Guide for more information.
CFG_WEBSTYLE_INSPECT_TEMPLATES = 0
## (deprecated) CFG_WEBSTYLE_CDSPAGEBOXLEFTTOP -- eventual global HTML
## left top box:
CFG_WEBSTYLE_CDSPAGEBOXLEFTTOP =
## (deprecated) CFG_WEBSTYLE_CDSPAGEBOXLEFTBOTTOM -- eventual global
## HTML left bottom box:
CFG_WEBSTYLE_CDSPAGEBOXLEFTBOTTOM =
## (deprecated) CFG_WEBSTYLE_CDSPAGEBOXRIGHTTOP -- eventual global
## HTML right top box:
CFG_WEBSTYLE_CDSPAGEBOXRIGHTTOP =
## (deprecated) CFG_WEBSTYLE_CDSPAGEBOXRIGHTBOTTOM -- eventual global
## HTML right bottom box:
CFG_WEBSTYLE_CDSPAGEBOXRIGHTBOTTOM =
## CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST -- when certain HTTP status
## codes are raised to the WSGI handler, the corresponding exceptions
## and error messages can be sent to the system administrator for
## inspecting. This is useful to detect and correct errors. The
## variable represents a comma-separated list of HTTP statuses that
## should alert admin. Wildcards are possible. If the status is
## followed by an "r", it means that a referer is required to exist
## (useful to distinguish broken known links from URL typos when 404
## errors are raised).
CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST = 404r,400,5*,41*
##################################
## Part 3: WebSearch parameters ##
##################################
## This section contains some configuration parameters for WebSearch
## module. Please note that WebSearch is mostly configured on
## run-time via its WebSearch Admin web interface. The parameters
## below are the ones that you do not probably want to modify very
## often during the runtime. (Note that you may modify them
## afterwards too, though.)
## CFG_WEBSEARCH_SEARCH_CACHE_SIZE -- how many queries we want to
## cache in memory per one Apache httpd process? This cache is used
## mainly for "next/previous page" functionality, but it caches also
## "popular" user queries if more than one user happen to search for
## the same thing. Note that large numbers may lead to great memory
## consumption. We recommend a value not greater than 100.
CFG_WEBSEARCH_SEARCH_CACHE_SIZE = 100
## CFG_WEBSEARCH_FIELDS_CONVERT -- if you migrate from an older
## system, you may want to map field codes of your old system (such as
## 'ti') to CDS Invenio/MySQL ("title"). Use Python dictionary syntax
## for the translation table, e.g. {'wau':'author', 'wti':'title'}.
## Usually you don't want to do that, and you would use empty dict {}.
CFG_WEBSEARCH_FIELDS_CONVERT = {}
## CFG_WEBSEARCH_LIGHTSEARCH_PATTERN_BOX_WIDTH -- width of the
## search pattern window in the light search interface, in
## characters. CFG_WEBSEARCH_LIGHTSEARCH_PATTERN_BOX_WIDTH = 60
CFG_WEBSEARCH_LIGHTSEARCH_PATTERN_BOX_WIDTH = 60
## CFG_WEBSEARCH_SIMPLESEARCH_PATTERN_BOX_WIDTH -- width of the search
## pattern window in the simple search interface, in characters.
CFG_WEBSEARCH_SIMPLESEARCH_PATTERN_BOX_WIDTH = 40
## CFG_WEBSEARCH_ADVANCEDSEARCH_PATTERN_BOX_WIDTH -- width of the
## search pattern window in the advanced search interface, in
## characters.
CFG_WEBSEARCH_ADVANCEDSEARCH_PATTERN_BOX_WIDTH = 30
## CFG_WEBSEARCH_NB_RECORDS_TO_SORT -- how many records do we still
## want to sort? For higher numbers we print only a warning and won't
## perform any sorting other than default 'latest records first', as
## sorting would be very time consuming then. We recommend a value of
## not more than a couple of thousands.
CFG_WEBSEARCH_NB_RECORDS_TO_SORT = 1000
## CFG_WEBSEARCH_CALL_BIBFORMAT -- if a record is being displayed but
## it was not preformatted in the "HTML brief" format, do we want to
## call BibFormatting on the fly? Put "1" for "yes" and "0" for "no".
## Note that "1" will display the record exactly as if it were fully
## preformatted, but it may be slow due to on-the-fly processing; "0"
## will display a default format very fast, but it may not have all
## the fields as in the fully preformatted HTML brief format. Note
## also that this option is active only for old (PHP) formats; the new
## (Python) formats are called on the fly by default anyway, since
## they are much faster. When usure, please set "0" here.
CFG_WEBSEARCH_CALL_BIBFORMAT = 0
## CFG_WEBSEARCH_USE_ALEPH_SYSNOS -- do we want to make old SYSNOs
## visible rather than MySQL's record IDs? You may use this if you
## migrate from a different e-doc system, and you store your old
## system numbers into 970__a. Put "1" for "yes" and "0" for
## "no". Usually you don't want to do that, though.
CFG_WEBSEARCH_USE_ALEPH_SYSNOS = 0
## CFG_WEBSEARCH_I18N_LATEST_ADDITIONS -- Put "1" if you want the
## "Latest Additions" in the web collection pages to show
## internationalized records. Useful only if your brief BibFormat
## templates contains internationalized strings. Otherwise put "0" in
## order not to slow down the creation of latest additions by WebColl.
CFG_WEBSEARCH_I18N_LATEST_ADDITIONS = 0
## CFG_WEBSEARCH_INSTANT_BROWSE -- the number of records to display
## under 'Latest Additions' in the web collection pages.
CFG_WEBSEARCH_INSTANT_BROWSE = 10
## CFG_WEBSEARCH_INSTANT_BROWSE_RSS -- the number of records to
## display in the RSS feed.
CFG_WEBSEARCH_INSTANT_BROWSE_RSS = 25
## CFG_WEBSEARCH_RSS_I18N_COLLECTIONS -- comma-separated list of
## collections that feature an internationalized RSS feed on their
## main seach interface page created by webcoll. Other collections
## will have RSS feed using CFG_SITE_LANG.
CFG_WEBSEARCH_RSS_I18N_COLLECTIONS =
## CFG_WEBSEARCH_RSS_TTL -- number of minutes that indicates how long
## a feed cache is valid.
CFG_WEBSEARCH_RSS_TTL = 360
## CFG_WEBSEARCH_RSS_MAX_CACHED_REQUESTS -- maximum number of request kept
## in cache. If the cache is filled, following request are not cached.
CFG_WEBSEARCH_RSS_MAX_CACHED_REQUESTS = 1000
## CFG_WEBSEARCH_AUTHOR_ET_AL_THRESHOLD -- up to how many author names
## to print explicitely; for more print "et al". Note that this is
## used in default formatting that is seldomly used, as usually
## BibFormat defines all the format. The value below is only used
## when BibFormat fails, for example.
CFG_WEBSEARCH_AUTHOR_ET_AL_THRESHOLD = 3
## CFG_WEBSEARCH_NARROW_SEARCH_SHOW_GRANDSONS -- whether to show or
## not collection grandsons in Narrow Search boxes (sons are shown by
## default, grandsons are configurable here). Use 0 for no and 1 for
## yes.
CFG_WEBSEARCH_NARROW_SEARCH_SHOW_GRANDSONS = 1
## CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX -- shall we
## create help links for Ellis, Nick or Ellis, Nicholas and friends
## when Ellis, N was searched for? Useful if you have one author
## stored in the database under several name formats, namely surname
## comma firstname and surname comma initial cataloging policy. Use 0
## for no and 1 for yes.
CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX = 1
## CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS -- jsMath is a JavaScript
## library that renders (La)TeX mathematical formulas in the client
## browser. This parameter must contain a comma-separated list of
## output formats for which to apply the jsMath rendering, for example
## "hb,hd". If the list is empty, jsMath is disabled.
CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS =
## CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_TIMEOUT -- when searching
## external collections (e.g. SPIRES, CiteSeer, etc), how many seconds
## do we wait for reply before abandonning?
CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_TIMEOUT = 5
## CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_MAXRESULTS -- how many
## results do we fetch?
CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_MAXRESULTS = 10
## CFG_WEBSEARCH_SPLIT_BY_COLLECTION -- do we want to split the search
## results by collection or not? Use 0 for not, 1 for yes.
CFG_WEBSEARCH_SPLIT_BY_COLLECTION = 1
## CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS -- the default number of
## records to display per page in the search results pages.
CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS = 10
## CFG_WEBSEARCH_MAX_RECORDS_IN_GROUPS -- in order to limit denial of
## service attacks the total number of records per group displayed as a
## result of a search query will be limited to this number. Only the superuser
## queries will not be affected by this limit.
CFG_WEBSEARCH_MAX_RECORDS_IN_GROUPS = 200
## CFG_WEBSEARCH_PERMITTED_RESTRICTED_COLLECTIONS_LEVEL -- logged in users
## might have rights to access some restricted collections. This variable
## tweaks the kind of support the system will automatically provide to the
## user with respect to searching into these restricted collections.
## Set this to 0 in order to have the user to explicitly activate restricted
## collections in order to search into them. Set this to 1 in order to
## propose to the user the list of restricted collections to which he/she has
## rights (note: this is not yet implemented). Set this to 2 in order to
## silently add all the restricted collections to which the user has rights to
## to any query.
## Note: the system will discover which restricted collections a user has
## rights to, at login time. The time complexity of this procedure is
## proportional to the number of restricted collections. E.g. for a system
## with ~50 restricted collections, you might expect ~1s of delay in the
## login time, when this variable is set to a value higher than 0.
CFG_WEBSEARCH_PERMITTED_RESTRICTED_COLLECTIONS_LEVEL = 0
## CFG_WEBSEARCH_SHOW_COMMENT_COUNT -- do we want to show the 'N comments'
## links on the search engine pages? (useful only when you have allowed
## commenting)
CFG_WEBSEARCH_SHOW_COMMENT_COUNT = 1
## CFG_WEBSEARCH_SHOW_REVIEW_COUNT -- do we want to show the 'N reviews'
## links on the search engine pages? (useful only when you have allowed
## reviewing)
CFG_WEBSEARCH_SHOW_REVIEW_COUNT = 1
## CFG_WEBSEARCH_FULLTEXT_SNIPPETS -- how many full-text snippets to
## display for full-text searches?
CFG_WEBSEARCH_FULLTEXT_SNIPPETS = 0
## CFG_WEBSEARCH_FULLTEXT_SNIPPETS_WORDS -- how many context words
## to display around the pattern in the snippet?
CFG_WEBSEARCH_FULLTEXT_SNIPPETS_WORDS = 4
#######################################
## Part 4: BibHarvest OAI parameters ##
#######################################
## This part defines parameters for the CDS Invenio OAI gateway.
## Useful if you are running CDS Invenio as OAI data provider.
## CFG_OAI_ID_FIELD -- OAI identifier MARC field:
CFG_OAI_ID_FIELD = 909COo
## CFG_OAI_SET_FIELD -- OAI set MARC field:
CFG_OAI_SET_FIELD = 909COp
## CFG_OAI_DELETED_POLICY -- OAI deletedrecordspolicy
## (no/transient/persistent).
CFG_OAI_DELETED_POLICY = no
## CFG_OAI_ID_PREFIX -- OAI identifier prefix:
CFG_OAI_ID_PREFIX = atlantis.cern.ch
## CFG_OAI_SAMPLE_IDENTIFIER -- OAI sample identifier:
CFG_OAI_SAMPLE_IDENTIFIER = oai:atlantis.cern.ch:CERN-TH-4036
## CFG_OAI_IDENTIFY_DESCRIPTION -- description for the OAI Identify verb:
CFG_OAI_IDENTIFY_DESCRIPTION =
oai
atlantis.cern.ch
:
oai:atlantis.cern.ch:CERN-TH-4036
http://atlantis.cern.ch/
Free and unlimited use by anybody with obligation to refer to original record
Full content, i.e. preprints may not be harvested by robots
Submission restricted. Submitted documents are subject of approval by OAI repository admins.
## CFG_OAI_LOAD -- OAI number of records in a response:
CFG_OAI_LOAD = 1000
## CFG_OAI_EXPIRE -- OAI resumptionToken expiration time:
CFG_OAI_EXPIRE = 90000
## CFG_OAI_SLEEP -- service unavailable between two consecutive
## requests for CFG_OAI_SLEEP seconds:
CFG_OAI_SLEEP = 10
##################################
## Part 5: WebSubmit parameters ##
##################################
## This section contains some configuration parameters for WebSubmit
## module. Please note that WebSubmit is mostly configured on
## run-time via its WebSubmit Admin web interface. The parameters
## below are the ones that you do not probably want to modify during
## the runtime.
## CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT -- the fulltext
## documents are stored under "/opt/cds-invenio/var/data/files/gX/Y"
## directories where X is 0,1,... and Y stands for bibdoc ID. Thusly
## documents Y are grouped into directories X and this variable
## indicates the maximum number of documents Y stored in each
## directory X. This limit is imposed solely for filesystem
## performance reasons in order not to have too many subdirectories in
## a given directory.
CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT = 5000
## CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS -- a comma-separated
## list of document extensions not listed in Python standard mimetype
## library that should be recognized by Invenio.
CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS = hpg,link,lis,llb,mat,mpp,msg,docx,docm,xlsx,xlsm,xlsb,pptx,pptm,ppsx,ppsm
## CFG_BIBDOCFILE_USE_XSENDFILE -- if your web server supports
## XSendfile header, you may want to enable this feature in order for
## to Invenio tell the web server to stream files for download (after
## proper authorization checks) by web server's means. This helps to
## liberate Invenio worker processes from being busy with sending big
## files to clients. The web server will take care of that. Note:
## this feature is still somewhat experimental. Note: when enabled
## (set to 1), then you have to also regenerate Apache vhost conf
## snippets (inveniocfg --update-config-py --create-apache-conf).
CFG_BIBDOCFILE_USE_XSENDFILE = 0
## CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY -- a number between 0 and
## 1 that indicates probability with which MD5 checksum will be
## verified when streaming bibdocfile-managed files. (0.1 will cause
## the check to be performed once for every 10 downloads)
CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY = 0.1
## CFG_OPENOFFICE_SERVER_HOST -- the host where an OpenOffice Server is
## listening to. If localhost an OpenOffice server will be started
## automatically if it is not already running.
## Note: if you set this to an empty value this will disable the usage of
## OpenOffice for converting documents.
## If you set this to something different than localhost you'll have to take
## care to have an OpenOffice server running on the corresponding host and
## to install the same OpenOffice release both on the client and on the server
## side.
## In order to launch an OpenOffice server on a remote machine, just start
## the usual 'soffice' executable in this way:
## $> soffice -headless -nologo -nodefault -norestore -nofirststartwizard \
## .. -accept=socket,host=HOST,port=PORT;urp;StarOffice.ComponentContext
CFG_OPENOFFICE_SERVER_HOST = localhost
## CFG_OPENOFFICE_SERVER_PORT -- the port where an OpenOffice Server is
## listening to.
CFG_OPENOFFICE_SERVER_PORT = 2002
## CFG_OPENOFFICE_USER -- the user that will be used to launch the OpenOffice
## client. It is recommended to set this to a user who don't own files, like
## e.g. 'nobody'. You should also authorize your Apache server user to be
## able to become this user, e.g. by adding to your /etc/sudoers the following
## line:
## "apache ALL=(nobody) NOPASSWD: ALL"
## provided that apache is the username corresponding to the Apache user.
## On some machine this might be apache2 or www-data.
CFG_OPENOFFICE_USER = nobody
#################################
## Part 6: BibIndex parameters ##
#################################
## This section contains some configuration parameters for BibIndex
## module. Please note that BibIndex is mostly configured on run-time
## via its BibIndex Admin web interface. The parameters below are the
## ones that you do not probably want to modify very often during the
## runtime.
## CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY -- when fulltext indexing, do
## you want to index locally stored files only, or also external URLs?
## Use "0" to say "no" and "1" to say "yes".
CFG_BIBINDEX_FULLTEXT_INDEX_LOCAL_FILES_ONLY = 0
## CFG_BIBINDEX_REMOVE_STOPWORDS -- when indexing, do we want to remove
## stopwords? Use "0" to say "no" and "1" to say "yes".
CFG_BIBINDEX_REMOVE_STOPWORDS = 0
## CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS -- characters considered as
## alphanumeric separators of word-blocks inside words. You probably
## don't want to change this.
CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS = \!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~
## CFG_BIBINDEX_CHARS_PUNCTUATION -- characters considered as punctuation
## between word-blocks inside words. You probably don't want to
## change this.
CFG_BIBINDEX_CHARS_PUNCTUATION = \.\,\:\;\?\!\"
## CFG_BIBINDEX_REMOVE_HTML_MARKUP -- should we attempt to remove HTML markup
## before indexing? Use 1 if you have HTML markup inside metadata
## (e.g. in abstracts), use 0 otherwise.
CFG_BIBINDEX_REMOVE_HTML_MARKUP = 0
## CFG_BIBINDEX_REMOVE_LATEX_MARKUP -- should we attempt to remove LATEX markup
## before indexing? Use 1 if you have LATEX markup inside metadata
## (e.g. in abstracts), use 0 otherwise.
CFG_BIBINDEX_REMOVE_LATEX_MARKUP = 0
## CFG_BIBINDEX_MIN_WORD_LENGTH -- minimum word length allowed to be added to
## index. The terms smaller then this amount will be discarded.
## Useful to keep the database clean, however you can safely leave
## this value on 0 for up to 1,000,000 documents.
CFG_BIBINDEX_MIN_WORD_LENGTH = 0
## CFG_BIBINDEX_URLOPENER_USERNAME and CFG_BIBINDEX_URLOPENER_PASSWORD --
## access credentials to access restricted URLs, interesting only if
## you are fulltext-indexing files located on a remote server that is
## only available via username/password. But it's probably better to
## handle this case via IP or some convention; the current scheme is
## mostly there for demo only.
CFG_BIBINDEX_URLOPENER_USERNAME = mysuperuser
CFG_BIBINDEX_URLOPENER_PASSWORD = mysuperpass
## CFG_INTBITSET_ENABLE_SANITY_CHECKS --
## Enable sanity checks for integers passed to the intbitset data
## structures. It is good to enable this during debugging
## and to disable this value for speed improvements.
CFG_INTBITSET_ENABLE_SANITY_CHECKS = False
## CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES -- regular expression that matches
## docnames for which OCR is desired (set this to .* in order to enable
## OCR in general, set this to empty in order to disable it.)
CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES = scan-.*
## CFG_BIBINDEX_SPLASH_PAGES -- regular expression that matches URLs
## that are not to be indexed but that indirectly refers to documents
## that are supposed to be indexed.
CFG_BIBINDEX_SPLASH_PAGES = http://documents\.cern\.ch/setlink\?.*
#######################################
## Part 7: Access control parameters ##
#######################################
## This section contains some configuration parameters for the access
## control system. Please note that WebAccess is mostly configured on
## run-time via its WebAccess Admin web interface. The parameters
## below are the ones that you do not probably want to modify very
## often during the runtime. (If you do want to modify them during
## runtime, for example te deny access temporarily because of backups,
## you can edit access_control_config.py directly, no need to get back
## here and no need to redo the make process.)
## CFG_ACCESS_CONTROL_LEVEL_SITE -- defines how open this site is.
## Use 0 for normal operation of the site, 1 for read-only site (all
## write operations temporarily closed), 2 for site fully closed,
## 3 for also disabling any database connection.
## Useful for site maintenance.
CFG_ACCESS_CONTROL_LEVEL_SITE = 0
## CFG_ACCESS_CONTROL_LEVEL_GUESTS -- guest users access policy. Use
## 0 to allow guest users, 1 not to allow them (all users must login).
CFG_ACCESS_CONTROL_LEVEL_GUESTS = 0
## CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS -- account registration and
## activation policy. When 0, users can register and accounts are
## automatically activated. When 1, users can register but admin must
## activate the accounts. When 2, users cannot register nor update
## their email address, only admin can register accounts. When 3,
## users cannot register nor update email address nor password, only
## admin can register accounts. When 4, the same as 3 applies, nor
## user cannot change his login method. When 5, then the same as 4
## applies, plus info about how to get an account is hidden from the
## login page.
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS = 0
## CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN -- limit account
## registration to certain email addresses? If wanted, give domain
## name below, e.g. "cern.ch". If not wanted, leave it empty.
CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN =
## CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS -- send a
## notification email to the administrator when a new account is
## created? Use 0 for no, 1 for yes.
CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS = 0
## CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT -- send a
## notification email to the user when a new account is created in order to
## to verify the validity of the provided email address? Use
## 0 for no, 1 for yes.
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT = 1
## CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION -- send a
## notification email to the user when a new account is activated?
## Use 0 for no, 1 for yes.
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION = 0
## CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION -- send a
## notification email to the user when a new account is deleted or
## account demand rejected? Use 0 for no, 1 for yes.
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION = 0
## CFG_APACHE_PASSWORD_FILE -- the file where Apache user credentials
## are stored. Must be an absolute pathname. If the value does not
## start by a slash, it is considered to be the filename of a file
## located under prefix/var/tmp directory. This is useful for the
## demo site testing purposes. For the production site, if you plan
## to restrict access to some collections based on the Apache user
## authentication mechanism, you should put here an absolute path to
## your Apache password file.
CFG_APACHE_PASSWORD_FILE = demo-site-apache-user-passwords
## CFG_APACHE_GROUP_FILE -- the file where Apache user groups are
## defined. See the documentation of the preceding config variable.
CFG_APACHE_GROUP_FILE = demo-site-apache-user-groups
###################################
## Part 8: WebSession parameters ##
###################################
## This section contains some configuration parameters for tweaking
## session handling.
## CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT -- number of days after which a session
## and the corresponding cookie is considered expired.
CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT = 2
## CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER -- number of days after which a session
## and the corresponding cookie is considered expired, when the user has
## requested to permanently stay logged in.
CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER = 365
## CFG_WEBSESSION_RESET_PASSWORD_EXPIRE_IN_DAYS -- when user requested
## a password reset, for how many days is the URL valid?
CFG_WEBSESSION_RESET_PASSWORD_EXPIRE_IN_DAYS = 3
## CFG_WEBSESSION_ADDRESS_ACTIVATION_EXPIRE_IN_DAYS -- when an account
## activation email was sent, for how many days is the URL valid?
CFG_WEBSESSION_ADDRESS_ACTIVATION_EXPIRE_IN_DAYS = 3
## CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS -- when
## user won't confirm his email address and not complete
## registeration, after how many days will it expire?
CFG_WEBSESSION_NOT_CONFIRMED_EMAIL_ADDRESS_EXPIRE_IN_DAYS = 10
## CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS -- when set to 1, the session
## system allocates the same uid=0 to all guests users regardless of where they
## come from. 0 allocate a unique uid to each guest.
CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS = 0
################################
## Part 9: BibRank parameters ##
################################
## This section contains some configuration parameters for the ranking
## system.
## CFG_BIBRANK_SHOW_READING_STATS -- do we want to show reading
## similarity stats? ('People who viewed this page also viewed')
CFG_BIBRANK_SHOW_READING_STATS = 1
## CFG_BIBRANK_SHOW_DOWNLOAD_STATS -- do we want to show the download
## similarity stats? ('People who downloaded this document also
## downloaded')
CFG_BIBRANK_SHOW_DOWNLOAD_STATS = 1
## CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS -- do we want to show download
## history graph?
CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS = 1
## CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS_CLIENT_IP_DISTRIBUTION -- do we
## want to show a graph representing the distribution of client IPs
## downloading given document?
CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS_CLIENT_IP_DISTRIBUTION = 0
## CFG_BIBRANK_SHOW_CITATION_LINKS -- do we want to show the 'Cited
## by' links? (useful only when you have citations in the metadata)
CFG_BIBRANK_SHOW_CITATION_LINKS = 1
## CFG_BIBRANK_SHOW_CITATION_STATS -- de we want to show citation
## stats? ('Cited by M recors', 'Co-cited with N records')
CFG_BIBRANK_SHOW_CITATION_STATS = 1
## CFG_BIBRANK_SHOW_CITATION_GRAPHS -- do we want to show citation
## history graph?
CFG_BIBRANK_SHOW_CITATION_GRAPHS = 1
####################################
## Part 10: WebComment parameters ##
####################################
## This section contains some configuration parameters for the
## commenting and reviewing facilities.
## CFG_WEBCOMMENT_ALLOW_COMMENTS -- do we want to allow users write
## public comments on records?
CFG_WEBCOMMENT_ALLOW_COMMENTS = 1
## CFG_WEBCOMMENT_ALLOW_REVIEWS -- do we want to allow users write
## public reviews of records?
CFG_WEBCOMMENT_ALLOW_REVIEWS = 1
## CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS -- do we want to allow short
## reviews, that is just the attribution of stars without submitting
## detailed review text?
CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS = 0
## CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN -- if users
## report a comment to be abusive, how many they have to be before the
## site admin is alerted?
CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN = 5
## CFG_WEBCOMMENT_NB_COMMENTS_IN_DETAILED_VIEW -- how many comments do
## we display in the detailed record page upon welcome?
CFG_WEBCOMMENT_NB_COMMENTS_IN_DETAILED_VIEW = 1
## CFG_WEBCOMMENT_NB_REVIEWS_IN_DETAILED_VIEW -- how many reviews do
## we display in the detailed record page upon welcome?
CFG_WEBCOMMENT_NB_REVIEWS_IN_DETAILED_VIEW = 1
## CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL -- do we notify the site
## admin after every comment?
CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL = 1
## CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS -- how many
## elapsed seconds do we consider enough when checking for possible
## multiple comment submissions by a user?
CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS = 20
## CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_REVIEWS_IN_SECONDS -- how many
## elapsed seconds do we consider enough when checking for possible
## multiple review submissions by a user?
CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_REVIEWS_IN_SECONDS = 20
## CFG_WEBCOMMENT_USE_RICH_EDITOR -- enable the WYSIWYG
## Javascript-based editor when user edits comments?
CFG_WEBCOMMENT_USE_RICH_TEXT_EDITOR = False
## CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL -- the email address from which the
## alert emails will appear to be sent:
CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL = cds.support@cern.ch
## CFG_WEBCOMMENT_DEFAULT_MODERATOR -- if no rules are
## specified to indicate who is the comment moderator of
## a collection, this person will be used as default
CFG_WEBCOMMENT_DEFAULT_MODERATOR = cds.support@cern.ch
## CFG_WEBCOMMENT_USE_JSMATH_IN_COMMENTS -- do we want to allow the use
## of jsmath plugin to render latex input in comments?
CFG_WEBCOMMENT_USE_JSMATH_IN_COMMENTS = 1
## CFG_WEBCOMMENT_AUTHOR_DELETE_COMMENT_OPTION -- allow comment author to
## delete its own comment?
CFG_WEBCOMMENT_AUTHOR_DELETE_COMMENT_OPTION = 1
##################################
## Part 11: BibSched parameters ##
##################################
## This section contains some configuration parameters for the
## bibliographic task scheduler.
## CFG_BIBSCHED_REFRESHTIME -- how often do we want to refresh
## bibsched monitor? (in seconds)
CFG_BIBSCHED_REFRESHTIME = 5
## CFG_BIBSCHED_LOG_PAGER -- what pager to use to view bibsched task
## logs?
CFG_BIBSCHED_LOG_PAGER = /bin/more
## CFG_BIBSCHED_GC_TASKS_OLDER_THAN -- after how many days to perform the
## gargbage collector of BibSched queue (i.e. removing/moving task to archive).
CFG_BIBSCHED_GC_TASKS_OLDER_THAN = 30
## CFG_BIBSCHED_GC_TASKS_TO_REMOVE -- list of BibTask that can be safely
## removed from the BibSched queue once they are DONE.
CFG_BIBSCHED_GC_TASKS_TO_REMOVE = bibindex,bibreformat,webcoll,bibrank,inveniogc
## CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE -- list of BibTasks that should be safely
## archived out of the BibSched queue once they are DONE.
CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE = bibupload,oaiarchive
## CFG_BIBSCHED_MAX_NUMBER_CONCURRENT_TASKS -- maximum number of BibTasks
## that can run concurrently.
## NOTE: concurrent tasks are still considered as an experimental
## feature. Please keep this value set to 1 on production environments.
CFG_BIBSCHED_MAX_NUMBER_CONCURRENT_TASKS = 1
## CFG_BIBSCHED_PROCESS_USER -- bibsched and bibtask processes must
## usually run under the same identity as the Apache web server
## process in order to share proper file read/write privileges. If
## you want to force some other bibsched/bibtask user, e.g. because
## you are using a local `invenio' user that belongs to your
## `www-data' Apache user group and so shares writing rights with your
## Apache web server process in this way, then please set its username
## identity here. Otherwise we shall check whether your
## bibsched/bibtask processes are run under the same identity as your
## Apache web server process (in which case you can leave the default
## empty value here).
CFG_BIBSCHED_PROCESS_USER =
###################################
## Part 12: WebBasket parameters ##
###################################
## CFG_WEBBASKET_MAX_NUMBER_OF_DISPLAYED_BASKETS -- a safety limit for
## a maximum number of displayed baskets
CFG_WEBBASKET_MAX_NUMBER_OF_DISPLAYED_BASKETS = 20
## CFG_WEBBASKET_USE_RICH_TEXT_EDITOR -- enable the WYSIWYG
## Javascript-based editor when user edits comments in WebBasket?
CFG_WEBBASKET_USE_RICH_TEXT_EDITOR = False
##################################
## Part 13: WebAlert parameters ##
##################################
## This section contains some configuration parameters for the
## automatic email notification alert system.
## CFG_WEBALERT_ALERT_ENGINE_EMAIL -- the email address from which the
## alert emails will appear to be sent:
CFG_WEBALERT_ALERT_ENGINE_EMAIL = cds.support@cern.ch
## CFG_WEBALERT_MAX_NUM_OF_RECORDS_IN_ALERT_EMAIL -- how many records
## at most do we send in an outgoing alert email?
CFG_WEBALERT_MAX_NUM_OF_RECORDS_IN_ALERT_EMAIL = 20
## CFG_WEBALERT_MAX_NUM_OF_CHARS_PER_LINE_IN_ALERT_EMAIL -- number of
## chars per line in an outgoing alert email?
CFG_WEBALERT_MAX_NUM_OF_CHARS_PER_LINE_IN_ALERT_EMAIL = 72
## CFG_WEBALERT_SEND_EMAIL_NUMBER_OF_TRIES -- when sending alert
## emails fails, how many times we retry?
CFG_WEBALERT_SEND_EMAIL_NUMBER_OF_TRIES = 3
## CFG_WEBALERT_SEND_EMAIL_SLEEPTIME_BETWEEN_TRIES -- when sending
## alert emails fails, what is the sleeptime between tries? (in
## seconds)
CFG_WEBALERT_SEND_EMAIL_SLEEPTIME_BETWEEN_TRIES = 300
####################################
## Part 14: WebMessage parameters ##
####################################
## CFG_WEBMESSAGE_MAX_SIZE_OF_MESSAGE -- how large web messages do we
## allow?
CFG_WEBMESSAGE_MAX_SIZE_OF_MESSAGE = 20000
## CFG_WEBMESSAGE_MAX_NB_OF_MESSAGES -- how many messages for a
## regular user do we allow in its inbox?
CFG_WEBMESSAGE_MAX_NB_OF_MESSAGES = 30
## CFG_WEBMESSAGE_DAYS_BEFORE_DELETE_ORPHANS -- how many days before
## we delete orphaned messages?
CFG_WEBMESSAGE_DAYS_BEFORE_DELETE_ORPHANS = 60
##################################
## Part 15: MiscUtil parameters ##
##################################
-## CFG_MISCUTIL_SQL_MAX_CACHED_QUERIES -- maximum number of cached SQL
-## queries possible. After reaching this number the cache is pruned
-## by deleting half of the older queries.
-CFG_MISCUTIL_SQL_MAX_CACHED_QUERIES = 10000
-
## CFG_MISCUTIL_SQL_USE_SQLALCHEMY -- whether to use SQLAlchemy.pool
## in the DB engine of CDS Invenio. It is okay to enable this flag
## even if you have not installed SQLAlchemy. Note that Invenio will
## loose some perfomance if this option is enabled.
CFG_MISCUTIL_SQL_USE_SQLALCHEMY = False
## CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT -- how many queries can we run
## inside run_sql_many() in one SQL statement? The limit value
## depends on MySQL's max_allowed_packet configuration.
CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT = 10000
## CFG_MISCUTIL_SMTP_HOST -- which server to use as outgoing mail server to
## send outgoing emails generated by the system, for example concerning
## submissions or email notification alerts.
CFG_MISCUTIL_SMTP_HOST = localhost
## CFG_MISCUTIL_SMTP_PORT -- which port to use on the outgoing mail server
## defined in the previous step.
CFG_MISCUTIL_SMTP_PORT = 25
## CFG_MISCUTILS_DEFAULT_PROCESS_TIMEOUT -- the default number of seconds after
## which a process launched trough shellutils.run_process_with_timeout will
## be killed. This is useful to catch runaway processes.
CFG_MISCUTIL_DEFAULT_PROCESS_TIMEOUT = 300
## CFG_PLOTEXTRACTOR_ARXIV_BASE -- for acquiring source tarballs for plot
## extraction, where should we look? If nothing is set, we'll just go
## to arXiv, but this can be a filesystem location, too
CFG_PLOTEXTRACTOR_ARXIV_BASE =
#################################
## Part 16: BibEdit parameters ##
#################################
## CFG_BIBEDIT_TIMEOUT -- when a user edits a record, this record is
## locked to prevent other users to edit it at the same time.
## How many seconds of inactivity before the locked record again will be free
## for other people to edit?
CFG_BIBEDIT_TIMEOUT = 3600
## CFG_BIBEDIT_LOCKLEVEL -- when a user tries to edit a record which there
## is a pending bibupload task for in the queue, this shouldn't be permitted.
## The lock level determines how thouroughly the queue should be investigated
## to determine if this is the case.
## Level 0 - always permits editing, doesn't look at the queue
## (unsafe, use only if you know what you are doing)
## Level 1 - permits editing if there are no queued bibedit tasks for this record
## (safe with respect to bibedit, but not for other bibupload maintenance jobs)
## Level 2 - permits editing if there are no queued bibupload tasks of any sort
## (safe, but may lock more than necessary if many cataloguers around)
## Level 3 - permits editing if no queued bibupload task concerns given record
## (safe, most precise locking, but slow,
## checks for 001/EXTERNAL_SYSNO_TAG/EXTERNAL_OAIID_TAG)
## The recommended level is 3 (default) or 2 (if you use maintenance jobs often).
CFG_BIBEDIT_LOCKLEVEL = 3
## CFG_BIBEDIT_PROTECTED_FIELDS -- a comma-separated list of fields that BibEdit
## will not allow to be added, edited or deleted. Wildcards are not supported,
## but conceptually a wildcard is added at the end of every field specification.
## Examples:
## 500A - protect all MARC fields with tag 500 and first indicator A
## 5 - protect all MARC fields in the 500-series.
## 909C_a - protect subfield a in tag 909 with first indicator C and empty
## second indicator
## Note that 001 is protected by default, but if protection of other
## identifiers or automated fields is a requirement, they should be added to
## this list.
CFG_BIBEDIT_PROTECTED_FIELDS =
## CFG_BIBEDITMULTI_LIMIT_INSTANT_PROCESSING -- maximum number of records
## that can be modified instantly using the multi-record editor. Above
## this limit, modifications will only be executed in limited hours.
CFG_BIBEDITMULTI_LIMIT_INSTANT_PROCESSING = 2000
## CFG_BIBEDITMULTI_LIMIT_DELAYED_PROCESSING -- maximum number of records
## that can be send for modification without having a superadmin role.
## If the number of records is between CFG_BIBEDITMULTI_LIMIT_INSTANT_PROCESSING
## and this number, the modifications will take place only in limited hours.
CFG_BIBEDITMULTI_LIMIT_DELAYED_PROCESSING = 20000
## CFG_BIBEDITMULTI_LIMIT_DELAYED_PROCESSING_TIME -- Allowed time to
## execute modifications on records, when the number exceeds
## CFG_BIBEDITMULTI_LIMIT_INSTANT_PROCESSING.
CFG_BIBEDITMULTI_LIMIT_DELAYED_PROCESSING_TIME = 22:00-05:00
###################################
## Part 17: BibUpload parameters ##
###################################
## CFG_BIBUPLOAD_REFERENCE_TAG -- where do we store references?
CFG_BIBUPLOAD_REFERENCE_TAG = 999
## CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG -- where do we store external
## system numbers? Useful for matching when our records come from an
## external digital library system.
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG = 970__a
## CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG -- where do we store OAI ID tags
## of harvested records? Useful for matching when we harvest stuff
## via OAI that we do not want to reexport via Invenio OAI; so records
## may have only the source OAI ID stored in this tag (kind of like
## external system number too).
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG = 035__a
## CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG -- where do we store OAI SRC
## tags of harvested records? Useful for matching when we harvest stuff
## via OAI that we do not want to reexport via Invenio OAI; so records
## may have only the source OAI SRC stored in this tag (kind of like
## external system number too). Note that the field should be the same of
## CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG.
CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG = 035__9
## CFG_BIBUPLOAD_STRONG_TAGS -- a comma-separated list of tags that
## are strong enough to resist the replace mode. Useful for tags that
## might be created from an external non-metadata-like source,
## e.g. the information about the number of copies left.
CFG_BIBUPLOAD_STRONG_TAGS = 964
## CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS -- a comma-separated list
## of tags that contain provenance information that should be checked
## in the bibupload correct mode via matching provenance codes. (Only
## field instances of the same provenance information would be acted
## upon.) Please specify the whole tag info up to subfield codes.
CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS = 6531_9
## CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS -- a comma-separated list of system
## paths from which it is allowed to take fulltextes that will be uploaded via
## FFT (CFG_TMPDIR is included by default).
CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS = /tmp,/home
## CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE -- do we want to serialize
## internal representation of records (Pythonic record structure) into
## the database? This can improve internal processing speed of some
## operations at the price of somewhat bigger disk space usage.
## If you change this value after some records have already been added
## to your installation, you may want to run:
## $ /opt/cds-invenio/bin/inveniocfg --reset-recstruct-cache
## in order to either erase the cache thus freeing database space,
## or to fill the cache for all records that have not been cached yet.
CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE = 1
## CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY -- a comma-separated list
## indicating which fields match the file names of the documents to be
## uploaded.
## The matching will be done in the same order as the list provided.
CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY = reportnumber,recid
## CFG_BATCHUPLOADER_DAEMON_DIR -- Directory where the batchuploader daemon
## will look for the subfolders metadata and document by default.
## If path is relative, CFG_PREFIX will be joined as a prefix
CFG_BATCHUPLOADER_DAEMON_DIR = var/batchupload
## CFG_BATCHUPLOADER_WEB_ROBOT_AGENT -- Comma-separated list to specify the
## agents permitted when calling batch uploader web interface
## cdsweb.cern.ch/batchuploader/robotupload
## if using a curl, eg: curl xxx -A invenio_webupload
CFG_BATCHUPLOADER_WEB_ROBOT_AGENT = invenio_webupload
## CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS -- Access list specifying for each
## IP address, which collections are allowed using batch uploader robot
## interface.
CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS = {
'10.0.0.1': ['BOOK', 'REPORT'], # Example 1
'10.0.0.2': ['POETRY', 'PREPRINT'], # Example 2
}
####################################
## Part 18: BibCatalog parameters ##
####################################
## EXPERIMENTAL: Please do not use.
CFG_BIBCATALOG_SYSTEM =
CFG_BIBCATALOG_SYSTEM_RT_CLI = /usr/bin/rt
CFG_BIBCATALOG_SYSTEM_RT_URL = http://localhost/rt3
CFG_BIBCATALOG_QUEUES = General
####################################
## Part 19: BibFormat parameters ##
####################################
## CFG_BIBFORMAT_HIDDEN_TAGS -- comma-separated list of MARC tags that
## are not shown to users not having cataloging authorizations.
CFG_BIBFORMAT_HIDDEN_TAGS = 595
##########################
## THAT's ALL, FOLKS! ##
##########################
diff --git a/modules/miscutil/doc/hacking/miscutil-dbquery.webdoc b/modules/miscutil/doc/hacking/miscutil-dbquery.webdoc
index 33501d170..833c9f539 100644
--- a/modules/miscutil/doc/hacking/miscutil-dbquery.webdoc
+++ b/modules/miscutil/doc/hacking/miscutil-dbquery.webdoc
@@ -1,175 +1,133 @@
## -*- mode: html; coding: utf-8; -*-
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
dbquery module handles automatically connection (and reconnection)
to the database and provides the run_sql() function to
-perform SQL queries and run_sql_cached() for rarely
-changing SELECT queries. It also exports DB exceptions for the client
+perform SQL queries. It also exports DB exceptions for the client
code to use (see below).
run_sql() API
run_sql() signature:
def run_sql(sql, param=None, n=0, with_desc=0):
"""Run SQL on the server with PARAM and return result.
@param param: tuple of string params to insert in the query
(see notes below)
@param n: number of tuples in result (0 for unbounded)
@param with_desc: if true, will return a
DB API 7-tuple describing columns in query
@return: if SELECT, SHOW, DESCRIBE statements: tuples of data, followed
by description if parameter
provided
if INSERT: last row id.
else: SQL result as provided by database
When the site is closed for maintenance (as governed by the
config variable CFG_ACCESS_CONTROL_LEVEL_SITE), do not attempt
to run any SQL queries but return empty list immediately.
Useful to be able to have the website up while MySQL database
is down for maintenance, hot copies, table repairs, etc.
In case of problems, exceptions are returned according to the
Python DB API 2.0. The client code can import them from this
file and catch them.
"""
run_sql() normally escapes its parameters if you
pass them in a tuple. Usually the params must use the string format (%s):
from invenio.dbquery import run_sql
[...]
res = run_sql("SELECT id FROM collection WHERE name=%s", (c,))
if res:
colID = res[0][0]
If you want to escape the parameters yourself in the client code, you
could in principle import and make use of the
function real_escape_string():
from invenio.dbquery import run_sql, real_escape_string
[...]
res = run_sql("SELECT id FROM collection WHERE name='%s'" % real_escape_string(c), None)
if res:
colID = res[0][0]
but it is better to use the former automatic technique.
The run_sql() raises Python DB API 2.0 exceptions that
the client code should catch and handle. An example:
from invenio.dbquery import run_sql, OperationalError
[...]
query = "select citation_data from rnkCITATIONDATA"
try:
compressed_citation_dic = run_sql(query)
except OperationalError:
compressed_citation_dic = []
For the list of all exceptions and the conditions when they are
raised, see PEP 249.
Note for handling date types
There is an incompatibility in handling date types between MySQLdb 0.9 and MySQLdb 1.2 (while using Python 2.2 or 2.3). If a date field is in the received tuple, its format will be:
- string with MySQLdb 0.9
- datetime with MySQLdb 1.2
As Python 2.2 doesn't provide datetime class, handling of this
problem should be done for backwards compatibility reasons. The
solution is to force MySQL to convert date to a textual format:
SELECT DATE_FORMAT(date_field,'%%Y-%%m-%%d %%H:%%i:%%s') FROM table
This conversion will return a datetext format as described in dateutils library(YEAR-MONTH-DAY HOUR:MINUTE:SECOND).
-run_sql_cached() API
-
- If you execute a certain SELECT query often, you can
-use run_sql_cached() that will cache its result in memory
-and return it faster next time. The function signature and usage is
-similar to the one known from run_sql():
-
-
-
-def run_sql_cached(sql, param=None, n=0, with_desc=0, affected_tables=['bibrec']):
- """
- Run the SQL query and cache the SQL command for later reuse.
-
- @param param: tuple of string params to insert in the query
- (see notes below)
-
- @param n: number of tuples in result (0 for unbounded)
-
- @param with_desc: if true, will return a
- DB API 7-tuple describing columns in query
-
- @param affected_tables is a list of tablenames of affected tables,
- used to decide whether we should update the cache or whether we
- can return cached result, depending on the last modification time
- for corresponding tables. If empty, and if the cached result is
- present in the cache, always return the cached result without
- recomputing it. (This is useful to speed up queries that operate
- on objects that virtually never change, e.g. list of defined
- logical fields, that remain usually constant in between Apache
- restarts. Note that this would be a dangerous default for any
- query.)
-
- @return the result as provided by run_sql()
-
- Note that it is pointless and even wrong to use this function with
- SQL commands different from SELECT.
- """
-
-
-
-
Logging SQL Queries
If you want to investigate some DB related problems, note that you
can uncomment some lines in dbquery.py to obtain detailed
log of every SQL query and its parameters. Look for
string log_sql_query to know more.
diff --git a/modules/miscutil/lib/dbquery.py b/modules/miscutil/lib/dbquery.py
index 16838d12a..a7c44b7b2 100644
--- a/modules/miscutil/lib/dbquery.py
+++ b/modules/miscutil/lib/dbquery.py
@@ -1,451 +1,359 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
CDS Invenio utilities to run SQL queries.
The main API functions are:
- run_sql()
- - run_sql_cached()
- run_sql_many()
but see the others as well.
"""
__revision__ = "$Id$"
# dbquery clients can import these from here:
# pylint: disable=W0611
from MySQLdb import Warning, Error, InterfaceError, DataError, \
DatabaseError, OperationalError, IntegrityError, \
InternalError, NotSupportedError, \
ProgrammingError
import string
import time
import marshal
import re
from zlib import compress, decompress
from thread import get_ident
from invenio.config import CFG_ACCESS_CONTROL_LEVEL_SITE, \
- CFG_MISCUTIL_SQL_MAX_CACHED_QUERIES, CFG_MISCUTIL_SQL_USE_SQLALCHEMY, \
+ CFG_MISCUTIL_SQL_USE_SQLALCHEMY, \
CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT
if CFG_MISCUTIL_SQL_USE_SQLALCHEMY:
try:
import sqlalchemy.pool as pool
import MySQLdb as mysqldb
mysqldb = pool.manage(mysqldb, use_threadlocal=True)
connect = mysqldb.connect
except ImportError:
CFG_MISCUTIL_SQL_USE_SQLALCHEMY = False
from MySQLdb import connect
else:
from MySQLdb import connect
## DB config variables. These variables are to be set in
## invenio-local.conf by admins and then replaced in situ in this file
## by calling "inveniocfg --update-dbexec".
## Note that they are defined here and not in config.py in order to
## prevent them from being exported accidentally elsewhere, as no-one
## should know DB credentials but this file.
## FIXME: this is more of a blast-from-the-past that should be fixed
## both here and in inveniocfg when the time permits.
CFG_DATABASE_HOST = 'localhost'
CFG_DATABASE_PORT = '3306'
CFG_DATABASE_NAME = 'cdsinvenio'
CFG_DATABASE_USER = 'cdsinvenio'
CFG_DATABASE_PASS = 'my123p$ss'
_DB_CONN = {}
-try:
- _db_cache
-except NameError:
- _db_cache = {}
-
def _db_login(relogin = 0):
"""Login to the database."""
## Note: we are using "use_unicode=False", because we want to
## receive strings from MySQL as Python UTF-8 binary string
## objects, not as Python Unicode string objects, as of yet.
## Note: "charset='utf8'" is needed for recent MySQLdb versions
## (such as 1.2.1_p2 and above). For older MySQLdb versions such
## as 1.2.0, an explicit "init_command='SET NAMES utf8'" parameter
## would constitute an equivalent. But we are not bothering with
## older MySQLdb versions here, since we are recommending to
## upgrade to more recent versions anyway.
if CFG_MISCUTIL_SQL_USE_SQLALCHEMY:
return connect(host=CFG_DATABASE_HOST, port=int(CFG_DATABASE_PORT),
db=CFG_DATABASE_NAME, user=CFG_DATABASE_USER,
passwd=CFG_DATABASE_PASS,
use_unicode=False, charset='utf8')
else:
thread_ident = get_ident()
if relogin:
_DB_CONN[thread_ident] = connect(host=CFG_DATABASE_HOST,
port=int(CFG_DATABASE_PORT),
db=CFG_DATABASE_NAME,
user=CFG_DATABASE_USER,
passwd=CFG_DATABASE_PASS,
use_unicode=False, charset='utf8')
return _DB_CONN[thread_ident]
else:
if _DB_CONN.has_key(thread_ident):
return _DB_CONN[thread_ident]
else:
_DB_CONN[thread_ident] = connect(host=CFG_DATABASE_HOST,
port=int(CFG_DATABASE_PORT),
db=CFG_DATABASE_NAME,
user=CFG_DATABASE_USER,
passwd=CFG_DATABASE_PASS,
use_unicode=False, charset='utf8')
return _DB_CONN[thread_ident]
def _db_logout():
"""Close a connection."""
try:
del _DB_CONN[get_ident()]
except KeyError:
pass
-def run_sql_cached(sql, param=None, n=0, with_desc=0, affected_tables=['bibrec']):
- """
- Run the SQL query and cache the SQL command for later reuse.
-
- @param param: tuple of string params to insert in the query
- (see notes below)
-
- @param n: number of tuples in result (0 for unbounded)
-
- @param with_desc: if true, will return a
- DB API 7-tuple describing columns in query
-
- @param affected_tables: is a list of tablenames of affected tables,
- used to decide whether we should update the cache or whether we
- can return cached result, depending on the last modification time
- for corresponding tables. If empty, and if the cached result is
- present in the cache, always return the cached result without
- recomputing it. (This is useful to speed up queries that operate
- on objects that virtually never change, e.g. list of defined
- logical fields, that remain usually constant in between Apache
- restarts. Note that this would be a dangerous default for any
- query.)
-
- @return: the result as provided by run_sql()
-
- Note that it is pointless and even wrong to use this function with
- SQL commands different from SELECT.
- """
-
- ## FIXME: The code below, checking table update times, was found
- ## to be slow in user storm situations. So let us rather run SQL
- ## statement live; it seems faster to let MySQL use its own cache
- ## than to constantly verify table update time. Later, a proper
- ## time-driven data cacher might be introduced here. Or, better
- ## yet, we can plug dedicated data cachers to every place that
- ## called run_sql_cached.
- return run_sql(sql, param, n, with_desc)
-
- global _db_cache
-
- if CFG_ACCESS_CONTROL_LEVEL_SITE == 3:
- # do not connect to the database as the site is closed for maintenance:
- return []
-
- key = repr((sql, param, n, with_desc))
-
- # Garbage collecting needed?
- if len(_db_cache) >= CFG_MISCUTIL_SQL_MAX_CACHED_QUERIES:
- _db_cache = {}
-
- # Query already in the cache?
- if not _db_cache.has_key(key) or \
- (affected_tables and _db_cache[key][1] <= max([get_table_update_time(table) for table in affected_tables])):
- # Let's update the cache
- result = run_sql(sql, param, n, with_desc)
- _db_cache[key] = (result, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
- ### log_sql_query_cached(key, result, False) ### UNCOMMENT ONLY IF you REALLY want to log all queries
-
- else:
- result = _db_cache[key][0]
- ### log_sql_query_cached(key, result, True) ### UNCOMMENT ONLY IF you REALLY want to log all queries
-
- return result
-
def run_sql(sql, param=None, n=0, with_desc=0):
"""Run SQL on the server with PARAM and return result.
@param param: tuple of string params to insert in the query (see
notes below)
@param n: number of tuples in result (0 for unbounded)
@param with_desc: if True, will return a DB API 7-tuple describing
columns in query.
@return: If SELECT, SHOW, DESCRIBE statements, return tuples of
data, followed by description if parameter with_desc is
provided. If INSERT, return last row id. Otherwise return
SQL result as provided by database.
@note: When the site is closed for maintenance (as governed by the
config variable CFG_ACCESS_CONTROL_LEVEL_SITE), do not attempt
to run any SQL queries but return empty list immediately.
Useful to be able to have the website up while MySQL database
is down for maintenance, hot copies, table repairs, etc.
@note: In case of problems, exceptions are returned according to
the Python DB API 2.0. The client code can import them from
this file and catch them.
"""
if CFG_ACCESS_CONTROL_LEVEL_SITE == 3:
# do not connect to the database as the site is closed for maintenance:
return []
### log_sql_query(sql, param) ### UNCOMMENT ONLY IF you REALLY want to log all queries
if param:
param = tuple(param)
try:
db = _db_login()
cur = db.cursor()
rc = cur.execute(sql, param)
except OperationalError: # unexpected disconnect, bad malloc error, etc
# FIXME: now reconnect is always forced, we may perhaps want to ping() first?
try:
db = _db_login(relogin=1)
cur = db.cursor()
rc = cur.execute(sql, param)
except OperationalError: # again an unexpected disconnect, bad malloc error, etc
raise
if string.upper(string.split(sql)[0]) in ("SELECT", "SHOW", "DESC", "DESCRIBE"):
if n:
recset = cur.fetchmany(n)
else:
recset = cur.fetchall()
if with_desc:
return recset, cur.description
else:
return recset
else:
if string.upper(string.split(sql)[0]) == "INSERT":
rc = cur.lastrowid
return rc
def run_sql_many(query, params, limit=CFG_MISCUTIL_SQL_RUN_SQL_MANY_LIMIT):
"""Run SQL on the server with PARAM.
This method does executemany and is therefore more efficient than execute
but it has sense only with queries that affect state of a database
(INSERT, UPDATE). That is why the results just count number of affected rows
@param params: tuple of tuple of string params to insert in the query
@param limit: query will be executed in parts when number of
parameters is greater than limit (each iteration runs at most
`limit' parameters)
@return: SQL result as provided by database
"""
i = 0
r = None
while i < len(params):
## make partial query safely (mimicking procedure from run_sql())
try:
db = _db_login()
cur = db.cursor()
rc = cur.executemany(query, params[i:i+limit])
except OperationalError:
try:
db = _db_login(relogin=1)
cur = db.cursor()
rc = cur.executemany(query, params[i:i+limit])
except OperationalError:
raise
## collect its result:
if r is None:
r = rc
else:
r += rc
i += limit
return r
def blob_to_string(ablob):
"""Return string representation of ABLOB. Useful to treat MySQL
BLOBs in the same way for both recent and old MySQLdb versions.
"""
if ablob:
if type(ablob) is str:
# BLOB is already a string in MySQLdb 0.9.2
return ablob
else:
# BLOB is array.array in MySQLdb 1.0.0 and later
return ablob.tostring()
else:
return ablob
-def log_sql_query_cached(key, result, hit_p):
- """Log SQL query cached into prefix/var/log/dbquery.log log file. In order
- to enable logging of all SQL queries, please uncomment two lines
- in run_sql_cached() above. Useful for fine-level debugging only!
- """
- from invenio.config import CFG_LOGDIR
- from invenio.dateutils import convert_datestruct_to_datetext
- from invenio.textutils import indent_text
- log_path = CFG_LOGDIR + '/dbquery.log'
- date_of_log = convert_datestruct_to_datetext(time.localtime())
- message = date_of_log + '-->\n'
- message += indent_text('Key:\n' + indent_text(str(key), 2, wrap=True), 2)
- message += indent_text('Result:\n' + indent_text(str(result) + (hit_p and ' HIT' or ' MISS'), 2, wrap=True), 2)
- message += 'Cached queries: %i\n\n' % len(_db_cache)
- try:
- log_file = open(log_path, 'a+')
- log_file.writelines(message)
- log_file.close()
- except:
- pass
-
-
def log_sql_query(sql, param=None):
"""Log SQL query into prefix/var/log/dbquery.log log file. In order
to enable logging of all SQL queries, please uncomment one line
in run_sql() above. Useful for fine-level debugging only!
"""
from invenio.config import CFG_LOGDIR
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.textutils import indent_text
log_path = CFG_LOGDIR + '/dbquery.log'
date_of_log = convert_datestruct_to_datetext(time.localtime())
message = date_of_log + '-->\n'
message += indent_text('Query:\n' + indent_text(str(sql), 2, wrap=True), 2)
message += indent_text('Params:\n' + indent_text(str(param), 2, wrap=True), 2)
message += '-----------------------------\n\n'
try:
log_file = open(log_path, 'a+')
log_file.writelines(message)
log_file.close()
except:
pass
def get_table_update_time(tablename):
"""Return update time of TABLENAME. TABLENAME can contain
wildcard `%' in which case we return the maximum update time
value.
"""
# Note: in order to work with all of MySQL 4.0, 4.1, 5.0, this
# function uses SHOW TABLE STATUS technique with a dirty column
# position lookup to return the correct value. (Making use of
# Index_Length column that is either of type long (when there are
# some indexes defined) or of type None (when there are no indexes
# defined, e.g. table is empty). When we shall use solely
# MySQL-5.0, we can employ a much cleaner technique of using
# SELECT UPDATE_TIME FROM INFORMATION_SCHEMA.TABLES WHERE
# table_name='collection'.
res = run_sql("SHOW TABLE STATUS LIKE %s", (tablename, ))
update_times = [] # store all update times
for row in res:
if type(row[10]) is long or \
row[10] is None:
# MySQL-4.1 and 5.0 have creation_time in 11th position,
# so return next column:
update_times.append(str(row[12]))
else:
# MySQL-4.0 has creation_time in 10th position, which is
# of type datetime.datetime or str (depending on the
# version of MySQLdb), so return next column:
update_times.append(str(row[11]))
return max(update_times)
def get_table_status_info(tablename):
"""Return table status information on TABLENAME. Returned is a
dict with keys like Name, Rows, Data_length, Max_data_length,
etc. If TABLENAME does not exist, return empty dict.
"""
# Note: again a hack so that it works on all MySQL 4.0, 4.1, 5.0
res = run_sql("SHOW TABLE STATUS LIKE %s", (tablename, ))
table_status_info = {} # store all update times
for row in res:
if type(row[10]) is long or \
row[10] is None:
# MySQL-4.1 and 5.0 have creation time in 11th position:
table_status_info['Name'] = row[0]
table_status_info['Rows'] = row[4]
table_status_info['Data_length'] = row[6]
table_status_info['Max_data_length'] = row[8]
table_status_info['Create_time'] = row[11]
table_status_info['Update_time'] = row[12]
else:
# MySQL-4.0 has creation_time in 10th position, which is
# of type datetime.datetime or str (depending on the
# version of MySQLdb):
table_status_info['Name'] = row[0]
table_status_info['Rows'] = row[3]
table_status_info['Data_length'] = row[5]
table_status_info['Max_data_length'] = row[7]
table_status_info['Create_time'] = row[10]
table_status_info['Update_time'] = row[11]
return table_status_info
def serialize_via_marshal(obj):
"""Serialize Python object via marshal into a compressed string."""
return compress(marshal.dumps(obj))
def deserialize_via_marshal(astring):
"""Decompress and deserialize string into a Python object via marshal."""
return marshal.loads(decompress(astring))
try:
import psyco
psyco.bind(serialize_via_marshal)
psyco.bind(deserialize_via_marshal)
except StandardError, e:
pass
def wash_table_column_name(colname):
"""
Evaluate table-column name to see if it is clean.
This function accepts only names containing [a-zA-Z0-9_].
@param colname: The string to be checked
@type colname: str
@return: colname if test passed
@rtype: str
@raise Exception: Raises an exception if colname is invalid.
"""
if re.search('[^\w]', colname):
raise Exception('The table column %s is not valid.' % repr(colname))
return colname
def real_escape_string(unescaped_string):
"""
Escapes special characters in the unescaped string for use in a DB query.
@param unescaped_string: The string to be escaped
@type unescaped_string: str
@return: Returns the escaped string
@rtype: str
"""
connection_object = _db_login()
escaped_string = connection_object.escape_string(unescaped_string)
return escaped_string
diff --git a/modules/webaccess/lib/access_control_admin.py b/modules/webaccess/lib/access_control_admin.py
index 964e51ad6..20af8648f 100644
--- a/modules/webaccess/lib/access_control_admin.py
+++ b/modules/webaccess/lib/access_control_admin.py
@@ -1,1820 +1,1820 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""CDS Invenio Access Control Admin."""
__revision__ = "$Id$"
# check this: def acc_add_user_role(id_user, id_role=0, name_role=0):
## import interesting modules:
import sys
if sys.hexversion < 0x2040000:
# pylint: disable=W0622
from sets import Set as set
# pylint: enable=W0622
from invenio.messages import gettext_set_language
from invenio.config import CFG_SITE_ADMIN_EMAIL, CFG_SITE_LANG
from invenio.access_control_config import CFG_ACC_EMPTY_ROLE_DEFINITION_SER, \
CFG_ACC_EMPTY_ROLE_DEFINITION_SRC, DELEGATEADDUSERROLE, SUPERADMINROLE, \
DEF_USERS, DEF_ROLES, DEF_AUTHS, DEF_ACTIONS, CFG_ACC_ACTIVITIES_URLS
-from invenio.dbquery import run_sql, ProgrammingError, run_sql_cached
+from invenio.dbquery import run_sql, ProgrammingError
from invenio.access_control_firerole import compile_role_definition, \
acc_firerole_check_user, serialize, deserialize, load_role_definition
from invenio.intbitset import intbitset
id_tmp = run_sql('SELECT id FROM accROLE WHERE name=%s', (SUPERADMINROLE, ))
if id_tmp:
CFG_SUPERADMINROLE_ID = int(id_tmp[0][0])
else:
CFG_SUPERADMINROLE_ID = 0
# ACTIONS
def acc_add_action(name_action='', description='', optional='no',
*allowedkeywords):
"""function to create new entry in accACTION for an action
name_action - name of the new action, must be unique
keyvalstr - string with allowed keywords
allowedkeywords - a list of allowedkeywords
keyvalstr and allowedkeywordsdict can not be in use simultanously
success -> return id_action, name_action, description and allowedkeywords
failure -> return 0 """
keystr = ''
# action with this name all ready exists, return 0
if run_sql("""SELECT name FROM accACTION WHERE name = %s""",
(name_action, )):
return 0
# create keyword string
for value in allowedkeywords:
if keystr:
keystr += ','
keystr += value
if not allowedkeywords:
optional = 'no'
# insert the new entry
try:
res = run_sql("""INSERT INTO accACTION (name, description,
allowedkeywords, optional) VALUES (%s, %s, %s, %s)""",
(name_action, description, keystr, optional))
except ProgrammingError:
return 0
if res:
return res, name_action, description, keystr, optional
return 0
def acc_delete_action(id_action=0, name_action=0):
"""delete action in accACTION according to id, or secondly name.
entries in accROLE_accACTION_accARGUMENT will also be removed.
id_action - id of action to be deleted, prefered variable
name_action - this is used if id_action is not given
if the name or id is wrong, the function does nothing
"""
id_action = id_action or acc_get_action_id(name_action=name_action)
if not id_action:
return 0
# delete the action
if run_sql("""DELETE FROM accACTION WHERE id=%s""", (id_action, )):
# delete all entries related
return 1 + run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT WHERE
id_accACTION=%s""", (id_action, ))
else:
return 0
def acc_verify_action(name_action='', description='', allowedkeywords='',
dummy=''):
"""check if all the values of a given action are the same as
those in accACTION in the database. self explanatory parameters.
return id if identical, 0 if not. """
id_action = acc_get_action_id(name_action=name_action)
if not id_action:
return 0
res_desc = acc_get_action_description(id_action=id_action)
res_keys = acc_get_action_keywords_string(id_action=id_action)
bool_desc = res_desc == description and 1 or 0
bool_keys = res_keys == allowedkeywords and 1 or 0
bool_opti = acc_get_action_is_optional(id_action=id_action)
return bool_desc and bool_keys and bool_opti and id_action or 0
def acc_update_action(id_action=0, name_action='', verbose=0, **update):
"""try to change the values of given action details.
if there is no change nothing is done.
some changes require to update other parts of the database.
id_action - id of the action to change
name_action - if no id_action is given try to find it using this name
**update - dictionary containg keywords: description,
allowedkeywords and/or
optional
other keywords are ignored """
id_action = id_action or acc_get_action_id(name_action=name_action)
if not id_action:
return 0
try:
if update.has_key('description'):
# change the description, no other effects
if verbose:
print 'desc'
run_sql("""UPDATE accACTION SET description = %s WHERE id = %s""",
(update['description'], id_action))
if update.has_key('allowedkeywords'):
# change allowedkeywords
if verbose:
print 'keys'
# check if changing allowedkeywords or not
if run_sql("""SELECT id FROM accACTION
WHERE id = %s AND allowedkeywords != %s """,
(id_action, update['allowedkeywords'])):
# change allowedkeywords
if verbose:
print ' changing'
run_sql("""UPDATE accACTION SET allowedkeywords = %s
WHERE id = %s""", (update['allowedkeywords'], id_action))
# delete entries, but keep optional authorizations
# if there still is keywords
if verbose:
print ' deleting auths'
run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accACTION = %s %s """,
(id_action, update['allowedkeywords'] and
'AND id_accARGUMENT != -1' or ''))
if update.has_key('optional'):
# check if there changing optional or not
if verbose:
print 'optional'
if run_sql("""SELECT id FROM accACTION
WHERE id = %s AND optional != %s """,
(id_action, update['optional'])):
# change optional
if verbose:
print ' changing'
run_sql("""UPDATE accACTION SET optional = %s WHERE id = %s""",
(update['optional'], id_action))
# setting it to no, delete authorizations with
# optional arguments
if update['optional'] == 'no':
if verbose:
print ' deleting optional'
run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1 """, (id_action, ))
except ProgrammingError:
return 0
return 1
# ROLES
def acc_add_role(name_role, description,
firerole_def_ser = CFG_ACC_EMPTY_ROLE_DEFINITION_SER,
firerole_def_src = CFG_ACC_EMPTY_ROLE_DEFINITION_SRC):
"""add a new role to accROLE in the database.
name_role - name of the role, must be unique
description - text to describe the role
firerole_def_ser - compiled firewall like role definition
firerole_def_src - firewall like role definition sources
"""
if not run_sql("""SELECT name FROM accROLE WHERE name = %s""", (name_role, )):
res = run_sql("""INSERT INTO accROLE (name, description,
firerole_def_ser, firerole_def_src)
VALUES (%s, %s, %s, %s)""",
(name_role, description, firerole_def_ser, firerole_def_src))
return res, name_role, description, firerole_def_src
return 0
def acc_is_role(name_action, **arguments):
""" check whether the role which allows action name_action on arguments
exists (different from SUPERADMINROLE)
action_name - name of the action
arguments - arguments for authorization"""
# first check if an action exists with this name
id_action = acc_get_action_id(name_action)
arole = run_sql("SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT WHERE id_accACTION=%s AND argumentlistid <= 0 LIMIT 1", (id_action, ), 1)
if arole:
return True
other_roles_to_check = run_sql("SELECT id_accROLE, keyword, value, argumentlistid FROM accROLE_accACTION_accARGUMENT JOIN accARGUMENT ON id_accARGUMENT=id WHERE id_accACTION=%s AND argumentlistid > 0", (id_action, ))
other_roles_to_check_dict = {}
for id_accROLE, keyword, value, argumentlistid in other_roles_to_check:
try:
other_roles_to_check_dict[(id_accROLE, argumentlistid)][keyword] = value
except KeyError:
other_roles_to_check_dict[(id_accROLE, argumentlistid)] = {keyword : value}
for ((id_accROLE, argumentlistid), stored_arguments) in other_roles_to_check_dict.iteritems():
for key, value in stored_arguments.iteritems():
if (value != arguments.get(key, '*') != '*') and value != '*':
break
else:
return True
return False
def acc_delete_role(id_role=0, name_role=0):
""" delete role entry in table accROLE and all references from
other tables.
id_role - id of role to be deleted, prefered variable
name_role - this is used if id_role is not given
note: you can't delete the SUPERADMINROLE
"""
count = 0
id_role = id_role or acc_get_role_id(name_role=name_role)
if SUPERADMINROLE == acc_get_role_name(id_role):
return 0
# try to delete
if run_sql("""DELETE FROM accROLE WHERE id = %s """ % (id_role, )):
# delete everything related
# authorization entries
count += 1 + run_sql("""DELETE FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s""",
(id_role, ))
# connected users
count += run_sql("""DELETE FROM user_accROLE WHERE id_accROLE = %s""",
(id_role, ))
# delegated rights over the role
rolenames = run_sql("""SELECT name FROM accROLE""")
# string of rolenames
roles_str = ''
for (name, ) in rolenames:
roles_str += (roles_str and ',' or '') + \
'"%s"' % (name, )
# arguments with non existing rolenames
not_valid = run_sql("""SELECT ar.id FROM accARGUMENT ar
WHERE keyword = 'role' AND value NOT IN (%s)""" % (roles_str, ))
if not_valid:
nv_str = ''
for (id_value, ) in not_valid:
nv_str += (nv_str and ',' or '') + \
'%s' % (id_value, )
# delete entries
count += run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accACTION = %s AND id_accARGUMENT IN (%s) """ %
(acc_get_action_id(name_action=DELEGATEADDUSERROLE), nv_str))
# return number of deletes
return count
def acc_update_role(id_role=0, name_role='', dummy=0, description='', \
firerole_def_ser=CFG_ACC_EMPTY_ROLE_DEFINITION_SER, \
firerole_def_src=CFG_ACC_EMPTY_ROLE_DEFINITION_SRC):
"""try to change the description.
id_role - id of the role to change
name_role - use this to find id if not present
verbose - extra output
description - new description
firerole_def_ser - compiled firewall like role definition
firerole_def_src - firewall like role definition
"""
id_role = id_role or acc_get_role_id(name_role=name_role)
if not id_role:
return 0
return run_sql("""UPDATE accROLE SET description = %s,
firerole_def_ser = %s, firerole_def_src = %s
WHERE id = %s""", (description, firerole_def_ser,
firerole_def_src, id_role))
# CONNECTIONS BETWEEN USER AND ROLE
def acc_add_user_role(id_user=0, id_role=0, email='', name_role='',
expiration='9999-12-31 23:59:59'):
""" this function adds a new entry to table user_accROLE and returns it
id_user, id_role - self explanatory
email - email of the user
name_role - name of the role, to be used instead of id. """
id_user = id_user or acc_get_user_id(email=email)
id_role = id_role or acc_get_role_id(name_role=name_role)
# check if the id_role exists
if id_role and not acc_get_role_name(id_role=id_role):
return 0
# check that the user actually exist
if not acc_get_user_email(id_user=id_user):
return 0
# control if existing entry
if run_sql("""SELECT id_user FROM user_accROLE WHERE id_user = %s AND
id_accROLE = %s""", (id_user, id_role)):
run_sql("""UPDATE user_accROLE SET expiration=%s WHERE id_user=%s AND
id_accROLE=%s AND expiration<%s""",
(expiration, id_user, id_role, expiration) )
return id_user, id_role, 0
else:
run_sql("""INSERT INTO user_accROLE (id_user, id_accROLE, expiration)
VALUES (%s, %s, %s) """, (id_user, id_role, expiration))
return id_user, id_role, 1
def acc_delete_user_role(id_user, id_role=0, name_role=0):
""" function deletes entry from user_accROLE and reports the success.
id_user - user in database
id_role - role in the database, prefered parameter
name_role - can also delete role on background of role name. """
# need to find id of the role
id_role = id_role or acc_get_role_id(name_role=name_role)
# number of deleted entries will be returned (0 or 1)
return run_sql("""DELETE FROM user_accROLE WHERE id_user = %s
AND id_accROLE = %s """, (id_user, id_role))
# ARGUMENTS
def acc_add_argument(keyword='', value=''):
""" function to insert an argument into table accARGUMENT.
if it exists the old id is returned, if it does not the entry is
created and the new id is returned.
keyword - inserted in keyword column
value - inserted in value column. """
# if one of the values are missing, return 0
if not keyword or not value:
return 0
# try to return id of existing argument
try:
return run_sql("""SELECT id from accARGUMENT where keyword = %s and
value = %s""", (keyword, value))[0][0]
# return id of newly added argument
except IndexError:
return run_sql("""INSERT INTO accARGUMENT (keyword, value)
VALUES (%s, %s) """, (keyword, value))
def acc_delete_argument(id_argument):
""" functions deletes one entry in table accARGUMENT.
the success of the operation is returned.
id_argument - id of the argument to be deleted"""
# return number of deleted entries, 1 or 0
return run_sql("""DELETE FROM accARGUMENT WHERE id = %s """,
(id_argument, ))
def acc_delete_argument_names(keyword='', value=''):
"""delete argument according to keyword and value,
send call to another function..."""
# one of the values is missing
if not keyword or not value:
return 0
# find id of the entry
try:
return run_sql("""SELECT id from accARGUMENT where keyword = %s
and value = %s""", (keyword, value))[0][0]
except IndexError:
return 0
# AUTHORIZATIONS
# ADD WITH names and keyval list
def acc_add_authorization(name_role='', name_action='', optional=0, **keyval):
""" function inserts entries in accROLE_accACTION_accARGUMENT if all
references are valid.
this function is made specially for the webaccessadmin web interface.
always inserting only one authorization.
id_role, id_action - self explanatory, preferably used
name_role, name_action - self explanatory, used if id not given
optional - if this is set to 1, check that function can have optional
arguments and add with arglistid -1 and id_argument -1
**keyval - dictionary of keyword=value pairs, used to find ids. """
inserted = []
# check that role and action exist
id_role = run_sql("""SELECT id FROM accROLE where name = %s""",
(name_role, ))
action_details = run_sql("""SELECT id,name,description,allowedkeywords,optional from accACTION where name = %s """,
(name_action, ))
if not id_role or not action_details:
return []
# get role id and action id and details
id_role, id_action = id_role[0][0], action_details[0][0]
allowedkeywords_str = action_details[0][3]
allowedkeywords_lst = acc_get_action_keywords(id_action=id_action)
optional_action = action_details[0][4] == 'yes' and 1 or 0
optional = int(optional)
# this action does not take arguments
if not optional and not keyval:
# can not add if user is doing a mistake
if allowedkeywords_str:
return []
# check if entry exists
if not run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s AND
argumentlistid = %s AND id_accARGUMENT = %s""",
(id_role, id_action, 0, 0)):
# insert new authorization
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, %s, %s)""", (id_role, id_action, 0, 0))
return [[id_role, id_action, 0, 0], ]
return []
# try to add authorization without the optional arguments
elif optional:
# optional not allowed for this action
if not optional_action:
return []
# check if authorization already exists
if not run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1""" % (id_role, id_action, )):
# insert new authorization
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, -1, -1) """, (id_role, id_action))
return [[id_role, id_action, -1, -1], ]
return []
else:
# regular authorization
# get list of ids, if they don't exist, create arguments
id_arguments = []
argstr = ''
for key in keyval.keys():
if key not in allowedkeywords_lst:
return []
id_argument = (acc_get_argument_id(key, keyval[key])
or
run_sql("""INSERT INTO accARGUMENT (keyword, value) values
(%s, %s) """, (key, keyval[key])))
id_arguments.append(id_argument)
argstr += (argstr and ',' or '') + str(id_argument)
# check if equal authorization exists
for (id_trav, ) in run_sql("""SELECT DISTINCT argumentlistid FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND
id_accACTION = %s """, (id_role, id_action)):
listlength = run_sql("""SELECT COUNT(*) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %%s AND
id_accACTION = %%s AND argumentlistid = %%s AND
id_accARGUMENT IN (%s) """ % (argstr),
(id_role, id_action, id_trav))[0][0]
notlist = run_sql("""SELECT COUNT(*) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %%s AND
id_accACTION = %%s AND argumentlistid = %%s AND
id_accARGUMENT NOT IN (%s) """ % (argstr),
(id_role, id_action, id_trav))[0][0]
# this means that a duplicate already exists
if not notlist and listlength == len(id_arguments):
return []
# find new arglistid, highest + 1
try:
arglistid = 1 + run_sql("""SELECT MAX(argumentlistid) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s
AND id_accACTION = %s""", (id_role, id_action))[0][0]
except (IndexError, TypeError):
arglistid = 1
if arglistid <= 0:
arglistid = 1
# insert
for id_argument in id_arguments:
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, %s, %s) """,
(id_role, id_action, id_argument, arglistid))
inserted.append([id_role, id_action, id_argument, arglistid])
return inserted
def acc_add_role_action_arguments(id_role=0, id_action=0, arglistid=-1,
optional=0, verbose=0, id_arguments=[]):
""" function inserts entries in accROLE_accACTION_accARGUMENT if all
references are valid.
id_role, id_action - self explanatory
arglistid - argumentlistid for the inserted entries
if -1: create new group
other values: add to this group, if it exists or not
optional - if this is set to 1, check that function can have
optional arguments and add with arglistid -1 and
id_argument -1
verbose - extra output
id_arguments - list of arguments to add to group."""
inserted = []
if verbose:
print 'ids: starting'
if verbose:
print 'ids: checking ids'
# check that all the ids are valid and reference something...
if not run_sql("""SELECT id FROM accROLE WHERE id = %s""", (id_role, )):
return 0
if verbose:
print 'ids: get allowed keywords'
# check action exist and get allowed keywords
try:
allowedkeys = acc_get_action_keywords(id_action=id_action)
# allowedkeys = run_sql("""SELECT id FROM accACTION WHERE id = %s""" %
# (id_action, ))[0][3].split(',')
except (IndexError, AttributeError):
return 0
if verbose:
print 'ids: is it optional'
# action with optional arguments
if optional:
if verbose:
print 'ids: yes - optional'
if not acc_get_action_is_optional(id_action=id_action):
return []
if verbose:
print 'ids: run query to check if exists'
if not run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1""", (id_role, id_action, )):
if verbose:
print 'ids: does not exist'
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, -1, -1) """, (id_role, id_action))
return ((id_role, id_action, -1, -1), )
if verbose:
print 'ids: exists'
return []
if verbose:
print 'ids: check if not arguments'
# action without arguments
if not allowedkeys:
if verbose:
print 'ids: not arguments'
if not run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s AND
argumentlistid = %s AND id_accARGUMENT = %s""",
(id_role, id_action, 0, 0)):
if verbose:
print 'ids: try to insert'
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, %s, %s)""", (id_role, id_action, 0, 0))
return ((id_role, id_action, 0, 0), )
else:
if verbose:
print 'ids: already existed'
return 0
else:
if verbose:
print 'ids: arguments exist'
argstr = ''
# check that the argument exists, and that it is a valid key
if verbose:
print 'ids: checking all the arguments'
for id_argument in id_arguments:
res_arg = run_sql("""SELECT id,keyword,value FROM accARGUMENT WHERE id = %s""",
(id_argument, ))
if not res_arg or res_arg[0][1] not in allowedkeys:
return 0
else:
if argstr:
argstr += ','
argstr += '%s' % (id_argument, )
# arglistid = -1 means that the user wants a new group
if verbose:
print 'ids: find arglistid'
if arglistid < 0:
# check if such single group already exists
for (id_trav, ) in run_sql("""SELECT DISTINCT argumentlistid FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND
id_accACTION = %s""", (id_role, id_action)):
listlength = run_sql("""SELECT COUNT(*) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %%s AND
id_accACTION = %%s AND argumentlistid = %%s AND
id_accARGUMENT IN (%s)""" % (argstr),
(id_role, id_action, id_trav))[0][0]
notlist = run_sql("""SELECT COUNT(*) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %%s AND
id_accACTION = %%s AND argumentlistid = %%s AND
id_accARGUMENT NOT IN (%s)""" % (argstr),
(id_role, id_action, id_trav))[0][0]
# this means that a duplicate already exists
if not notlist and listlength == len(id_arguments):
return 0
# find new arglistid
try:
arglistid = run_sql("""SELECT MAX(argumentlistid) FROM
accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND
id_accACTION = %s""", (id_role, id_action))[0][0] + 1
except ProgrammingError:
return 0
except (IndexError, TypeError):
arglistid = 1
if arglistid <= 0:
arglistid = 1
if verbose:
print 'ids: insert all the entries'
# all references are valid, insert: one entry in raa for each argument
for id_argument in id_arguments:
if not run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s AND
id_accARGUMENT = %s AND argumentlistid = %s""",
(id_role, id_action, id_argument, arglistid)):
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT (id_accROLE,
id_accACTION, id_accARGUMENT, argumentlistid)
VALUES (%s, %s, %s, %s)""",
(id_role, id_action, id_argument, arglistid))
inserted.append((id_role, id_action, id_argument, arglistid))
# [(r, ac, ar1, aid), (r, ac, ar2, aid)]
if verbose:
print 'ids: inside add function'
for r in acc_find_possible_actions(id_role=id_role,
id_action=id_action):
print 'ids: ', r
return inserted
def acc_add_role_action_arguments_names(name_role='', name_action='',
arglistid=-1, optional=0, verbose=0, **keyval):
""" this function makes it possible to pass names when creating new entries
instead of ids.
get ids for all the names,
create entries in accARGUMENT that does not exist,
pass on to id based function.
name_role, name_action - self explanatory
arglistid - add entries to or create group with arglistid, default -1
create new.
optional - create entry with optional keywords, **keyval is ignored, but
should be empty
verbose - used to print extra information
**keyval - dictionary of keyword=value pairs, used to find ids. """
if verbose:
print 'names: starting'
if verbose:
print 'names: checking ids'
# find id of the role, return 0 if it doesn't exist
id_role = run_sql("""SELECT id FROM accROLE where name = %s""",
(name_role, ))
if id_role:
id_role = id_role[0][0]
else:
return 0
# find id of the action, return 0 if it doesn't exist
res = run_sql("""SELECT id from accACTION where name = %s""",
(name_action, ))
if res:
id_action = res[0][0]
else:
return 0
if verbose:
print 'names: checking arguments'
id_arguments = []
if not optional:
if verbose:
print 'names: not optional'
# place to keep ids of arguments and list of allowed keywords
allowedkeys = acc_get_action_keywords(id_action=id_action)
# res[0][3].split(',')
# find all the id_arguments and create those that does not exist
for key in keyval.keys():
# this key does not exist
if key not in allowedkeys:
return 0
id_argument = acc_get_argument_id(key, keyval[key])
id_argument = id_argument or \
run_sql("""INSERT INTO accARGUMENT (keyword, value)
VALUES (%s, %s) """, (key, keyval[key]))
id_arguments.append(id_argument) # append the id to the list
else:
if verbose:
print 'names: optional'
# use the other function
return acc_add_role_action_arguments(id_role=id_role,
id_action=id_action,
arglistid=arglistid,
optional=optional,
verbose=verbose,
id_arguments=id_arguments)
# DELETE WITH ID OR NAMES
def acc_delete_role_action_arguments(id_role, id_action, arglistid=1,
auths=[[]]):
"""delete all entries in accROLE_accACTION_accARGUMENT that satisfy the
parameters.
return number of actual deletes.
this function relies on the id-lists in auths to have the same order has
the possible actions...
id_role, id_action - self explanatory
arglistid - group to delete from.
if more entries than deletes, split the group before delete.
id_arguments - list of ids to delete."""
keepauths = [] # these will be kept
# find all possible actions
pas = acc_find_possible_actions_ids(id_role, id_action)
dummy = pas[0]
# decide which to keep or throw away
for pa in pas[1:]:
if pa[0] == arglistid and pa[1:] not in auths:
keepauths.append(pa[1:])
# delete everything
run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
argumentlistid = %s""", (id_role, id_action, arglistid))
# insert those to be kept
for auth in keepauths:
acc_add_role_action_arguments(id_role=id_role,
id_action=id_action,
arglistid=-1,
id_arguments=auth)
return 1
def acc_delete_role_action_arguments_names(name_role='', name_action='',
arglistid=1, **keyval):
"""utilize the function on ids by first finding all ids and redirecting the
function call.
break of and return 0 if any of the ids can't be found.
name_role = name of the role
name_action - name of the action
arglistid - the argumentlistid, all keyword=value pairs must be in this
same group.
**keyval - dictionary of keyword=value pairs for the arguments."""
# find ids for role and action
id_role = acc_get_role_id(name_role=name_role)
id_action = acc_get_action_id(name_action=name_action)
# create string with the ids
idstr = ''
idlist = []
for key in keyval.keys():
argument_id = acc_get_argument_id(key, keyval[key])
if not argument_id:
return 0
if idstr:
idstr += ','
idstr += '%s' % argument_id
idlist.append(argument_id)
# control that a fitting group exists
try:
count = run_sql("""SELECT COUNT(*) FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %%s AND
id_accACTION = %%s AND
argumentlistid = %%s AND
id_accARGUMENT IN (%s)""" % (idstr),
(id_role, id_action, arglistid))[0][0]
except IndexError:
return 0
if count < len(keyval):
return 0
# call id based function
return acc_delete_role_action_arguments(id_role, id_action, arglistid,
[idlist])
def acc_delete_role_action_arguments_group(id_role=0, id_action=0, arglistid=0):
"""delete entire group of arguments for connection between
role and action."""
if not id_role or not id_action:
return []
return run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
argumentlistid = %s """, (id_role, id_action, arglistid))
def acc_delete_possible_actions(id_role=0, id_action=0, authids=[]):
"""delete authorizations in selected rows. utilization of the
delete function.
id_role - id of role to be connected to action.
id_action - id of action to be connected to role
authids - list of row indexes to be removed. """
# find all authorizations
pas = acc_find_possible_actions(id_role=id_role, id_action=id_action)
# get the keys
keys = pas[0][1:]
# create dictionary for all the argumentlistids
ald = {}
for authid in authids:
if authid > len(pas):
return authid, len(pas)
# get info from possible action
pas_auth_id = pas[authid][0]
values = pas[authid][1:]
# create list of authids for each authorization
auth = [acc_get_argument_id(keys[0], values[0])]
for i in range(1, len(keys)):
auth.append(acc_get_argument_id(keys[i], values[i]))
# create entries in the dictionary for each argumentlistid
try:
ald[pas_auth_id].append(auth)
except KeyError:
ald[pas_auth_id] = [auth]
# do the deletes
result = 1
for key in ald.keys():
result = 1 and acc_delete_role_action_arguments(id_role=id_role,
id_action=id_action,
arglistid=key,
auths=ald[key])
return result
def acc_delete_role_action(id_role=0, id_action=0):
"""delete all connections between a role and an action. """
count = run_sql("""DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s """, (id_role, id_action))
return count
# GET FUNCTIONS
# ACTION RELATED
def acc_get_action_id(name_action):
"""get id of action when name is given
name_action - name of the wanted action"""
try:
return run_sql("""SELECT id FROM accACTION WHERE name = %s""",
(name_action, ))[0][0]
except IndexError:
return 0
def acc_get_action_name(id_action):
"""get name of action when id is given. """
try:
return run_sql("""SELECT name FROM accACTION WHERE id = %s""",
(id_action, ))[0][0]
except (ProgrammingError, IndexError):
return ''
def acc_get_action_description(id_action):
"""get description of action when id is given. """
try:
return run_sql("""SELECT description FROM accACTION WHERE id = %s""",
(id_action, ))[0][0]
except (ProgrammingError, IndexError):
return ''
def acc_get_action_keywords(id_action=0, name_action=''):
"""get list of keywords for action when id is given.
empty list if no keywords."""
result = acc_get_action_keywords_string(id_action=id_action,
name_action=name_action)
if result:
return result.split(',')
else:
return []
def acc_get_action_keywords_string(id_action=0, name_action=''):
"""get keywordstring when id is given. """
id_action = id_action or acc_get_action_id(name_action)
try:
result = run_sql("""SELECT allowedkeywords from accACTION
where id = %s """, (id_action, ))[0][0]
except IndexError:
return ''
return result
def acc_get_action_is_optional(id_action=0):
"""get if the action arguments are optional or not.
return 1 if yes, 0 if no."""
result = acc_get_action_optional(id_action=id_action)
return result == 'yes' and 1 or 0
def acc_get_action_optional(id_action=0):
"""get if the action arguments are optional or not.
return result, but 0 if action does not exist. """
try:
result = run_sql("""SELECT optional from accACTION where id = %s""",
(id_action, ))[0][0]
except IndexError:
return 0
return result
def acc_get_action_details(id_action=0):
"""get all the fields for an action."""
try:
result = run_sql("""SELECT id,name,description,allowedkeywords,optional FROM accACTION WHERE id = %s""",
(id_action, ))[0]
except IndexError:
return []
if result:
return list(result)
else:
return []
def acc_get_all_actions():
"""returns all entries in accACTION."""
- return run_sql_cached("""SELECT id, name, description
- FROM accACTION ORDER BY name""", affected_tables=['accACTION'])
+ return run_sql("""SELECT id, name, description
+ FROM accACTION ORDER BY name""")
def acc_get_action_roles(id_action):
"""Returns all the roles connected with an action."""
- return run_sql_cached("""SELECT DISTINCT(r.id), r.name, r.description
+ return run_sql("""SELECT DISTINCT(r.id), r.name, r.description
FROM accROLE_accACTION_accARGUMENT raa, accROLE r
WHERE (raa.id_accROLE = r.id AND raa.id_accACTION = %s) OR r.name = %s
- ORDER BY r.name """, (id_action, SUPERADMINROLE), affected_tables=['accROLE_accACTION_accARGUMENT', 'accROLE'])
+ ORDER BY r.name """, (id_action, SUPERADMINROLE))
# ROLE RELATED
def acc_get_role_id(name_role):
"""get id of role, name given. """
try:
return run_sql("""SELECT id FROM accROLE WHERE name = %s""",
(name_role, ))[0][0]
except IndexError:
return 0
def acc_get_role_name(id_role):
"""get name of role, id given. """
try:
return run_sql("""SELECT name FROM accROLE WHERE id = %s""",
(id_role, ))[0][0]
except IndexError:
return ''
def acc_get_role_definition(id_role=0):
"""get firewall like role definition object for a role."""
try:
return run_sql("""SELECT firerole_def_ser FROM accROLE
WHERE id = %s""", (id_role, ))[0][0]
except IndexError:
return ''
def acc_get_role_details(id_role=0):
"""get all the fields for a role."""
try:
result = run_sql("""SELECT id, name, description, firerole_def_src
FROM accROLE WHERE id = %s """, (id_role, ))[0]
except IndexError:
return []
if result:
return list(result)
else:
return []
def acc_get_all_roles():
"""get all entries in accROLE."""
- return run_sql_cached("""SELECT id, name, description,
+ return run_sql("""SELECT id, name, description,
firerole_def_ser, firerole_def_src
- FROM accROLE ORDER BY name""", affected_tables=['accROLE'])
+ FROM accROLE ORDER BY name""")
def acc_get_role_actions(id_role):
"""get all actions connected to a role. """
if acc_get_role_name(id_role) == SUPERADMINROLE:
- return run_sql_cached("""SELECT id, name, description
+ return run_sql("""SELECT id, name, description
FROM accACTION
- ORDER BY name """, affected_tables=['accACTION'])
+ ORDER BY name """)
else:
- return run_sql_cached("""SELECT DISTINCT(a.id), a.name, a.description
+ return run_sql("""SELECT DISTINCT(a.id), a.name, a.description
FROM accROLE_accACTION_accARGUMENT raa, accACTION a
WHERE raa.id_accROLE = %s and
raa.id_accACTION = a.id
- ORDER BY a.name""", (id_role, ), affected_tables=['accACTION', 'accROLE_accACTION_accARGUMENT'])
+ ORDER BY a.name""", (id_role, ))
def acc_get_role_users(id_role):
"""get all users that have direct access to a role.
Note this function will not consider implicit user linked by the
FireRole definition.
"""
return run_sql("""SELECT DISTINCT(u.id), u.email, u.settings
FROM user_accROLE ur, user u
WHERE ur.id_accROLE = %s AND
ur.expiration >= NOW() AND
u.id = ur.id_user
ORDER BY u.email""", (id_role, ))
# ARGUMENT RELATED
def acc_get_argument_id(keyword, value):
"""get id of argument, keyword=value pair given.
value = 'optional value' is replaced for id_accARGUMENT = -1."""
try:
return run_sql("""SELECT DISTINCT id FROM accARGUMENT
WHERE keyword = %s and value = %s""", (keyword, value))[0][0]
except IndexError:
if value == 'optional value':
return -1
return 0
# USER RELATED
def acc_get_user_email(id_user=0):
"""get email of user, id given."""
try:
return run_sql("""SELECT email FROM user WHERE id = %s """,
(id_user, ))[0][0].lower().strip()
except IndexError:
return ''
def acc_get_user_id(email=''):
"""get id of user, email given."""
try:
return run_sql("""SELECT id FROM user WHERE email = %s """,
(email.lower().strip(), ))[0][0]
except IndexError:
return 0
def acc_is_user_in_role(user_info, id_role):
"""Return True if the user belong implicitly or explicitly to the role."""
if run_sql("""SELECT ur.id_accROLE
FROM user_accROLE ur
WHERE ur.id_user = %s AND ur.expiration >= NOW() AND
ur.id_accROLE = %s LIMIT 1""", (user_info['uid'], id_role), 1):
return True
return acc_firerole_check_user(user_info, load_role_definition(id_role))
def acc_get_user_roles_from_user_info(user_info):
"""get all roles a user is connected to."""
roles = intbitset(run_sql("""SELECT ur.id_accROLE
FROM user_accROLE ur
WHERE ur.id_user = %s AND ur.expiration >= NOW()
ORDER BY ur.id_accROLE""", (user_info['uid'], )))
potential_implicit_roles = run_sql("""SELECT id, firerole_def_ser FROM accROLE
WHERE firerole_def_ser IS NOT NULL""")
for role_id, firerole_def_ser in potential_implicit_roles:
if role_id not in roles:
if acc_firerole_check_user(user_info, deserialize(firerole_def_ser)):
roles.add(role_id)
return roles
def acc_get_user_roles(id_user):
"""get all roles a user is explicitly connected to."""
explicit_roles = run_sql("""SELECT ur.id_accROLE
FROM user_accROLE ur
WHERE ur.id_user = %s AND ur.expiration >= NOW()
ORDER BY ur.id_accROLE""", (id_user, ))
return [id_role[0] for id_role in explicit_roles]
def acc_find_possible_activities(user_info, ln=CFG_SITE_LANG):
"""Return a dictionary with all the possible activities for which the user
is allowed (i.e. all the administrative action which are connected to
an web area in Invenio) and the corresponding url.
"""
_ = gettext_set_language(ln)
your_role_actions = acc_find_user_role_actions(user_info)
your_admin_activities = {}
for (role, action) in your_role_actions:
if CFG_ACC_ACTIVITIES_URLS.has_key(action):
your_admin_activities[action] = CFG_ACC_ACTIVITIES_URLS[action]
if role == SUPERADMINROLE:
your_admin_activities = dict(CFG_ACC_ACTIVITIES_URLS)
break
ret = {}
for action, (name, url) in your_admin_activities.iteritems():
ret[_(name)] = url % ln
return ret
def acc_find_user_role_actions(user_info):
"""find name of all roles and actions connected to user_info."""
uid = user_info['uid']
# Let's check if user is superadmin
id_superadmin = acc_get_role_id(SUPERADMINROLE)
if id_superadmin in acc_get_user_roles_from_user_info(user_info):
return [(SUPERADMINROLE, action[1]) for action in acc_get_all_actions()]
query = """SELECT DISTINCT r.name, a.name
FROM user_accROLE ur, accROLE_accACTION_accARGUMENT raa, accACTION a,
accROLE r
WHERE ur.id_user = %s and
ur.expiration >= NOW() and
ur.id_accROLE = raa.id_accROLE and
raa.id_accACTION = a.id and
raa.id_accROLE = r.id """
res1 = run_sql(query, (uid, ))
res2 = []
for res in res1:
res2.append(res)
res2.sort()
if type(user_info) == type({}):
query = """SELECT DISTINCT r.name, a.name, r.firerole_def_ser
FROM accROLE_accACTION_accARGUMENT raa, accACTION a, accROLE r
WHERE raa.id_accACTION = a.id and
raa.id_accROLE = r.id """
res3 = run_sql(query)
res4 = []
for role_name, action_name, role_definition in res3:
if acc_firerole_check_user(user_info,
deserialize(role_definition)):
if role_name == SUPERADMINROLE:
# Ok, every action. There's no need to go on :-)
return [(id_superadmin, action[0]) for action in acc_get_all_actions()]
res4.append((role_name, action_name))
return list(set(res2) | set(res4))
else:
return res2
# POSSIBLE ACTIONS / AUTHORIZATIONS
def acc_find_possible_actions_all(id_role):
"""find all the possible actions for a role.
the function utilizes acc_find_possible_actions to find
all the entries from each of the actions under the given role
id_role - role to find all actions for
returns a list with headers"""
query = """SELECT DISTINCT(aar.id_accACTION)
FROM accROLE_accACTION_accARGUMENT aar
WHERE aar.id_accROLE = %s
ORDER BY aar.id_accACTION""" % (id_role, )
res = []
for (id_action, ) in run_sql(query):
hlp = acc_find_possible_actions(id_role, id_action)
if hlp:
res.append(['role', 'action'] + hlp[0])
for row in hlp[1:]:
res.append([id_role, id_action] + row)
return res
def acc_find_possible_actions_argument_listid(id_role, id_action, arglistid):
"""find all possible actions with the given arglistid only."""
# get all, independent of argumentlistid
res1 = acc_find_possible_actions_ids(id_role, id_action)
# create list with only those with the right arglistid
res2 = []
for row in res1[1:]:
if row[0] == arglistid:
res2.append(row)
# return this list
return res2
def acc_find_possible_roles(name_action, always_add_superadmin=True, **arguments):
"""Find all the possible roles that are enabled to action_name with
given arguments. roles is a list of role_id
"""
id_action = acc_get_action_id(name_action)
roles = intbitset(run_sql("SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT WHERE id_accACTION=%s AND argumentlistid <= 0", (id_action, )))
if always_add_superadmin:
roles.add(CFG_SUPERADMINROLE_ID)
other_roles_to_check = run_sql("SELECT id_accROLE, keyword, value, argumentlistid FROM accROLE_accACTION_accARGUMENT JOIN accARGUMENT ON id_accARGUMENT=id WHERE id_accACTION=%s AND argumentlistid > 0", (id_action, ))
other_roles_to_check_dict = {}
for id_accROLE, keyword, value, argumentlistid in other_roles_to_check:
if id_accROLE not in roles:
try:
other_roles_to_check_dict[(id_accROLE, argumentlistid)][keyword] = value
except KeyError:
other_roles_to_check_dict[(id_accROLE, argumentlistid)] = {keyword : value}
for ((id_accROLE, argumentlistid), stored_arguments) in other_roles_to_check_dict.iteritems():
for key, value in stored_arguments.iteritems():
if (value != arguments.get(key, '*') != '*') and value != '*':
break
else:
roles.add(id_accROLE)
return roles
def acc_find_possible_actions_user_from_user_info(user_info, id_action):
"""user based function to find all action combination for a given
user and action. find all the roles and utilize findPossibleActions
for all these.
user_info - user information dictionary, used to find roles
id_action - action id.
"""
res = []
for id_role in acc_get_user_roles_from_user_info(user_info):
hlp = acc_find_possible_actions(id_role, id_action)
if hlp and not res:
res.append(['role'] + hlp[0])
for row in hlp[1:]:
res.append([id_role] + row)
return res
def acc_find_possible_actions_user(id_user, id_action):
"""user based function to find all action combination for a given
user and action. find all the roles and utilize findPossibleActions
for all these.
id_user - user id, used to find roles
id_action - action id.
Note this function considers only explicit links between users and roles,
and not FireRole definitions.
"""
res = []
for id_role in acc_get_user_roles(id_user):
hlp = acc_find_possible_actions(id_role, id_action)
if hlp and not res:
res.append(['role'] + hlp[0])
for row in hlp[1:]:
res.append([id_role] + row)
return res
def acc_find_possible_actions_ids(id_role, id_action):
"""finds the ids of the possible actions.
utilization of acc_get_argument_id and acc_find_possible_actions. """
pas = acc_find_possible_actions(id_role, id_action)
if not pas:
return []
keys = pas[0]
pas_ids = [pas[0:1]]
for pa in pas[1:]:
auth = [pa[0]]
for i in range(1, len(pa)):
auth.append(acc_get_argument_id(keys[i], pa[i]))
pas_ids.append(auth)
return pas_ids
def acc_find_possible_actions(id_role, id_action):
"""Role based function to find all action combinations for a
give role and action.
id_role - id of role in the database
id_action - id of the action in the database
returns a list with all the combinations.
first row is used for header.
if SUPERADMINROLE, nothing is returned since an infinte number of
combination are possible.
"""
# query to find all entries for user and action
- res1 = run_sql_cached(""" SELECT raa.argumentlistid, ar.keyword, ar.value
+ res1 = run_sql(""" SELECT raa.argumentlistid, ar.keyword, ar.value
FROM accROLE_accACTION_accARGUMENT raa, accARGUMENT ar
WHERE raa.id_accROLE = %s and
raa.id_accACTION = %s and
- raa.id_accARGUMENT = ar.id """, (id_role, id_action), affected_tables=['accROLE_accACTION_accARGUMENT', 'accARGUMENT'])
+ raa.id_accARGUMENT = ar.id """, (id_role, id_action))
# find needed keywords, create header
keywords = acc_get_action_keywords(id_action=id_action)
keywords.sort()
if not keywords:
# action without arguments
- if run_sql_cached("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
+ if run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s AND id_accARGUMENT = 0
- AND argumentlistid = 0""", (id_role, id_action), affected_tables=['accROLE_accACTION_accARGUMENT']):
+ AND argumentlistid = 0""", (id_role, id_action)):
return [['#', 'argument keyword'],
['0', 'action without arguments']]
# tuples into lists
res2, arglistids = [], {}
for res in res1:
res2.append([])
for r in res:
res2[-1].append(r)
res2.sort()
# create multilevel dictionary
for res in res2:
a, kw, value = res # rolekey, argumentlistid, keyword, value
if kw not in keywords:
continue
if not arglistids.has_key(a):
arglistids[a] = {}
# fill dictionary
if not arglistids[a].has_key(kw):
arglistids[a][kw] = [value]
elif not value in arglistids[a][kw]:
arglistids[a][kw] = arglistids[a][kw] + [value]
# fill list with all possible combinations
res3 = []
# rolekeys = roles2.keys(); rolekeys.sort()
for a in arglistids.keys(): # argumentlistids
# fill a list with the new entries, shortcut and copying first
# keyword list
next_arglistid = []
for row in arglistids[a][keywords[0]]:
next_arglistid.append([a, row[:] ])
# run through the rest of the keywords
for kw in keywords[1:]:
if not arglistids[a].has_key(kw):
arglistids[a][kw] = ['optional value']
new_list = arglistids[a][kw][:]
new_len = len(new_list)
# duplicate the list
temp_list = []
for row in next_arglistid:
for i in range(new_len):
temp_list.append(row[:])
# append new values
for i in range(len(temp_list)):
new_item = new_list[i % new_len][:]
temp_list[i].append( new_item )
next_arglistid = temp_list[:]
res3.extend(next_arglistid)
res3.sort()
# if optional allowed, put on top
opt = run_sql("""SELECT id_accROLE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1""", (id_role, id_action))
if opt:
res3.insert(0, [-1] + ['optional value'] * len(keywords))
# put header on top
if res3:
res3.insert(0, ['#'] + keywords)
return res3
def acc_split_argument_group(id_role=0, id_action=0, arglistid=0):
"""collect the arguments, find all combinations, delete original entries
and insert the new ones with different argumentlistids for each group
id_role - id of the role
id_action - id of the action
arglistid - argumentlistid to be splittetd"""
if not id_role or not id_action or not arglistid:
return []
# don't split if none or one possible actions
res = acc_find_possible_actions_argument_listid(id_role, id_action,
arglistid)
if not res or len(res) <= 1:
return 0
# delete the existing group
acc_delete_role_action_arguments_group(id_role, id_action,
arglistid)
# add all authorizations with new and different argumentlistid
addlist = []
for row in res:
argids = row[1:]
addlist.append(acc_add_role_action_arguments(id_role=id_role,
id_action=id_action,
arglistid=-1,
id_arguments=argids))
# return list of added authorizations
return addlist
def acc_merge_argument_groups(id_role=0, id_action=0, arglistids=[]):
"""merge the authorizations from groups with different argumentlistids
into one single group.
this can both save entries in the database and create extra authorizations.
id_role - id of the role
id_action - role of the action
arglistids - list of groups to be merged together into one."""
if len(arglistids) < 2:
return []
argstr = ''
for arglist_id in arglistids:
argstr += 'raa.argumentlistid = %s or ' % (arglist_id, )
argstr = '(%s)' % (argstr[:-4], )
# query to find all entries that will be merged
query = """ SELECT ar.keyword, ar.value, raa.id_accARGUMENT
FROM accROLE_accACTION_accARGUMENT raa, accARGUMENT ar
WHERE raa.id_accROLE = %%s and
raa.id_accACTION = %%s and
%s and
raa.id_accARGUMENT = ar.id """ % argstr
q_del = """DELETE FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %%s and
id_accACTION = %%s and
%s """ % (argstr.replace('raa.', ''))
res = run_sql(query, (id_role, id_action))
if not res:
return []
run_sql(q_del, (id_role, id_action))
# list of entire entries
old = []
# list of only the ids
ids = []
for (keyword, value, argument_id) in res:
if [keyword, value, argument_id] not in old:
old.append([keyword, value, argument_id])
ids.append(argument_id)
# for (k, v, id) in res: if id not in ids: ids.append(id)
return acc_add_role_action_arguments(id_role=id_role,
id_action=id_action,
arglistid=-1,
id_arguments=ids)
def acc_reset_default_settings(superusers=(),
additional_def_user_roles=(),
additional_def_roles=(),
additional_def_auths=()):
"""reset to default by deleting everything and adding default.
superusers - list of superuser emails
additional_def_user_roles - additional list of pair email, rolename
(see DEF_DEMO_USER_ROLES in access_control_config.py)
additional_def_roles - additional list of default list of roles
(see DEF_DEMO_ROLES in access_control_config.py)
additional_def_auths - additional list of default authorizations
(see DEF_DEMO_AUTHS in access_control_config.py)
"""
remove = acc_delete_all_settings()
add = acc_add_default_settings(superusers, additional_def_user_roles, additional_def_roles, additional_def_auths)
return remove, add
def acc_delete_all_settings():
"""simply remove all data affiliated with webaccess by truncating
tables accROLE, accACTION, accARGUMENT and those connected. """
run_sql("""TRUNCATE accROLE""")
run_sql("""TRUNCATE accACTION""")
run_sql("""TRUNCATE accARGUMENT""")
run_sql("""TRUNCATE user_accROLE""")
run_sql("""TRUNCATE accROLE_accACTION_accARGUMENT""")
return 1
def acc_add_default_settings(superusers=(),
additional_def_user_roles=(),
additional_def_roles=(),
additional_def_auths=()):
"""add the default settings if they don't exist.
superusers - list of superuser emails
additional_def_user_roles - additional list of pair email, rolename
(see DEF_DEMO_USER_ROLES in access_control_config.py)
additional_def_roles - additional list of default list of roles
(see DEF_DEMO_ROLES in access_control_config.py)
additional_def_auths - additional list of default authorizations
(see DEF_DEMO_AUTHS in access_control_config.py)
"""
# from superusers: allow input formats ['email1', 'email2'] and
# [['email1'], ['email2']] and [['email1', id], ['email2', id]]
for user in superusers:
if type(user) is str:
user = [user]
DEF_USERS.append(user[0])
if CFG_SITE_ADMIN_EMAIL not in DEF_USERS:
DEF_USERS.append(CFG_SITE_ADMIN_EMAIL)
# add data
# add roles
insroles = []
def_roles = dict([(role[0], role[1:]) for role in DEF_ROLES])
def_roles.update(dict([(role[0], role[1:]) for role in additional_def_roles]))
for name, (description, firerole_def_src) in def_roles.iteritems():
# try to add, don't care if description is different
role_id = acc_add_role(name_role=name,
description=description, firerole_def_ser=serialize(
compile_role_definition(firerole_def_src)),
firerole_def_src=firerole_def_src)
if not role_id:
role_id = acc_get_role_id(name_role=name)
acc_update_role(id_role=role_id, description=description,
firerole_def_ser=serialize(compile_role_definition(
firerole_def_src)), firerole_def_src=firerole_def_src)
insroles.append([role_id, name, description, firerole_def_src])
# add users to superadmin
insuserroles = []
for user in DEF_USERS:
insuserroles.append(acc_add_user_role(email=user,
name_role=SUPERADMINROLE))
for user, role in additional_def_user_roles:
insuserroles.append(acc_add_user_role(email=user, name_role=role))
# add actions
insactions = []
for (name, description, allkeys, optional) in DEF_ACTIONS:
# try to add action as new
action_id = acc_add_action(name, description, optional, allkeys)
# action with the name exist
if not action_id:
action_id = acc_get_action_id(name_action=name)
# update the action, necessary updates to the database
# will also be done
acc_update_action(id_action=action_id, optional=optional,
allowedkeywords=allkeys)
# keep track of inserted actions
insactions.append([action_id, name, description, allkeys])
# add authorizations
insauths = []
def_auths = list(DEF_AUTHS) + list(additional_def_auths)
for (name_role, name_action, args) in def_auths:
# add the authorization
acc_add_role_action_arguments_names(name_role=name_role,
name_action=name_action,
**args)
# keep track of inserted authorizations
insauths.append([name_role, name_action, args])
return insroles, insactions, insuserroles, insauths
def acc_find_delegated_roles(id_role_admin=0):
"""find all the roles the admin role has delegation rights over.
return tuple of all the roles.
id_role_admin - id of the admin role """
id_action_delegate = acc_get_action_id(name_action=DELEGATEADDUSERROLE)
rolenames = run_sql("""SELECT DISTINCT(ar.value)
FROM accROLE_accACTION_accARGUMENT raa LEFT JOIN accARGUMENT ar
ON raa.id_accARGUMENT = ar.id
WHERE raa.id_accROLE = %s AND
raa.id_accACTION = %s""", (id_role_admin, id_action_delegate))
result = []
for (name_role, ) in rolenames:
roledetails = run_sql("""SELECT id,name,description,firerole_def_ser,firerole_def_src FROM accROLE WHERE name = %s """,
(name_role, ))
if roledetails:
result.append(roledetails)
return result
def acc_cleanup_arguments():
"""function deletes all accARGUMENTs that are not referenced by
accROLE_accACTION_accARGUMENT.
returns how many arguments where deleted and a list of the deleted
id_arguments"""
# find unreferenced arguments
ids1 = run_sql("""SELECT DISTINCT ar.id
FROM accARGUMENT ar LEFT JOIN accROLE_accACTION_accARGUMENT raa ON
ar.id = raa.id_accARGUMENT WHERE raa.id_accARGUMENT IS NULL """)
# it is clean
if not ids1:
return 1
# create list and string of the ids
ids2 = []
idstr = ''
for (argument_id, ) in ids1:
ids2.append(argument_id)
if idstr:
idstr += ','
idstr += '%s' % argument_id
# delete unreferenced arguments
count = run_sql("""DELETE FROM accARGUMENT
WHERE id in (%s)""" % (idstr, ))
# return count and ids of deleted arguments
return (count, ids2)
diff --git a/modules/webaccess/lib/access_control_engine.py b/modules/webaccess/lib/access_control_engine.py
index 6988d544e..9760f162e 100644
--- a/modules/webaccess/lib/access_control_engine.py
+++ b/modules/webaccess/lib/access_control_engine.py
@@ -1,148 +1,148 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""CDS Invenio Access Control Engine in mod_python."""
__revision__ = "$Id$"
import cgi
import sys
from urllib import quote
if sys.hexversion < 0x2040000:
# pylint: disable=W0622
from sets import Set as set
# pylint: enable=W0622
from invenio.config import CFG_SITE_SECURE_URL
-from invenio.dbquery import run_sql_cached
+from invenio.dbquery import run_sql
from invenio.access_control_admin import acc_find_possible_roles, acc_is_user_in_role, CFG_SUPERADMINROLE_ID, acc_get_role_users
from invenio.access_control_config import CFG_WEBACCESS_WARNING_MSGS, CFG_WEBACCESS_MSGS
from invenio.webuser import collect_user_info
from invenio.access_control_firerole import acc_firerole_suggest_apache_p, deserialize, load_role_definition, acc_firerole_extract_emails
from invenio.urlutils import make_canonical_urlargd
CFG_CALLED_FROM_APACHE = 1 #1=web,0=cli
try:
import _apache
except ImportError, e:
CFG_CALLED_FROM_APACHE = 0
def make_list_apache_firerole(name_action, arguments):
"""Given an action and a dictionary arguments returns a list of all the
roles (and their descriptions) which are authorized to perform this
action with these arguments, and whose FireRole definition expect
an Apache Password membership.
"""
roles = acc_find_possible_roles(name_action, **arguments)
ret = []
for role in roles:
- res = run_sql_cached('SELECT name, description, firerole_def_ser FROM accROLE WHERE id=%s', (role, ), affected_tables=['accROLE'])
+ res = run_sql('SELECT name, description, firerole_def_ser FROM accROLE WHERE id=%s', (role, ))
if acc_firerole_suggest_apache_p(deserialize(res[0][2])):
ret.append((res[0][0], res[0][1]))
return ret
def _format_list_of_apache_firerole(roles, referer):
"""Given a list of tuples (role, description) (returned by make_list_apache_firerole), and a referer url, returns a nice string for
presenting urls that let the user login with Apache password through
Firerole.
This function is needed only at CERN for aiding in the migration of
Apache Passwords restricted collections to FireRole roles.
Please use it with care."""
out = ""
if roles:
out += "1) Here is a list of administrative roles you may have " \
"received authorization for via an Apache password. If you are aware " \
"of such a password, please follow the corresponding link:"
out += "
"
for name, description in roles:
out += ""
out += "| %s | - %s | " % \
('%s%s' % (CFG_SITE_SECURE_URL, make_canonical_urlargd({'realm' : name, 'referer' : referer}, {})), name, description)
out += "
"
out += "
"
out += ""
return out
def make_apache_message(name_action, arguments, referer=None):
"""Given an action name and a dictionary of arguments and a refere url
it returns a a nice string for presenting urls that let the user login
with Apache password through Firerole authorized roles.
This function is needed only at CERN for aiding in the migration of
Apache Passwords restricted collections to FireRole roles.
Please use it with care."""
if not referer:
referer = '%s/youraccount/youradminactivities' % CFG_SITE_SECURE_URL
roles = make_list_apache_firerole(name_action, arguments)
if roles:
return _format_list_of_apache_firerole(roles, referer)
else:
return ""
def acc_authorize_action(req, name_action, authorized_if_no_roles=False, **arguments):
"""
Given the request object (or the user_info dictionary, or the uid), checks
if the user is allowed to run name_action with the given parameters.
If authorized_if_no_roles is True and no role exists (different
than superadmin) that are authorized to execute the given action, the
authorization will be granted.
Returns (0, msg) when the authorization is granted, (1, msg) when it's not.
"""
user_info = collect_user_info(req)
roles = acc_find_possible_roles(name_action, always_add_superadmin=False, **arguments)
for id_role in roles:
if acc_is_user_in_role(user_info, id_role):
## User belong to at least one authorized role.
return (0, CFG_WEBACCESS_WARNING_MSGS[0])
if acc_is_user_in_role(user_info, CFG_SUPERADMINROLE_ID):
## User is SUPERADMIN
return (0, CFG_WEBACCESS_WARNING_MSGS[0])
if not roles:
## No role is authorized for the given action/arguments
if authorized_if_no_roles:
## User is authorized because no authorization exists for the given
## action/arguments
return (0, CFG_WEBACCESS_WARNING_MSGS[0])
else:
## User is not authorized.
return (20, CFG_WEBACCESS_WARNING_MSGS[20] % cgi.escape(name_action))
## User is not authorized
return (1, "%s %s %s" % (CFG_WEBACCESS_WARNING_MSGS[1], (CFG_CALLED_FROM_APACHE and "%s %s" % (CFG_WEBACCESS_MSGS[0] % quote(user_info['uri']), CFG_WEBACCESS_MSGS[1]) or ""), make_apache_message(name_action, arguments, user_info['uri'])))
def acc_get_authorized_emails(name_action, **arguments):
"""
Given the action and its arguments, try to retireve all the matching
email addresses of users authorized.
This is a best effort operation, because if a role is authorized and
happens to be defined using a FireRole rule based on regular expression
or on IP addresses, non every email might be returned.
@param name_action: the name of the action.
@type name_action: string
@param arguments: the arguments to the action.
@return: the list of authorized emails.
@rtype: set of string
"""
authorized_emails = set()
roles = acc_find_possible_roles(name_action, always_add_superadmin=False, **arguments)
for id_role in roles:
for dummy1, email, dummy2 in acc_get_role_users(id_role):
authorized_emails.add(email.lower().strip())
firerole = load_role_definition(id_role)
authorized_emails.union(acc_firerole_extract_emails(firerole))
return authorized_emails
diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py
index 9095b3fe4..f29820353 100644
--- a/modules/websearch/lib/search_engine.py
+++ b/modules/websearch/lib/search_engine.py
@@ -1,5229 +1,5220 @@
# -*- coding: utf-8 -*-
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable=C0301
"""CDS Invenio Search Engine in mod_python."""
__lastupdated__ = """$Date$"""
__revision__ = "$Id$"
## import general modules:
import cgi
import cStringIO
import copy
import string
import os
import re
import time
import urllib
import urlparse
import zlib
import sys
if sys.hexversion < 0x2040000:
# pylint: disable=W0622
from sets import Set as set
# pylint: enable=W0622
## import CDS Invenio stuff:
from invenio.config import \
CFG_CERN_SITE, \
CFG_INSPIRE_SITE, \
CFG_OAI_ID_FIELD, \
CFG_WEBCOMMENT_ALLOW_REVIEWS, \
CFG_WEBSEARCH_CALL_BIBFORMAT, \
CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX, \
CFG_WEBSEARCH_FIELDS_CONVERT, \
CFG_WEBSEARCH_NB_RECORDS_TO_SORT, \
CFG_WEBSEARCH_SEARCH_CACHE_SIZE, \
CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS, \
CFG_WEBSEARCH_USE_ALEPH_SYSNOS, \
CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS, \
CFG_WEBSEARCH_FULLTEXT_SNIPPETS, \
CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE, \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG, \
CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_LOGDIR, \
CFG_BIBFORMAT_HIDDEN_TAGS, \
CFG_SITE_URL, \
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS
from invenio.search_engine_config import InvenioWebSearchUnknownCollectionError
from invenio.bibrecord import create_record, record_get_field_instances
from invenio.bibrank_record_sorter import get_bibrank_methods, rank_records, is_method_valid
from invenio.bibrank_downloads_similarity import register_page_view_event, calculate_reading_similarity_list
from invenio.bibindex_engine_stemmer import stem
from invenio.bibindex_engine_tokenizer import wash_author_name, author_name_requires_phrase_search
from invenio.bibformat import format_record, format_records, get_output_format_content_type, create_excel
from invenio.bibformat_config import CFG_BIBFORMAT_USE_OLD_BIBFORMAT
from invenio.bibrank_downloads_grapher import create_download_history_graph_and_box
from invenio.data_cacher import DataCacher
from invenio.websearch_external_collections import print_external_results_overview, perform_external_collection_search
from invenio.access_control_admin import acc_get_action_id
from invenio.access_control_config import VIEWRESTRCOLL, \
CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS
from invenio.websearchadminlib import get_detailed_page_tabs
from invenio.intbitset import intbitset as HitSet
from invenio.dbquery import DatabaseError, deserialize_via_marshal
from invenio.access_control_engine import acc_authorize_action
from invenio.errorlib import register_exception
from invenio.textutils import encode_for_xml
import invenio.template
webstyle_templates = invenio.template.load('webstyle')
webcomment_templates = invenio.template.load('webcomment')
from invenio.bibrank_citation_searcher import get_cited_by_count, calculate_cited_by_list, \
calculate_co_cited_with_list, get_records_with_num_cites, get_self_cited_by, \
get_refersto_hitset, get_citedby_hitset
from invenio.bibrank_citation_grapher import create_citation_history_graph_and_box
-from invenio.dbquery import run_sql, run_sql_cached, get_table_update_time
+from invenio.dbquery import run_sql, get_table_update_time
from invenio.webuser import getUid, collect_user_info
from invenio.webpage import pageheaderonly, pagefooteronly, create_error_box
from invenio.messages import gettext_set_language
from invenio.search_engine_query_parser import SearchQueryParenthesisedParser, \
SpiresToInvenioSyntaxConverter
from invenio import webinterface_handler_config as apache
try:
import invenio.template
websearch_templates = invenio.template.load('websearch')
except:
pass
from invenio.websearch_external_collections import calculate_hosted_collections_results, do_calculate_hosted_collections_results
from invenio.websearch_external_collections_config import CFG_HOSTED_COLLECTION_TIMEOUT_ANTE_SEARCH
from invenio.websearch_external_collections_config import CFG_HOSTED_COLLECTION_TIMEOUT_POST_SEARCH
from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_MAXRESULTS
VIEWRESTRCOLL_ID = acc_get_action_id(VIEWRESTRCOLL)
## global vars:
cfg_nb_browse_seen_records = 100 # limit of the number of records to check when browsing certain collection
cfg_nicely_ordered_collection_list = 0 # do we propose collection list nicely ordered or alphabetical?
## precompile some often-used regexp for speed reasons:
re_word = re.compile('[\s]')
re_quotes = re.compile('[\'\"]')
re_doublequote = re.compile('\"')
re_equal = re.compile('\=')
re_logical_and = re.compile('\sand\s', re.I)
re_logical_or = re.compile('\sor\s', re.I)
re_logical_not = re.compile('\snot\s', re.I)
re_operators = re.compile(r'\s([\+\-\|])\s')
re_pattern_wildcards_after_spaces = re.compile(r'(\s)[\*\%]+')
re_pattern_single_quotes = re.compile("'(.*?)'")
re_pattern_double_quotes = re.compile("\"(.*?)\"")
re_pattern_regexp_quotes = re.compile("\/(.*?)\/")
re_pattern_short_words = re.compile(r'([\s\"]\w{1,3})[\*\%]+')
re_pattern_space = re.compile("__SPACE__")
re_pattern_today = re.compile("\$TODAY\$")
re_pattern_parens = re.compile(r'\([^\)]+\s+[^\)]+\)')
re_unicode_lowercase_a = re.compile(unicode(r"(?u)[áàäâãå]", "utf-8"))
re_unicode_lowercase_ae = re.compile(unicode(r"(?u)[æ]", "utf-8"))
re_unicode_lowercase_e = re.compile(unicode(r"(?u)[éèëê]", "utf-8"))
re_unicode_lowercase_i = re.compile(unicode(r"(?u)[íìïî]", "utf-8"))
re_unicode_lowercase_o = re.compile(unicode(r"(?u)[óòöôõø]", "utf-8"))
re_unicode_lowercase_u = re.compile(unicode(r"(?u)[úùüû]", "utf-8"))
re_unicode_lowercase_y = re.compile(unicode(r"(?u)[ýÿ]", "utf-8"))
re_unicode_lowercase_c = re.compile(unicode(r"(?u)[çć]", "utf-8"))
re_unicode_lowercase_n = re.compile(unicode(r"(?u)[ñ]", "utf-8"))
re_unicode_uppercase_a = re.compile(unicode(r"(?u)[ÁÀÄÂÃÅ]", "utf-8"))
re_unicode_uppercase_ae = re.compile(unicode(r"(?u)[Æ]", "utf-8"))
re_unicode_uppercase_e = re.compile(unicode(r"(?u)[ÉÈËÊ]", "utf-8"))
re_unicode_uppercase_i = re.compile(unicode(r"(?u)[ÍÌÏÎ]", "utf-8"))
re_unicode_uppercase_o = re.compile(unicode(r"(?u)[ÓÒÖÔÕØ]", "utf-8"))
re_unicode_uppercase_u = re.compile(unicode(r"(?u)[ÚÙÜÛ]", "utf-8"))
re_unicode_uppercase_y = re.compile(unicode(r"(?u)[Ý]", "utf-8"))
re_unicode_uppercase_c = re.compile(unicode(r"(?u)[ÇĆ]", "utf-8"))
re_unicode_uppercase_n = re.compile(unicode(r"(?u)[Ñ]", "utf-8"))
re_latex_lowercase_a = re.compile("\\\\[\"H'`~^vu=k]\{?a\}?")
re_latex_lowercase_ae = re.compile("\\\\ae\\{\\}?")
re_latex_lowercase_e = re.compile("\\\\[\"H'`~^vu=k]\\{?e\\}?")
re_latex_lowercase_i = re.compile("\\\\[\"H'`~^vu=k]\\{?i\\}?")
re_latex_lowercase_o = re.compile("\\\\[\"H'`~^vu=k]\\{?o\\}?")
re_latex_lowercase_u = re.compile("\\\\[\"H'`~^vu=k]\\{?u\\}?")
re_latex_lowercase_y = re.compile("\\\\[\"']\\{?y\\}?")
re_latex_lowercase_c = re.compile("\\\\['uc]\\{?c\\}?")
re_latex_lowercase_n = re.compile("\\\\[c'~^vu]\\{?n\\}?")
re_latex_uppercase_a = re.compile("\\\\[\"H'`~^vu=k]\\{?A\\}?")
re_latex_uppercase_ae = re.compile("\\\\AE\\{?\\}?")
re_latex_uppercase_e = re.compile("\\\\[\"H'`~^vu=k]\\{?E\\}?")
re_latex_uppercase_i = re.compile("\\\\[\"H'`~^vu=k]\\{?I\\}?")
re_latex_uppercase_o = re.compile("\\\\[\"H'`~^vu=k]\\{?O\\}?")
re_latex_uppercase_u = re.compile("\\\\[\"H'`~^vu=k]\\{?U\\}?")
re_latex_uppercase_y = re.compile("\\\\[\"']\\{?Y\\}?")
re_latex_uppercase_c = re.compile("\\\\['uc]\\{?C\\}?")
re_latex_uppercase_n = re.compile("\\\\[c'~^vu]\\{?N\\}?")
class RestrictedCollectionDataCacher(DataCacher):
def __init__(self):
def cache_filler():
ret = []
try:
res = run_sql("""SELECT DISTINCT ar.value
FROM accROLE_accACTION_accARGUMENT raa JOIN accARGUMENT ar ON raa.id_accARGUMENT = ar.id
WHERE ar.keyword = 'collection' AND raa.id_accACTION = %s""", (VIEWRESTRCOLL_ID,))
except Exception:
# database problems, return empty cache
return []
for coll in res:
ret.append(coll[0])
return ret
def timestamp_verifier():
return max(get_table_update_time('accROLE_accACTION_accARGUMENT'), get_table_update_time('accARGUMENT'))
DataCacher.__init__(self, cache_filler, timestamp_verifier)
def collection_restricted_p(collection):
restricted_collection_cache.recreate_cache_if_needed()
return collection in restricted_collection_cache.cache
try:
restricted_collection_cache.is_ok_p
except Exception:
restricted_collection_cache = RestrictedCollectionDataCacher()
def get_permitted_restricted_collections(user_info):
"""Return a list of collection that are restricted but for which the user
is authorized."""
restricted_collection_cache.recreate_cache_if_needed()
ret = []
for collection in restricted_collection_cache.cache:
if acc_authorize_action(user_info, 'viewrestrcoll', collection=collection)[0] == 0:
ret.append(collection)
return ret
def get_restricted_collections_for_recid(recid):
"""
Return the list of restricted collection names to which recid belongs.
"""
restricted_collections = run_sql("""SELECT c.name, c.reclist FROM accROLE_accACTION_accARGUMENT raa JOIN accARGUMENT ar ON raa.id_accARGUMENT = ar.id JOIN collection c ON ar.value=c.name WHERE ar.keyword = 'collection' AND raa.id_accACTION = %s""", (VIEWRESTRCOLL_ID,))
return [row[0] for row in restricted_collections if recid in HitSet(row[1])]
def is_user_owner_of_record(user_info, recid):
"""
Check if the user is owner of the record, i.e. he is the submitter
and/or belongs to a owner-like group authorized to 'see' the record.
@param user_info: the user_info dictionary that describe the user.
@type user_info: user_info dictionary
@param recid: the record identifier.
@type recid: positive integer
@return: True if the user is 'owner' of the record; False otherwise
@rtype: bool
"""
authorized_emails_or_group = []
for tag in CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS:
authorized_emails_or_group.extend(get_fieldvalues(recid, tag))
for email_or_group in authorized_emails_or_group:
if email_or_group in user_info['group']:
return True
email = email_or_group.strip().lower()
if user_info['email'].strip().lower() == email:
return True
return False
def check_user_can_view_record(user_info, recid):
"""
Check if the user is authorized to view the given recid. The function
grants access in two cases: either user has author rights on this
record, or he has view rights to the primary collection this record
belongs to.
@param user_info: the user_info dictionary that describe the user.
@type user_info: user_info dictionary
@param recid: the record identifier.
@type recid: positive integer
@return: (0, ''), when authorization is granted, (>0, 'message') when
authorization is not granted
@rtype: (int, string)
"""
restricted_collections = get_restricted_collections_for_recid(recid)
if not restricted_collections or is_user_owner_of_record(user_info, recid):
return (0, '')
for collection in restricted_collections:
(auth_code, auth_msg) = acc_authorize_action(user_info, VIEWRESTRCOLL, collection=collection)
if auth_code == 0:
continue
else:
return (auth_code, auth_msg)
return (0, '')
class IndexStemmingDataCacher(DataCacher):
"""
Provides cache for stemming information for word/phrase indexes.
This class is not to be used directly; use function
get_index_stemming_language() instead.
"""
def __init__(self):
def cache_filler():
try:
res = run_sql("""SELECT id, stemming_language FROM idxINDEX""")
except DatabaseError:
# database problems, return empty cache
return {}
return dict(res)
def timestamp_verifier():
return get_table_update_time('idxINDEX')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
try:
index_stemming_cache.is_ok_p
except Exception:
index_stemming_cache = IndexStemmingDataCacher()
def get_index_stemming_language(index_id):
"""Return stemming langugage for given index."""
index_stemming_cache.recreate_cache_if_needed()
return index_stemming_cache.cache[index_id]
class CollectionRecListDataCacher(DataCacher):
"""
Provides cache for collection reclist hitsets. This class is not
to be used directly; use function get_collection_reclist() instead.
"""
def __init__(self):
def cache_filler():
ret = {}
try:
res = run_sql("SELECT name,reclist FROM collection")
except Exception:
# database problems, return empty cache
return {}
for name, reclist in res:
ret[name] = None # this will be filled later during runtime by calling get_collection_reclist(coll)
return ret
def timestamp_verifier():
return get_table_update_time('collection')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
try:
if not collection_reclist_cache.is_ok_p:
raise Exception
except Exception:
collection_reclist_cache = CollectionRecListDataCacher()
def get_collection_reclist(coll):
"""Return hitset of recIDs that belong to the collection 'coll'."""
collection_reclist_cache.recreate_cache_if_needed()
if not collection_reclist_cache.cache[coll]:
# not yet it the cache, so calculate it and fill the cache:
set = HitSet()
query = "SELECT nbrecs,reclist FROM collection WHERE name=%s"
res = run_sql(query, (coll, ), 1)
if res:
try:
set = HitSet(res[0][1])
except:
pass
collection_reclist_cache.cache[coll] = set
# finally, return reclist:
return collection_reclist_cache.cache[coll]
class SearchResultsCache(DataCacher):
"""
Provides temporary lazy cache for Search Results.
Useful when users click on `next page'.
"""
def __init__(self):
def cache_filler():
return {}
def timestamp_verifier():
return '1970-01-01 00:00:00' # lazy cache is always okay;
# its filling is governed by
# CFG_WEBSEARCH_SEARCH_CACHE_SIZE
DataCacher.__init__(self, cache_filler, timestamp_verifier)
try:
if not search_results_cache.is_ok_p:
raise Exception
except Exception:
search_results_cache = SearchResultsCache()
class CollectionI18nNameDataCacher(DataCacher):
"""
Provides cache for I18N collection names. This class is not to be
used directly; use function get_coll_i18nname() instead.
"""
def __init__(self):
def cache_filler():
ret = {}
try:
res = run_sql("SELECT c.name,cn.ln,cn.value FROM collectionname AS cn, collection AS c WHERE cn.id_collection=c.id AND cn.type='ln'") # ln=long name
except Exception:
# database problems
return {}
for c, ln, i18nname in res:
if i18nname:
if not ret.has_key(c):
ret[c] = {}
ret[c][ln] = i18nname
return ret
def timestamp_verifier():
return get_table_update_time('collectionname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
try:
if not collection_i18nname_cache.is_ok_p:
raise Exception
except Exception:
collection_i18nname_cache = CollectionI18nNameDataCacher()
def get_coll_i18nname(c, ln=CFG_SITE_LANG, verify_cache_timestamp=True):
"""
Return nicely formatted collection name (of the name type `ln'
(=long name)) for collection C in language LN.
This function uses collection_i18nname_cache, but it verifies
whether the cache is up-to-date first by default. This
verification step is performed by checking the DB table update
time. So, if you call this function 1000 times, it can get very
slow because it will do 1000 table update time verifications, even
though collection names change not that often.
Hence the parameter VERIFY_CACHE_TIMESTAMP which, when set to
False, will assume the cache is already up-to-date. This is
useful namely in the generation of collection lists for the search
results page.
"""
if verify_cache_timestamp:
collection_i18nname_cache.recreate_cache_if_needed()
out = c
try:
out = collection_i18nname_cache.cache[c][ln]
except KeyError:
pass # translation in LN does not exist
return out
class FieldI18nNameDataCacher(DataCacher):
"""
Provides cache for I18N field names. This class is not to be used
directly; use function get_field_i18nname() instead.
"""
def __init__(self):
def cache_filler():
ret = {}
try:
res = run_sql("SELECT f.name,fn.ln,fn.value FROM fieldname AS fn, field AS f WHERE fn.id_field=f.id AND fn.type='ln'") # ln=long name
except Exception:
# database problems, return empty cache
return {}
for f, ln, i18nname in res:
if i18nname:
if not ret.has_key(f):
ret[f] = {}
ret[f][ln] = i18nname
return ret
def timestamp_verifier():
return get_table_update_time('fieldname')
DataCacher.__init__(self, cache_filler, timestamp_verifier)
try:
if not field_i18nname_cache.is_ok_p:
raise Exception
except Exception:
field_i18nname_cache = FieldI18nNameDataCacher()
def get_field_i18nname(f, ln=CFG_SITE_LANG, verify_cache_timestamp=True):
"""
Return nicely formatted field name (of type 'ln', 'long name') for
field F in language LN.
If VERIFY_CACHE_TIMESTAMP is set to True, then verify DB timestamp
and field I18N name cache timestamp and refresh cache from the DB
if needed. Otherwise don't bother checking DB timestamp and
return the cached value. (This is useful when get_field_i18nname
is called inside a loop.)
"""
if verify_cache_timestamp:
field_i18nname_cache.recreate_cache_if_needed()
out = f
try:
out = field_i18nname_cache.cache[f][ln]
except KeyError:
pass # translation in LN does not exist
return out
def get_alphabetically_ordered_collection_list(level=0, ln=CFG_SITE_LANG):
"""Returns nicely ordered (score respected) list of collections, more exactly list of tuples
(collection name, printable collection name).
Suitable for create_search_box()."""
out = []
- res = run_sql_cached("SELECT id,name FROM collection ORDER BY name ASC",
- affected_tables=['collection',])
+ res = run_sql("SELECT id,name FROM collection ORDER BY name ASC")
for c_id, c_name in res:
# make a nice printable name (e.g. truncate c_printable for
# long collection names in given language):
c_printable_fullname = get_coll_i18nname(c_name, ln, False)
c_printable = wash_index_term(c_printable_fullname, 30, False)
if c_printable != c_printable_fullname:
c_printable = c_printable + "..."
if level:
c_printable = " " + level * '-' + " " + c_printable
out.append([c_name, c_printable])
return out
def get_nicely_ordered_collection_list(collid=1, level=0, ln=CFG_SITE_LANG):
"""Returns nicely ordered (score respected) list of collections, more exactly list of tuples
(collection name, printable collection name).
Suitable for create_search_box()."""
colls_nicely_ordered = []
res = run_sql("""SELECT c.name,cc.id_son FROM collection_collection AS cc, collection AS c
WHERE c.id=cc.id_son AND cc.id_dad=%s ORDER BY score DESC""", (collid, ))
for c, cid in res:
# make a nice printable name (e.g. truncate c_printable for
# long collection names in given language):
c_printable_fullname = get_coll_i18nname(c, ln, False)
c_printable = wash_index_term(c_printable_fullname, 30, False)
if c_printable != c_printable_fullname:
c_printable = c_printable + "..."
if level:
c_printable = " " + level * '-' + " " + c_printable
colls_nicely_ordered.append([c, c_printable])
colls_nicely_ordered = colls_nicely_ordered + get_nicely_ordered_collection_list(cid, level+1, ln=ln)
return colls_nicely_ordered
def get_index_id_from_field(field):
"""
Return index id with name corresponding to FIELD, or the first
index id where the logical field code named FIELD is indexed.
Return zero in case there is no index defined for this field.
Example: field='author', output=4.
"""
out = 0
if field == '':
field = 'global' # empty string field means 'global' index (field 'anyfield')
# first look in the index table:
res = run_sql("""SELECT id FROM idxINDEX WHERE name=%s""", (field,))
if res:
out = res[0][0]
return out
# not found in the index table, now look in the field table:
res = run_sql("""SELECT w.id FROM idxINDEX AS w, idxINDEX_field AS wf, field AS f
WHERE f.code=%s AND wf.id_field=f.id AND w.id=wf.id_idxINDEX
LIMIT 1""", (field,))
if res:
out = res[0][0]
return out
def get_words_from_pattern(pattern):
"Returns list of whitespace-separated words from pattern."
words = {}
for word in string.split(pattern):
if not words.has_key(word):
words[word] = 1
return words.keys()
def create_basic_search_units(req, p, f, m=None, of='hb'):
"""Splits search pattern and search field into a list of independently searchable units.
- A search unit consists of '(operator, pattern, field, type, hitset)' tuples where
'operator' is set union (|), set intersection (+) or set exclusion (-);
'pattern' is either a word (e.g. muon*) or a phrase (e.g. 'nuclear physics');
'field' is either a code like 'title' or MARC tag like '100__a';
'type' is the search type ('w' for word file search, 'a' for access file search).
- Optionally, the function accepts the match type argument 'm'.
If it is set (e.g. from advanced search interface), then it
performs this kind of matching. If it is not set, then a guess is made.
'm' can have values: 'a'='all of the words', 'o'='any of the words',
'p'='phrase/substring', 'r'='regular expression',
'e'='exact value'.
- Warnings are printed on req (when not None) in case of HTML output formats."""
opfts = [] # will hold (o,p,f,t,h) units
# FIXME: quick hack for the journal index
if f == 'journal':
opfts.append(['+', p, f, 'w'])
return opfts
## check arguments: is desired matching type set?
if m:
## A - matching type is known; good!
if m == 'e':
# A1 - exact value:
opfts.append(['+', p, f, 'a']) # '+' since we have only one unit
elif m == 'p':
# A2 - phrase/substring:
opfts.append(['+', "%" + p + "%", f, 'a']) # '+' since we have only one unit
elif m == 'r':
# A3 - regular expression:
opfts.append(['+', p, f, 'r']) # '+' since we have only one unit
elif m == 'a' or m == 'w':
# A4 - all of the words:
p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed
for word in get_words_from_pattern(p):
opfts.append(['+', word, f, 'w']) # '+' in all units
elif m == 'o':
# A5 - any of the words:
p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed
for word in get_words_from_pattern(p):
if len(opfts)==0:
opfts.append(['+', word, f, 'w']) # '+' in the first unit
else:
opfts.append(['|', word, f, 'w']) # '|' in further units
else:
if of.startswith("h"):
print_warning(req, "Matching type '%s' is not implemented yet." % cgi.escape(m), "Warning")
opfts.append(['+', "%" + p + "%", f, 'w'])
else:
## B - matching type is not known: let us try to determine it by some heuristics
if f and p[0] == '"' and p[-1] == '"':
## B0 - does 'p' start and end by double quote, and is 'f' defined? => doing ACC search
opfts.append(['+', p[1:-1], f, 'a'])
elif (f == 'author' or f == 'exactauthor') and author_name_requires_phrase_search(p):
## B1 - do we search in author, and does 'p' contain space/comma/dot/etc?
## => doing washed ACC search
opfts.append(['+', p, f, 'a'])
elif f and p[0] == "'" and p[-1] == "'":
## B0bis - does 'p' start and end by single quote, and is 'f' defined? => doing ACC search
opfts.append(['+', '%' + p[1:-1] + '%', f, 'a'])
elif f and p[0] == "/" and p[-1] == "/":
## B0ter - does 'p' start and end by a slash, and is 'f' defined? => doing regexp search
opfts.append(['+', p[1:-1], f, 'r'])
elif f and string.find(p, ',') >= 0:
## B1 - does 'p' contain comma, and is 'f' defined? => doing ACC search
opfts.append(['+', p, f, 'a'])
elif f and str(f[0:2]).isdigit():
## B2 - does 'f' exist and starts by two digits? => doing ACC search
opfts.append(['+', p, f, 'a'])
else:
## B3 - doing WRD search, but maybe ACC too
# search units are separated by spaces unless the space is within single or double quotes
# so, let us replace temporarily any space within quotes by '__SPACE__'
p = re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", p)
p = re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", p)
p = re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", p)
# wash argument:
p = re_equal.sub(":", p)
p = re_logical_and.sub(" ", p)
p = re_logical_or.sub(" |", p)
p = re_logical_not.sub(" -", p)
p = re_operators.sub(r' \1', p)
for pi in string.split(p): # iterate through separated units (or items, as "pi" stands for "p item")
pi = re_pattern_space.sub(" ", pi) # replace back '__SPACE__' by ' '
# firstly, determine set operator
if pi[0] == '+' or pi[0] == '-' or pi[0] == '|':
oi = pi[0]
pi = pi[1:]
else:
# okay, there is no operator, so let us decide what to do by default
oi = '+' # by default we are doing set intersection...
# secondly, determine search pattern and field:
if string.find(pi, ":") > 0:
fi, pi = string.split(pi, ":", 1)
fi = wash_field(fi)
# test whether fi is a real index code or a MARC-tag defined code:
if fi in get_fieldcodes() or '00' <= fi[:2] <= '99':
pass
else:
# it is not, so join it back:
fi, pi = f, fi + ":" + pi
else:
fi, pi = f, pi
# wash 'fi' argument:
fi = wash_field(fi)
# wash 'pi' argument:
if re_quotes.match(pi):
# B3a - quotes are found => do ACC search (phrase search)
if pi[0] == '"' and pi[-1] == '"':
pi = string.replace(pi, '"', '') # remove quote signs
opfts.append([oi, pi, fi, 'a'])
elif pi[0] == "'" and pi[-1] == "'":
pi = string.replace(pi, "'", "") # remove quote signs
opfts.append([oi, "%" + pi + "%", fi, 'a'])
else: # unbalanced quotes, so fall back to WRD query:
opfts.append([oi, pi, fi, 'w'])
elif fi and str(fi[0]).isdigit() and str(fi[0]).isdigit():
# B3b - fi exists and starts by two digits => do ACC search
opfts.append([oi, pi, fi, 'a'])
elif fi and not get_index_id_from_field(fi) and get_field_name(fi):
# B3c - logical field fi exists but there is no WRD index for fi => try ACC search
opfts.append([oi, pi, fi, 'a'])
elif pi.startswith('/') and pi.endswith('/'):
# B3d - pi has slashes around => do regexp search
opfts.append([oi, pi[1:-1], fi, 'r'])
else:
# B3e - general case => do WRD search
pi = strip_accents(pi) # strip accents for 'w' mode, FIXME: delete when not needed
for pii in get_words_from_pattern(pi):
opfts.append([oi, pii, fi, 'w'])
## sanity check:
for i in range(0, len(opfts)):
try:
pi = opfts[i][1]
if pi == '*':
if of.startswith("h"):
print_warning(req, "Ignoring standalone wildcard word.", "Warning")
del opfts[i]
if pi == '' or pi == ' ':
fi = opfts[i][2]
if fi:
if of.startswith("h"):
print_warning(req, "Ignoring empty %s search term." % fi, "Warning")
del opfts[i]
except:
pass
## replace old logical field names if applicable:
if CFG_WEBSEARCH_FIELDS_CONVERT:
opfts = [[o,p,wash_field(f),t] for o,p,f,t in opfts]
## return search units:
return opfts
def page_start(req, of, cc, aas, ln, uid, title_message=None,
description='', keywords='', recID=-1, tab='', p=''):
"Start page according to given output format."
_ = gettext_set_language(ln)
if not req or isinstance(req, cStringIO.OutputType):
return # we were called from CLI
if not title_message:
title_message = _("Search Results")
content_type = get_output_format_content_type(of)
if of.startswith('x'):
if of == 'xr':
# we are doing RSS output
req.content_type = "application/rss+xml"
req.send_http_header()
req.write("""\n""")
else:
# we are doing XML output:
req.content_type = "text/xml"
req.send_http_header()
req.write("""\n""")
elif of.startswith('t') or str(of[0:3]).isdigit():
# we are doing plain text output:
req.content_type = "text/plain"
req.send_http_header()
elif of == "id":
pass # nothing to do, we shall only return list of recIDs
elif content_type == 'text/html':
# we are doing HTML output:
req.content_type = "text/html"
req.send_http_header()
if not description:
description = "%s %s." % (cc, _("Search Results"))
if not keywords:
keywords = "%s, WebSearch, %s" % (get_coll_i18nname(CFG_SITE_NAME, ln, False), get_coll_i18nname(cc, ln, False))
## generate RSS URL:
argd = {}
if req.args:
argd = cgi.parse_qs(req.args)
rssurl = websearch_templates.build_rss_url(argd)
## add jsmath if displaying single records (FIXME: find
## eventual better place to this code)
if of.lower() in CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS:
metaheaderadd = """
"""
else:
metaheaderadd = ''
## generate navtrail:
navtrail = create_navtrail_links(cc, aas, ln)
if navtrail != '':
navtrail += ' > '
if (tab != '' or ((of != '' or of.lower() != 'hd') and of != 'hb')) and \
recID != -1:
# If we are not in information tab in HD format, customize
# the nav. trail to have a link back to main record. (Due
# to the way perform_request_search() works, hb
# (lowercase) is equal to hd)
navtrail += ' %s' % \
(CFG_SITE_URL, recID, title_message)
if (of != '' or of.lower() != 'hd') and of != 'hb':
# Export
format_name = of
query = "SELECT name FROM format WHERE code=%s"
res = run_sql(query, (of,))
if res:
format_name = res[0][0]
navtrail += ' > ' + format_name
else:
# Discussion, citations, etc. tabs
tab_label = get_detailed_page_tabs(cc, ln=ln)[tab]['label']
navtrail += ' > ' + _(tab_label)
else:
navtrail += title_message
if p:
# we are serving search/browse results pages, so insert pattern:
navtrail += ": " + cgi.escape(p)
title_message = cgi.escape(p) + " - " + title_message
## finally, print page header:
req.write(pageheaderonly(req=req, title=title_message,
navtrail=navtrail,
description=description,
keywords=keywords,
metaheaderadd=metaheaderadd,
uid=uid,
language=ln,
navmenuid='search',
navtrail_append_title_p=0,
rssurl=rssurl))
req.write(websearch_templates.tmpl_search_pagestart(ln=ln))
#else:
# req.send_http_header()
def page_end(req, of="hb", ln=CFG_SITE_LANG):
"End page according to given output format: e.g. close XML tags, add HTML footer, etc."
if of == "id":
return [] # empty recID list
if not req:
return # we were called from CLI
if of.startswith('h'):
req.write(websearch_templates.tmpl_search_pageend(ln = ln)) # pagebody end
req.write(pagefooteronly(lastupdated=__lastupdated__, language=ln, req=req))
return
def create_page_title_search_pattern_info(p, p1, p2, p3):
"""Create the search pattern bit for the page web page
HTML header. Basically combine p and (p1,p2,p3) together so that
the page header may be filled whether we are in the Simple Search
or Advanced Search interface contexts."""
out = ""
if p:
out = p
else:
out = p1
if p2:
out += ' ' + p2
if p3:
out += ' ' + p3
return out
def create_inputdate_box(name="d1", selected_year=0, selected_month=0, selected_day=0, ln=CFG_SITE_LANG):
"Produces 'From Date', 'Until Date' kind of selection box. Suitable for search options."
_ = gettext_set_language(ln)
box = ""
# day
box += """<select name="%sd">""" % name
box += """<option value="">%s""" % _("any day")
for day in range(1, 32):
box += """<option value="%02d"%s>%02d""" % (day, is_selected(day, selected_day), day)
box += """</select>"""
# month
box += """<select name="%sm">""" % name
box += """<option value="">%s""" % _("any month")
for mm, month in [(1, _("January")), (2, _("February")), (3, _("March")), (4, _("April")), \
(5, _("May")), (6, _("June")), (7, _("July")), (8, _("August")), \
(9, _("September")), (10, _("October")), (11, _("November")), (12, _("December"))]:
box += """<option value="%02d"%s>%s""" % (mm, is_selected(mm, selected_month), month)
box += """</select>"""
# year
box += """<select name="%sy">""" % name
box += """<option value="">%s""" % _("any year")
this_year = int(time.strftime("%Y", time.localtime()))
for year in range(this_year-20, this_year+1):
box += """<option value="%d"%s>%d""" % (year, is_selected(year, selected_year), year)
box += """</select>"""
return box
def create_search_box(cc, colls, p, f, rg, sf, so, sp, rm, of, ot, aas,
ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3,
m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec,
action=""):
"""Create search box for 'search again in the results page' functionality."""
# load the right message language
_ = gettext_set_language(ln)
# some computations
cc_intl = get_coll_i18nname(cc, ln, False)
cc_colID = get_colID(cc)
colls_nicely_ordered = []
if cfg_nicely_ordered_collection_list:
colls_nicely_ordered = get_nicely_ordered_collection_list(ln=ln)
else:
colls_nicely_ordered = get_alphabetically_ordered_collection_list(ln=ln)
colls_nice = []
for (cx, cx_printable) in colls_nicely_ordered:
if not cx.startswith("Unnamed collection"):
colls_nice.append({ 'value' : cx,
'text' : cx_printable
})
coll_selects = []
if colls and colls[0] != CFG_SITE_NAME:
# some collections are defined, so print these first, and only then print 'add another collection' heading:
for c in colls:
if c:
temp = []
temp.append({ 'value' : CFG_SITE_NAME,
'text' : '*** %s ***' % _("any public collection")
})
# this field is used to remove the current collection from the ones to be searched.
temp.append({ 'value' : '',
'text' : '*** %s ***' % _("remove this collection")
})
for val in colls_nice:
# print collection:
if not cx.startswith("Unnamed collection"):
temp.append({ 'value' : val['value'],
'text' : val['text'],
'selected' : (c == re.sub("^[\s\-]*","", val['value']))
})
coll_selects.append(temp)
coll_selects.append([{ 'value' : '',
'text' : '*** %s ***' % _("add another collection")
}] + colls_nice)
else: # we searched in CFG_SITE_NAME, so print 'any public collection' heading
coll_selects.append([{ 'value' : CFG_SITE_NAME,
'text' : '*** %s ***' % _("any public collection")
}] + colls_nice)
## ranking methods
ranks = [{
'value' : '',
'text' : "- %s %s -" % (_("OR").lower (), _("rank by")),
}]
for (code, name) in get_bibrank_methods(cc_colID, ln):
# propose found rank methods:
ranks.append({
'value' : code,
'text' : name,
})
formats = []
query = """SELECT code,name FROM format WHERE visibility='1' ORDER BY name ASC"""
res = run_sql(query)
if res:
# propose found formats:
for code, name in res:
formats.append({ 'value' : code,
'text' : name
})
else:
formats.append({'value' : 'hb',
'text' : _("HTML brief")
})
# show collections in the search box? (not if there is only one
# collection defined, and not if we are in light search)
show_colls = True
show_title = True
if len(collection_reclist_cache.cache.keys()) == 1 or \
aas == -1:
show_colls = False
show_title = False
if cc == CFG_SITE_NAME:
show_title = False
return websearch_templates.tmpl_search_box(
ln = ln,
aas = aas,
cc_intl = cc_intl,
cc = cc,
ot = ot,
sp = sp,
action = action,
fieldslist = get_searchwithin_fields(ln=ln, colID=cc_colID),
f1 = f1,
f2 = f2,
f3 = f3,
m1 = m1,
m2 = m2,
m3 = m3,
p1 = p1,
p2 = p2,
p3 = p3,
op1 = op1,
op2 = op2,
rm = rm,
p = p,
f = f,
coll_selects = coll_selects,
d1y = d1y, d2y = d2y, d1m = d1m, d2m = d2m, d1d = d1d, d2d = d2d,
dt = dt,
sort_fields = get_sortby_fields(ln=ln, colID=cc_colID),
sf = sf,
so = so,
ranks = ranks,
sc = sc,
rg = rg,
formats = formats,
of = of,
pl = pl,
jrec = jrec,
ec = ec,
show_colls = show_colls,
show_title = show_title,
)
def create_navtrail_links(cc=CFG_SITE_NAME, aas=0, ln=CFG_SITE_LANG, self_p=1, tab=''):
"""Creates navigation trail links, i.e. links to collection
ancestors (except Home collection). If aas==1, then links to
Advanced Search interfaces; otherwise Simple Search.
"""
dads = []
for dad in get_coll_ancestors(cc):
if dad != CFG_SITE_NAME: # exclude Home collection
dads.append ((dad, get_coll_i18nname(dad, ln, False)))
if self_p and cc != CFG_SITE_NAME:
dads.append((cc, get_coll_i18nname(cc, ln, False)))
return websearch_templates.tmpl_navtrail_links(
aas=aas, ln=ln, dads=dads)
def get_searchwithin_fields(ln='en', colID=None):
"""Retrieves the fields name used in the 'search within' selection box for the collection ID colID."""
res = None
if colID:
- res = run_sql_cached("""SELECT f.code,f.name FROM field AS f, collection_field_fieldvalue AS cff
+ res = run_sql("""SELECT f.code,f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE cff.type='sew' AND cff.id_collection=%s AND cff.id_field=f.id
- ORDER BY cff.score DESC, f.name ASC""", (colID,),
- affected_tables=['field', 'collection_field_fieldvalue'])
+ ORDER BY cff.score DESC, f.name ASC""", (colID,))
if not res:
- res = run_sql_cached("SELECT code,name FROM field ORDER BY name ASC",
- affected_tables=['field',])
+ res = run_sql("SELECT code,name FROM field ORDER BY name ASC")
fields = [{
'value' : '',
'text' : get_field_i18nname("any field", ln, False)
}]
for field_code, field_name in res:
if field_code and field_code != "anyfield":
fields.append({ 'value' : field_code,
'text' : get_field_i18nname(field_name, ln, False)
})
return fields
def get_sortby_fields(ln='en', colID=None):
"""Retrieves the fields name used in the 'sort by' selection box for the collection ID colID."""
_ = gettext_set_language(ln)
res = None
if colID:
- res = run_sql_cached("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
+ res = run_sql("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE cff.type='soo' AND cff.id_collection=%s AND cff.id_field=f.id
- ORDER BY cff.score DESC, f.name ASC""", (colID,),
- affected_tables=['field', 'collection_field_fieldvalue'])
+ ORDER BY cff.score DESC, f.name ASC""", (colID,))
if not res:
# no sort fields defined for this colID, try to take Home collection:
- res = run_sql_cached("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
+ res = run_sql("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE cff.type='soo' AND cff.id_collection=%s AND cff.id_field=f.id
- ORDER BY cff.score DESC, f.name ASC""", (1,),
- affected_tables=['field', 'collection_field_fieldvalue'])
+ ORDER BY cff.score DESC, f.name ASC""", (1,))
if not res:
# no sort fields defined for the Home collection, take all sort fields defined wherever they are:
- res = run_sql_cached("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
+ res = run_sql("""SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE cff.type='soo' AND cff.id_field=f.id
- ORDER BY cff.score DESC, f.name ASC""",
- affected_tables=['field', 'collection_field_fieldvalue'])
+ ORDER BY cff.score DESC, f.name ASC""",)
fields = [{
'value' : '',
'text' : _("latest first")
}]
for field_code, field_name in res:
if field_code and field_code != "anyfield":
fields.append({ 'value' : field_code,
'text' : get_field_i18nname(field_name, ln, False)
})
return fields
def create_andornot_box(name='op', value='', ln='en'):
"Returns HTML code for the AND/OR/NOT selection box."
_ = gettext_set_language(ln)
out = """
<select name="%s">
<option value="a"%s>%s
<option value="o"%s>%s
<option value="n"%s>%s
</select>
""" % (name,
is_selected('a', value), _("AND"),
is_selected('o', value), _("OR"),
is_selected('n', value), _("AND NOT"))
return out
def create_matchtype_box(name='m', value='', ln='en'):
"Returns HTML code for the 'match type' selection box."
_ = gettext_set_language(ln)
out = """
<select name="%s">
<option value="a"%s>%s
<option value="o"%s>%s
<option value="e"%s>%s
<option value="p"%s>%s
<option value="r"%s>%s
</select>
""" % (name,
is_selected('a', value), _("All of the words:"),
is_selected('o', value), _("Any of the words:"),
is_selected('e', value), _("Exact phrase:"),
is_selected('p', value), _("Partial phrase:"),
is_selected('r', value), _("Regular expression:"))
return out
def is_selected(var, fld):
"Checks if the two are equal, and if yes, returns ' selected'. Useful for select boxes."
if type(var) is int and type(fld) is int:
if var == fld:
return " selected"
elif str(var) == str(fld):
return " selected"
elif fld and len(fld)==3 and fld[0] == "w" and var == fld[1:]:
return " selected"
return ""
def wash_colls(cc, c, split_colls=0, verbose=0):
"""Wash collection list by checking whether user has deselected
anything under 'Narrow search'. Checks also if cc is a list or not.
Return list of cc, colls_to_display, colls_to_search since the list
of collections to display is different from that to search in.
This is because users might have chosen 'split by collection'
functionality.
The behaviour of "collections to display" depends solely whether
user has deselected a particular collection: e.g. if it started
from 'Articles and Preprints' page, and deselected 'Preprints',
then collection to display is 'Articles'. If he did not deselect
anything, then collection to display is 'Articles & Preprints'.
The behaviour of "collections to search in" depends on the
'split_colls' parameter:
* if is equal to 1, then we can wash the colls list down
and search solely in the collection the user started from;
* if is equal to 0, then we are splitting to the first level
of collections, i.e. collections as they appear on the page
we started to search from;
The function raises exception
InvenioWebSearchUnknownCollectionError
if cc or one of c collections is not known.
"""
colls_out = []
colls_out_for_display = []
# list to hold the hosted collections to be searched and displayed
hosted_colls_out = []
debug = ""
if verbose:
debug += "<br />"
debug += "<br />1) --- initial parameters ---"
debug += "<br />cc : %s" % cc
debug += "<br />c : %s" % c
debug += "<br />"
# check what type is 'cc':
if type(cc) is list:
for ci in cc:
if collection_reclist_cache.cache.has_key(ci):
# yes this collection is real, so use it:
cc = ci
break
else:
# check once if cc is real:
if not collection_reclist_cache.cache.has_key(cc):
if cc:
raise InvenioWebSearchUnknownCollectionError(cc)
else:
cc = CFG_SITE_NAME # cc is not set, so replace it with Home collection
# check type of 'c' argument:
if type(c) is list:
colls = c
else:
colls = [c]
if verbose:
debug += "<br />2) --- after check for the integrity of cc and the being or not c a list ---"
debug += "<br />cc : %s" % cc
debug += "<br />c : %s" % c
debug += "<br />"
# remove all 'unreal' collections:
colls_real = []
for coll in colls:
if collection_reclist_cache.cache.has_key(coll):
colls_real.append(coll)
else:
if coll:
raise InvenioWebSearchUnknownCollectionError(coll)
colls = colls_real
if verbose:
debug += "<br />3) --- keeping only the real colls of c ---"
debug += "<br />colls : %s" % colls
debug += "<br />"
# check if some real collections remain:
if len(colls)==0:
colls = [cc]
if verbose:
debug += "<br />4) --- in case no colls were left we use cc directly ---"
debug += "<br />colls : %s" % colls
debug += "<br />"
# then let us check the list of non-restricted "real" sons of 'cc' and compare it to 'coll':
res = run_sql("""SELECT c.name FROM collection AS c,
collection_collection AS cc,
collection AS ccc
WHERE c.id=cc.id_son AND cc.id_dad=ccc.id
AND ccc.name=%s AND cc.type='r'""", (cc,))
# list that holds all the non restricted sons of cc that are also not hosted collections
l_cc_nonrestricted_sons_and_nonhosted_colls = []
res_hosted = run_sql("""SELECT c.name FROM collection AS c,
collection_collection AS cc,
collection AS ccc
WHERE c.id=cc.id_son AND cc.id_dad=ccc.id
AND ccc.name=%s AND cc.type='r'
AND (c.dbquery NOT LIKE 'hostedcollection:%%' OR c.dbquery IS NULL)""", (cc,))
for row_hosted in res_hosted:
l_cc_nonrestricted_sons_and_nonhosted_colls.append(row_hosted[0])
l_cc_nonrestricted_sons_and_nonhosted_colls.sort()
l_cc_nonrestricted_sons = []
l_c = colls
for row in res:
if not collection_restricted_p(row[0]):
l_cc_nonrestricted_sons.append(row[0])
l_c.sort()
l_cc_nonrestricted_sons.sort()
if l_cc_nonrestricted_sons == l_c:
colls_out_for_display = [cc] # yep, washing permitted, it is sufficient to display 'cc'
# the following elif is a hack that preserves the above funcionality when we start searching from
# the frontpage with some hosted collections deselected (either by default or manually)
elif set(l_cc_nonrestricted_sons_and_nonhosted_colls).issubset(set(l_c)):
colls_out_for_display = colls
split_colls = 0
else:
colls_out_for_display = colls # nope, we need to display all 'colls' successively
# remove duplicates:
#colls_out_for_display_nondups=filter(lambda x, colls_out_for_display=colls_out_for_display: colls_out_for_display[x-1] not in colls_out_for_display[x:], range(1, len(colls_out_for_display)+1))
#colls_out_for_display = map(lambda x, colls_out_for_display=colls_out_for_display:colls_out_for_display[x-1], colls_out_for_display_nondups)
colls_out_for_display = list(set(colls_out_for_display))
if verbose:
debug += "<br />5) --- decide whether colls_out_for_diplay should be colls or is it sufficient for it to be cc; remove duplicates ---"
debug += "<br />colls_out_for_display : %s" % colls_out_for_display
debug += "<br />"
# the following piece of code takes care of removing collections whose ancestors are going to be searched anyway
# list to hold the collections to be removed
colls_to_be_removed = []
# first calculate the collections that can safely be removed
for coll in colls_out_for_display:
for ancestor in get_coll_ancestors(coll):
#if ancestor in colls_out_for_display: colls_to_be_removed.append(coll)
if ancestor in colls_out_for_display and not is_hosted_collection(coll): colls_to_be_removed.append(coll)
# secondly remove the collections
for coll in colls_to_be_removed:
colls_out_for_display.remove(coll)
if verbose:
debug += "<br />6) --- remove collections that have ancestors about to be search, unless they are hosted ---"
debug += "<br />colls_out_for_display : %s" % colls_out_for_display
debug += "<br />"
# calculate the hosted collections to be searched.
if colls_out_for_display == [cc]:
if is_hosted_collection(cc):
hosted_colls_out.append(cc)
else:
for coll in get_coll_sons(cc):
if is_hosted_collection(coll):
hosted_colls_out.append(coll)
else:
for coll in colls_out_for_display:
if is_hosted_collection(coll):
hosted_colls_out.append(coll)
if verbose:
debug += "<br />7) --- calculate the hosted_colls_out ---"
debug += "<br />hosted_colls_out : %s" % hosted_colls_out
debug += "<br />"
# second, let us decide on collection splitting:
if split_colls == 0:
# type A - no sons are wanted
colls_out = colls_out_for_display
else:
# type B - sons (first-level descendants) are wanted
for coll in colls_out_for_display:
coll_sons = get_coll_sons(coll)
if coll_sons == []:
colls_out.append(coll)
else:
for coll_son in coll_sons:
if not is_hosted_collection(coll_son):
colls_out.append(coll_son)
#else:
# colls_out = colls_out + coll_sons
# remove duplicates:
#colls_out_nondups=filter(lambda x, colls_out=colls_out: colls_out[x-1] not in colls_out[x:], range(1, len(colls_out)+1))
#colls_out = map(lambda x, colls_out=colls_out:colls_out[x-1], colls_out_nondups)
colls_out = list(set(colls_out))
if verbose:
debug += "<br />8) --- calculate the colls_out; remove duplicates ---"
debug += "<br />colls_out : %s" % colls_out
debug += "<br />"
# remove the hosted collections from the collections to be searched
if hosted_colls_out:
for coll in hosted_colls_out:
try:
colls_out.remove(coll)
except ValueError:
# in case coll was not found in colls_out
pass
if verbose:
debug += "<br />9) --- remove the hosted_colls from the colls_out ---"
debug += "<br />colls_out : %s" % colls_out
return (cc, colls_out_for_display, colls_out, hosted_colls_out, debug)
def strip_accents(x):
"""Strip accents in the input phrase X (assumed in UTF-8) by replacing
accented characters with their unaccented cousins (e.g. é by e).
Return such a stripped X."""
x = re_latex_lowercase_a.sub("a", x)
x = re_latex_lowercase_ae.sub("ae", x)
x = re_latex_lowercase_e.sub("e", x)
x = re_latex_lowercase_i.sub("i", x)
x = re_latex_lowercase_o.sub("o", x)
x = re_latex_lowercase_u.sub("u", x)
x = re_latex_lowercase_y.sub("x", x)
x = re_latex_lowercase_c.sub("c", x)
x = re_latex_lowercase_n.sub("n", x)
x = re_latex_uppercase_a.sub("A", x)
x = re_latex_uppercase_ae.sub("AE", x)
x = re_latex_uppercase_e.sub("E", x)
x = re_latex_uppercase_i.sub("I", x)
x = re_latex_uppercase_o.sub("O", x)
x = re_latex_uppercase_u.sub("U", x)
x = re_latex_uppercase_y.sub("Y", x)
x = re_latex_uppercase_c.sub("C", x)
x = re_latex_uppercase_n.sub("N", x)
# convert input into Unicode string:
try:
y = unicode(x, "utf-8")
except:
return x # something went wrong, probably the input wasn't UTF-8
# asciify Latin-1 lowercase characters:
y = re_unicode_lowercase_a.sub("a", y)
y = re_unicode_lowercase_ae.sub("ae", y)
y = re_unicode_lowercase_e.sub("e", y)
y = re_unicode_lowercase_i.sub("i", y)
y = re_unicode_lowercase_o.sub("o", y)
y = re_unicode_lowercase_u.sub("u", y)
y = re_unicode_lowercase_y.sub("y", y)
y = re_unicode_lowercase_c.sub("c", y)
y = re_unicode_lowercase_n.sub("n", y)
# asciify Latin-1 uppercase characters:
y = re_unicode_uppercase_a.sub("A", y)
y = re_unicode_uppercase_ae.sub("AE", y)
y = re_unicode_uppercase_e.sub("E", y)
y = re_unicode_uppercase_i.sub("I", y)
y = re_unicode_uppercase_o.sub("O", y)
y = re_unicode_uppercase_u.sub("U", y)
y = re_unicode_uppercase_y.sub("Y", y)
y = re_unicode_uppercase_c.sub("C", y)
y = re_unicode_uppercase_n.sub("N", y)
# return UTF-8 representation of the Unicode string:
return y.encode("utf-8")
def wash_index_term(term, max_char_length=50, lower_term=True):
"""
Return washed form of the index term TERM that would be suitable
for storing into idxWORD* tables. I.e., lower the TERM if
LOWER_TERM is True, and truncate it safely to MAX_CHAR_LENGTH
UTF-8 characters (meaning, in principle, 4*MAX_CHAR_LENGTH bytes).
The function works by an internal conversion of TERM, when needed,
from its input Python UTF-8 binary string format into Python
Unicode format, and then truncating it safely to the given number
of UTF-8 characters, without possible mis-truncation in the middle
of a multi-byte UTF-8 character that could otherwise happen if we
would have been working with UTF-8 binary representation directly.
Note that MAX_CHAR_LENGTH corresponds to the length of the term
column in idxINDEX* tables.
"""
if lower_term:
washed_term = unicode(term, 'utf-8').lower()
else:
washed_term = unicode(term, 'utf-8')
if len(washed_term) <= max_char_length:
# no need to truncate the term, because it will fit
# nicely even if it uses four-byte UTF-8 characters
return washed_term.encode('utf-8')
else:
# truncate the term in a safe position:
return washed_term[:max_char_length].encode('utf-8')
def lower_index_term(term):
"""
Return safely lowered index term TERM. This is done by converting
to UTF-8 first, because standard Python lower() function is not
UTF-8 safe. To be called by both the search engine and the
indexer when appropriate (e.g. before stemming).
In case of problems with UTF-8 compliance, this function raises
UnicodeDecodeError, so the client code may want to catch it.
"""
return unicode(term, 'utf-8').lower().encode('utf-8')
def wash_output_format(format):
"""Wash output format FORMAT. Currently only prevents input like
'of=9' for backwards-compatible format that prints certain fields
only. (for this task, 'of=tm' is preferred)"""
if str(format[0:3]).isdigit() and len(format) != 6:
# asked to print MARC tags, but not enough digits,
# so let's switch back to HTML brief default
return 'hb'
else:
return format
def wash_pattern(p):
"""Wash pattern passed by URL. Check for sanity of the wildcard by
removing wildcards if they are appended to extremely short words
(1-3 letters). TODO: instead of this approximative treatment, it
will be much better to introduce a temporal limit, e.g. to kill a
query if it does not finish in 10 seconds."""
# strip accents:
# p = strip_accents(p) # FIXME: when available, strip accents all the time
# add leading/trailing whitespace for the two following wildcard-sanity checking regexps:
p = " " + p + " "
# replace spaces within quotes by __SPACE__ temporarily:
p = re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", p)
p = re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", p)
p = re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", p)
# get rid of unquoted wildcards after spaces:
p = re_pattern_wildcards_after_spaces.sub("\\1", p)
# get rid of extremely short words (1-3 letters with wildcards):
p = re_pattern_short_words.sub("\\1", p)
# replace back __SPACE__ by spaces:
p = re_pattern_space.sub(" ", p)
# replace special terms:
p = re_pattern_today.sub(time.strftime("%Y-%m-%d", time.localtime()), p)
# remove unnecessary whitespace:
p = string.strip(p)
return p
def wash_field(f):
"""Wash field passed by URL."""
# get rid of unnecessary whitespace:
if f:
f = f.strip()
# wash legacy 'f' field names, e.g. replace 'wau' or `au' by
# 'author', if applicable:
if CFG_WEBSEARCH_FIELDS_CONVERT:
f = CFG_WEBSEARCH_FIELDS_CONVERT.get(f, f)
return f
def wash_dates(d1="", d1y=0, d1m=0, d1d=0, d2="", d2y=0, d2m=0, d2d=0):
"""
Take user-submitted date arguments D1 (full datetime string) or
(D1Y, D1M, D1Y) year, month, day tuple and D2 or (D2Y, D2M, D2Y)
and return (YYY1-M1-D2 H1:M1:S2, YYY2-M2-D2 H2:M2:S2) datetime
strings in the YYYY-MM-DD HH:MM:SS format suitable for time
restricted searching.
Note that when both D1 and (D1Y, D1M, D1D) parameters are present,
the precedence goes to D1. Ditto for D2*.
Note that when (D1Y, D1M, D1D) are taken into account, some values
may be missing and are completed e.g. to 01 or 12 according to
whether it is the starting or the ending date.
"""
datetext1, datetext2 = "", ""
# sanity checking:
if d1 == "" and d1y == 0 and d1m == 0 and d1d == 0 and d2 == "" and d2y == 0 and d2m == 0 and d2d == 0:
return ("", "") # nothing selected, so return empty values
# wash first (starting) date:
if d1:
# full datetime string takes precedence:
datetext1 = d1
else:
# okay, first date passed as (year,month,day):
if d1y:
datetext1 += "%04d" % d1y
else:
datetext1 += "0000"
if d1m:
datetext1 += "-%02d" % d1m
else:
datetext1 += "-01"
if d1d:
datetext1 += "-%02d" % d1d
else:
datetext1 += "-01"
datetext1 += " 00:00:00"
# wash second (ending) date:
if d2:
# full datetime string takes precedence:
datetext2 = d2
else:
# okay, second date passed as (year,month,day):
if d2y:
datetext2 += "%04d" % d2y
else:
datetext2 += "9999"
if d2m:
datetext2 += "-%02d" % d2m
else:
datetext2 += "-12"
if d2d:
datetext2 += "-%02d" % d2d
else:
datetext2 += "-31" # NOTE: perhaps we should add max(datenumber) in
# given month, but for our quering it's not
# needed, 31 will always do
datetext2 += " 00:00:00"
# okay, return constructed YYYY-MM-DD HH:MM:SS datetexts:
return (datetext1, datetext2)
def is_hosted_collection(coll):
"""Check if the given collection is a hosted one; i.e. its dbquery starts with hostedcollection:
Returns True if it is, False if it's not or if the result is empty or if the query failed"""
res = run_sql("SELECT dbquery FROM collection WHERE name=%s", (coll, ))
try:
return res[0][0].startswith("hostedcollection:")
except:
return False
def get_colID(c):
"Return collection ID for collection name C. Return None if no match found."
colID = None
res = run_sql("SELECT id FROM collection WHERE name=%s", (c,), 1)
if res:
colID = res[0][0]
return colID
def get_coll_ancestors(coll):
"Returns a list of ancestors for collection 'coll'."
coll_ancestors = []
coll_ancestor = coll
while 1:
res = run_sql("""SELECT c.name FROM collection AS c
LEFT JOIN collection_collection AS cc ON c.id=cc.id_dad
LEFT JOIN collection AS ccc ON ccc.id=cc.id_son
WHERE ccc.name=%s ORDER BY cc.id_dad ASC LIMIT 1""",
(coll_ancestor,))
if res:
coll_name = res[0][0]
coll_ancestors.append(coll_name)
coll_ancestor = coll_name
else:
break
# ancestors found, return reversed list:
coll_ancestors.reverse()
return coll_ancestors
def get_coll_sons(coll, type='r', public_only=1):
"""Return a list of sons (first-level descendants) of type 'type' for collection 'coll'.
If public_only, then return only non-restricted son collections.
"""
coll_sons = []
query = "SELECT c.name FROM collection AS c "\
"LEFT JOIN collection_collection AS cc ON c.id=cc.id_son "\
"LEFT JOIN collection AS ccc ON ccc.id=cc.id_dad "\
"WHERE cc.type=%s AND ccc.name=%s"
query += " ORDER BY cc.score DESC"
res = run_sql(query, (type, coll))
for name in res:
if not public_only or not collection_restricted_p(name[0]):
coll_sons.append(name[0])
return coll_sons
def get_coll_real_descendants(coll, type='_', get_hosted_colls=True):
"""Return a list of all descendants of collection 'coll' that are defined by a 'dbquery'.
IOW, we need to decompose compound collections like "A & B" into "A" and "B" provided
that "A & B" has no associated database query defined.
"""
coll_sons = []
res = run_sql("""SELECT c.name,c.dbquery FROM collection AS c
LEFT JOIN collection_collection AS cc ON c.id=cc.id_son
LEFT JOIN collection AS ccc ON ccc.id=cc.id_dad
WHERE ccc.name=%s AND cc.type LIKE %s ORDER BY cc.score DESC""",
(coll, type,))
for name, dbquery in res:
if dbquery: # this is 'real' collection, so return it:
if get_hosted_colls:
coll_sons.append(name)
else:
if not dbquery.startswith("hostedcollection:"):
coll_sons.append(name)
else: # this is 'composed' collection, so recurse:
coll_sons.extend(get_coll_real_descendants(name))
return coll_sons
def browse_pattern(req, colls, p, f, rg, ln=CFG_SITE_LANG):
"""Browse either biliographic phrases or words indexes, and display it."""
# load the right message language
_ = gettext_set_language(ln)
## is p enclosed in quotes? (coming from exact search)
if p.startswith('"') and p.endswith('"'):
p = p[1:-1]
p_orig = p
## okay, "real browse" follows:
## FIXME: the maths in the get_nearest_terms_in_bibxxx is just a test
if not f and string.find(p, ":") > 0: # does 'p' contain ':'?
f, p = string.split(p, ":", 1)
## do we search in words indexes?
if not f:
return browse_in_bibwords(req, p, f)
index_id = get_index_id_from_field(f)
if index_id != 0:
coll = HitSet()
for coll_name in colls:
coll |= get_collection_reclist(coll_name)
browsed_phrases_in_colls = get_nearest_terms_in_idxphrase_with_collection(p, index_id, rg/2, rg/2, coll)
else:
browsed_phrases = get_nearest_terms_in_bibxxx(p, f, (rg+1)/2+1, (rg-1)/2+1)
while not browsed_phrases:
# try again and again with shorter and shorter pattern:
try:
p = p[:-1]
browsed_phrases = get_nearest_terms_in_bibxxx(p, f, (rg+1)/2+1, (rg-1)/2+1)
except:
# probably there are no hits at all:
req.write(_("No values found."))
return
## try to check hits in these particular collection selection:
browsed_phrases_in_colls = []
if 0:
for phrase in browsed_phrases:
phrase_hitset = HitSet()
phrase_hitsets = search_pattern("", phrase, f, 'e')
for coll in colls:
phrase_hitset.union_update(phrase_hitsets[coll])
if len(phrase_hitset) > 0:
# okay, this phrase has some hits in colls, so add it:
browsed_phrases_in_colls.append([phrase, len(phrase_hitset)])
## were there hits in collections?
if browsed_phrases_in_colls == []:
if browsed_phrases != []:
#print_warning(req, """<p>No match close to <em>%s</em> found in given collections.
#Please try different term.<p>Displaying matches in any collection...""" % p_orig)
## try to get nbhits for these phrases in any collection:
for phrase in browsed_phrases:
browsed_phrases_in_colls.append([phrase, get_nbhits_in_bibxxx(phrase, f)])
## display results now:
out = websearch_templates.tmpl_browse_pattern(
f=f,
fn=get_field_i18nname(get_field_name(f) or f, ln, False),
ln=ln,
browsed_phrases_in_colls=browsed_phrases_in_colls,
colls=colls,
rg=rg,
)
req.write(out)
return
def browse_in_bibwords(req, p, f, ln=CFG_SITE_LANG):
"""Browse inside words indexes."""
if not p:
return
_ = gettext_set_language(ln)
urlargd = {}
urlargd.update(req.argd)
urlargd['action'] = 'search'
nearest_box = create_nearest_terms_box(urlargd, p, f, 'w', ln=ln, intro_text_p=0)
req.write(websearch_templates.tmpl_search_in_bibwords(
p = p,
f = f,
ln = ln,
nearest_box = nearest_box
))
return
def search_pattern(req=None, p=None, f=None, m=None, ap=0, of="id", verbose=0, ln=CFG_SITE_LANG, display_nearest_terms_box=True):
"""Search for complex pattern 'p' within field 'f' according to
matching type 'm'. Return hitset of recIDs.
The function uses multi-stage searching algorithm in case of no
exact match found. See the Search Internals document for
detailed description.
The 'ap' argument governs whether an alternative patterns are to
be used in case there is no direct hit for (p,f,m). For
example, whether to replace non-alphanumeric characters by
spaces if it would give some hits. See the Search Internals
document for detailed description. (ap=0 forbits the
alternative pattern usage, ap=1 permits it.)
The 'of' argument governs whether to print or not some
information to the user in case of no match found. (Usually it
prints the information in case of HTML formats, otherwise it's
silent).
The 'verbose' argument controls the level of debugging information
to be printed (0=least, 9=most).
All the parameters are assumed to have been previously washed.
This function is suitable as a mid-level API.
"""
_ = gettext_set_language(ln)
hitset_empty = HitSet()
# sanity check:
if not p:
hitset_full = HitSet(trailing_bits=1)
hitset_full.discard(0)
# no pattern, so return all universe
return hitset_full
# search stage 1: break up arguments into basic search units:
if verbose and of.startswith("h"):
t1 = os.times()[4]
basic_search_units = create_basic_search_units(req, p, f, m, of)
if verbose and of.startswith("h"):
t2 = os.times()[4]
print_warning(req, "Search stage 1: basic search units are: %s" % cgi.escape(repr(basic_search_units)))
print_warning(req, "Search stage 1: execution took %.2f seconds." % (t2 - t1))
# search stage 2: do search for each search unit and verify hit presence:
if verbose and of.startswith("h"):
t1 = os.times()[4]
basic_search_units_hitsets = []
#prepare hiddenfield-related..
myhiddens = CFG_BIBFORMAT_HIDDEN_TAGS
can_see_hidden = False
if req:
user_info = collect_user_info(req)
can_see_hidden = (acc_authorize_action(user_info, 'runbibedit')[0] == 0)
if can_see_hidden:
myhiddens = []
if CFG_INSPIRE_SITE and of.startswith('h'):
# fulltext/caption search warnings for INSPIRE:
fields_to_be_searched = [f for o,p,f,m in basic_search_units]
if 'fulltext' in fields_to_be_searched:
print_warning(req, _("Warning: full-text search is only available for a subset of papers mostly from 2006-2010."))
elif 'caption' in fields_to_be_searched:
print_warning(req, _("Warning: figure caption search is only available for a subset of papers mostly from 2008-2010."))
for idx_unit in xrange(len(basic_search_units)):
bsu_o, bsu_p, bsu_f, bsu_m = basic_search_units[idx_unit]
basic_search_unit_hitset = search_unit(bsu_p, bsu_f, bsu_m)
# FIXME: workaround for not having phrase index yet
if bsu_f == 'fulltext' and bsu_m != 'w' and of.startswith('h'):
print_warning(req, _("No phrase index available for fulltext yet, looking for word combination..."))
#check that the user is allowed to search with this tag
#if he/she tries it
if bsu_f and len(bsu_f) > 1 and bsu_f[0].isdigit() and bsu_f[1].isdigit():
for htag in myhiddens:
ltag = len(htag)
samelenfield = bsu_f[0:ltag]
if samelenfield == htag: #user searches by a hidden tag
#we won't show you anything..
basic_search_unit_hitset = HitSet()
if verbose >= 9 and of.startswith("h"):
print_warning(req, "Pattern %s hitlist omitted since \
it queries in a hidden tag %s" %
(repr(bsu_p), repr(myhiddens)))
display_nearest_terms_box=False #..and stop spying, too.
if verbose >= 9 and of.startswith("h"):
print_warning(req, "Search stage 1: pattern %s gave hitlist %s" % (cgi.escape(bsu_p), basic_search_unit_hitset))
if len(basic_search_unit_hitset) > 0 or \
ap==0 or \
bsu_o=="|" or \
((idx_unit+1)<len(basic_search_units) and basic_search_units[idx_unit+1][0]=="|"):
# stage 2-1: this basic search unit is retained, since
# either the hitset is non-empty, or the approximate
# pattern treatment is switched off, or the search unit
# was joined by an OR operator to preceding/following
# units so we do not require that it exists
basic_search_units_hitsets.append(basic_search_unit_hitset)
else:
# stage 2-2: no hits found for this search unit, try to replace non-alphanumeric chars inside pattern:
if re.search(r'[^a-zA-Z0-9\s\:]', bsu_p) and bsu_f != 'refersto' and bsu_f != 'citedby':
if bsu_p.startswith('"') and bsu_p.endswith('"'): # is it ACC query?
bsu_pn = re.sub(r'[^a-zA-Z0-9\s\:]+', "*", bsu_p)
else: # it is WRD query
bsu_pn = re.sub(r'[^a-zA-Z0-9\s\:]+', " ", bsu_p)
if verbose and of.startswith('h') and req:
print_warning(req, "Trying (%s,%s,%s)" % (cgi.escape(bsu_pn), cgi.escape(bsu_f), cgi.escape(bsu_m)))
basic_search_unit_hitset = search_pattern(req=None, p=bsu_pn, f=bsu_f, m=bsu_m, of="id", ln=ln)
if len(basic_search_unit_hitset) > 0:
# we retain the new unit instead
if of.startswith('h'):
print_warning(req, _("No exact match found for %(x_query1)s, using %(x_query2)s instead...") % \
{'x_query1': "<em>" + cgi.escape(bsu_p) + "</em>",
'x_query2': "<em>" + cgi.escape(bsu_pn) + "</em>"})
basic_search_units[idx_unit][1] = bsu_pn
basic_search_units_hitsets.append(basic_search_unit_hitset)
else:
# stage 2-3: no hits found either, propose nearest indexed terms:
if of.startswith('h') and display_nearest_terms_box:
if req:
if bsu_f == "recid":
print_warning(req, _("Requested record does not seem to exist."))
else:
print_warning(req, create_nearest_terms_box(req.argd, bsu_p, bsu_f, bsu_m, ln=ln))
return hitset_empty
else:
# stage 2-3: no hits found either, propose nearest indexed terms:
if of.startswith('h') and display_nearest_terms_box:
if req:
if bsu_f == "recid":
print_warning(req, _("Requested record does not seem to exist."))
else:
print_warning(req, create_nearest_terms_box(req.argd, bsu_p, bsu_f, bsu_m, ln=ln))
return hitset_empty
if verbose and of.startswith("h"):
t2 = os.times()[4]
for idx_unit in range(0, len(basic_search_units)):
print_warning(req, "Search stage 2: basic search unit %s gave %d hits." %
(basic_search_units[idx_unit][1:], len(basic_search_units_hitsets[idx_unit])))
print_warning(req, "Search stage 2: execution took %.2f seconds." % (t2 - t1))
# search stage 3: apply boolean query for each search unit:
if verbose and of.startswith("h"):
t1 = os.times()[4]
# let the initial set be the complete universe:
hitset_in_any_collection = HitSet(trailing_bits=1)
hitset_in_any_collection.discard(0)
for idx_unit in xrange(len(basic_search_units)):
this_unit_operation = basic_search_units[idx_unit][0]
this_unit_hitset = basic_search_units_hitsets[idx_unit]
if this_unit_operation == '+':
hitset_in_any_collection.intersection_update(this_unit_hitset)
elif this_unit_operation == '-':
hitset_in_any_collection.difference_update(this_unit_hitset)
elif this_unit_operation == '|':
hitset_in_any_collection.union_update(this_unit_hitset)
else:
if of.startswith("h"):
print_warning(req, "Invalid set operation %s." % cgi.escape(this_unit_operation), "Error")
if len(hitset_in_any_collection) == 0:
# no hits found, propose alternative boolean query:
if of.startswith('h') and display_nearest_terms_box:
nearestterms = []
for idx_unit in range(0, len(basic_search_units)):
bsu_o, bsu_p, bsu_f, bsu_m = basic_search_units[idx_unit]
if bsu_p.startswith("%") and bsu_p.endswith("%"):
bsu_p = "'" + bsu_p[1:-1] + "'"
bsu_nbhits = len(basic_search_units_hitsets[idx_unit])
# create a similar query, but with the basic search unit only
argd = {}
argd.update(req.argd)
argd['p'] = bsu_p
argd['f'] = bsu_f
nearestterms.append((bsu_p, bsu_nbhits, argd))
text = websearch_templates.tmpl_search_no_boolean_hits(
ln=ln, nearestterms=nearestterms)
print_warning(req, text)
if verbose and of.startswith("h"):
t2 = os.times()[4]
print_warning(req, "Search stage 3: boolean query gave %d hits." % len(hitset_in_any_collection))
print_warning(req, "Search stage 3: execution took %.2f seconds." % (t2 - t1))
return hitset_in_any_collection
def search_pattern_parenthesised(req=None, p=None, f=None, m=None, ap=0, of="id", verbose=0, ln=CFG_SITE_LANG, display_nearest_terms_box=True):
"""Search for complex pattern 'p' containing parenthesis within field 'f' according to
matching type 'm'. Return hitset of recIDs.
For more details on the parameters see 'search_pattern'
"""
_ = gettext_set_language(ln)
spires_syntax_converter = SpiresToInvenioSyntaxConverter()
spires_syntax_query = False
# if the pattern uses SPIRES search syntax, convert it to Invenio syntax
if spires_syntax_converter.is_applicable(p):
spires_syntax_query = True
p = spires_syntax_converter.convert_query(p)
# sanity check: do not call parenthesised parser for search terms
# like U(1):
if not re_pattern_parens.search(p):
return search_pattern(req, p, f, m, ap, of, verbose, ln, display_nearest_terms_box=display_nearest_terms_box)
# Try searching with parentheses
try:
parser = SearchQueryParenthesisedParser()
# get a hitset with all recids
result_hitset = HitSet(trailing_bits=1)
# parse the query. The result is list of [op1, expr1, op2, expr2, ..., opN, exprN]
parsing_result = parser.parse_query(p)
if verbose and of.startswith("h"):
print_warning(req, "Search stage 1: search_pattern_parenthesised() returned %s." % repr(parsing_result))
# go through every pattern
# calculate hitset for it
# combine pattern's hitset with the result using the corresponding operator
for index in xrange(0, len(parsing_result)-1, 2 ):
current_operator = parsing_result[index]
current_pattern = parsing_result[index+1]
if CFG_INSPIRE_SITE and spires_syntax_query:
# setting ap=0 to turn off approximate matching for 0 results.
# Doesn't work well in combinations.
# FIXME: The right fix involves collecting statuses for each
# hitset, then showing a nearest terms box exactly once,
# outside this loop.
ap = 0
display_nearest_terms_box=False
# obtain a hitset for the current pattern
current_hitset = search_pattern(req, current_pattern, f, m, ap, of, verbose, ln, display_nearest_terms_box=display_nearest_terms_box)
# combine the current hitset with resulting hitset using the current operator
if current_operator == '+':
result_hitset = result_hitset & current_hitset
elif current_operator == '-':
result_hitset = result_hitset - current_hitset
elif current_operator == '|':
result_hitset = result_hitset | current_hitset
else:
assert False, "Unknown operator in search_pattern_parenthesised()"
return result_hitset
# If searching with parenteses fails, perform search ignoring parentheses
except SyntaxError:
print_warning(req, _("Search syntax misunderstood. Ignoring all parentheses in the query. If this doesn't help, please check your search and try again."))
# remove the parentheses in the query. Current implementation removes all the parentheses,
# but it could be improved to romove only these that are not inside quotes
p = p.replace('(', ' ')
p = p.replace(')', ' ')
return search_pattern(req, p, f, m, ap, of, verbose, ln, display_nearest_terms_box=display_nearest_terms_box)
def search_unit(p, f=None, m=None):
"""Search for basic search unit defined by pattern 'p' and field
'f' and matching type 'm'. Return hitset of recIDs.
All the parameters are assumed to have been previously washed.
'p' is assumed to be already a ``basic search unit'' so that it
is searched as such and is not broken up in any way. Only
wildcard and span queries are being detected inside 'p'.
This function is suitable as a low-level API.
"""
## create empty output results set:
set = HitSet()
if not p: # sanity checking
return set
if f == 'datecreated':
set = search_unit_in_bibrec(p, p, 'c')
elif f == 'datemodified':
set = search_unit_in_bibrec(p, p, 'm')
elif f == 'refersto':
# we are doing search by the citation count
set = search_unit_refersto(p)
elif f == 'citedby':
# we are doing search by the citation count
set = search_unit_citedby(p)
elif m == 'a' or m == 'r':
# FIXME: workaround for not having phrase index yet
if f == 'fulltext':
return search_pattern(None, p, f, 'w')
# we are doing either phrase search or regexp search
index_id = get_index_id_from_field(f)
if index_id != 0:
set = search_unit_in_idxphrases(p, f, m)
else:
set = search_unit_in_bibxxx(p, f, m)
elif p.startswith("cited:"):
# we are doing search by the citation count
set = search_unit_by_times_cited(p[6:])
else:
# we are doing bibwords search by default
set = search_unit_in_bibwords(p, f)
return set
def search_unit_in_bibwords(word, f, decompress=zlib.decompress):
"""Searches for 'word' inside bibwordsX table for field 'f' and returns hitset of recIDs."""
set = HitSet() # will hold output result set
set_used = 0 # not-yet-used flag, to be able to circumvent set operations
# deduce into which bibwordsX table we will search:
stemming_language = get_index_stemming_language(get_index_id_from_field("anyfield"))
bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield")
if f:
index_id = get_index_id_from_field(f)
if index_id:
bibwordsX = "idxWORD%02dF" % index_id
stemming_language = get_index_stemming_language(index_id)
else:
return HitSet() # word index f does not exist
# wash 'word' argument and run query:
word = string.replace(word, '*', '%') # we now use '*' as the truncation character
words = string.split(word, "->", 1) # check for span query
if len(words) == 2:
word0 = re_word.sub('', words[0])
word1 = re_word.sub('', words[1])
if stemming_language:
word0 = lower_index_term(word0)
word1 = lower_index_term(word1)
word0 = stem(word0, stemming_language)
word1 = stem(word1, stemming_language)
res = run_sql("SELECT term,hitlist FROM %s WHERE term BETWEEN %%s AND %%s" % bibwordsX,
(wash_index_term(word0), wash_index_term(word1)))
else:
if f == 'journal':
pass # FIXME: quick hack for the journal index
else:
word = re_word.sub('', word)
if stemming_language:
word = lower_index_term(word)
word = stem(word, stemming_language)
if string.find(word, '%') >= 0: # do we have wildcard in the word?
if f == 'journal':
# FIXME: quick hack for the journal index
# FIXME: we can run a sanity check here for all indexes
res = ()
else:
res = run_sql("SELECT term,hitlist FROM %s WHERE term LIKE %%s" % bibwordsX,
(wash_index_term(word),))
else:
res = run_sql("SELECT term,hitlist FROM %s WHERE term=%%s" % bibwordsX,
(wash_index_term(word),))
# fill the result set:
for word, hitlist in res:
hitset_bibwrd = HitSet(hitlist)
# add the results:
if set_used:
set.union_update(hitset_bibwrd)
else:
set = hitset_bibwrd
set_used = 1
# okay, return result set:
return set
def search_unit_in_idxphrases(p, f, type):
"""Searches for phrase 'p' inside idxPHRASE*F table for field 'f' and returns hitset of recIDs found.
The search type is defined by 'type' (e.g. equals to 'r' for a regexp search)."""
set = HitSet() # will hold output result set
set_used = 0 # not-yet-used flag, to be able to circumvent set operations
# deduce in which idxPHRASE table we will search:
idxphraseX = "idxPHRASE%02dF" % get_index_id_from_field("anyfield")
if f:
index_id = get_index_id_from_field(f)
if index_id:
idxphraseX = "idxPHRASE%02dF" % index_id
else:
return HitSet() # phrase index f does not exist
# detect query type (exact phrase, partial phrase, regexp):
if type == 'r':
query_addons = "REGEXP %s"
query_params = (p,)
else:
p = string.replace(p, '*', '%') # we now use '*' as the truncation character
ps = string.split(p, "->", 1) # check for span query:
if len(ps) == 2 and not (ps[0].endswith(' ') or ps[1].startswith(' ')):
query_addons = "BETWEEN %s AND %s"
query_params = (ps[0], ps[1])
else:
if string.find(p, '%') > -1:
query_addons = "LIKE %s"
query_params = (p,)
else:
query_addons = "= %s"
query_params = (p,)
# special washing for fuzzy author index:
if f == 'author' or f == 'exactauthor':
query_params_washed = ()
for query_param in query_params:
query_params_washed += (wash_author_name(query_param),)
query_params = query_params_washed
# perform search:
res = run_sql("SELECT term,hitlist FROM %s WHERE term %s" % (idxphraseX, query_addons),
query_params)
# fill the result set:
for word, hitlist in res:
hitset_bibphrase = HitSet(hitlist)
# add the results:
if set_used:
set.union_update(hitset_bibphrase)
else:
set = hitset_bibphrase
set_used = 1
# okay, return result set:
return set
def search_unit_in_bibxxx(p, f, type):
"""Searches for pattern 'p' inside bibxxx tables for field 'f' and returns hitset of recIDs found.
The search type is defined by 'type' (e.g. equals to 'r' for a regexp search)."""
# FIXME: quick hack for the journal index
if f == 'journal':
return search_unit_in_bibwords(p, f)
p_orig = p # saving for eventual future 'no match' reporting
query_addons = "" # will hold additional SQL code for the query
query_params = () # will hold parameters for the query (their number may vary depending on TYPE argument)
# wash arguments:
f = string.replace(f, '*', '%') # replace truncation char '*' in field definition
if type == 'r':
query_addons = "REGEXP %s"
query_params = (p,)
else:
p = string.replace(p, '*', '%') # we now use '*' as the truncation character
ps = string.split(p, "->", 1) # check for span query:
if len(ps) == 2 and not (ps[0].endswith(' ') or ps[1].startswith(' ')):
query_addons = "BETWEEN %s AND %s"
query_params = (ps[0], ps[1])
else:
if string.find(p, '%') > -1:
query_addons = "LIKE %s"
query_params = (p,)
else:
query_addons = "= %s"
query_params = (p,)
# construct 'tl' which defines the tag list (MARC tags) to search in:
tl = []
if str(f[0]).isdigit() and str(f[1]).isdigit():
tl.append(f) # 'f' seems to be okay as it starts by two digits
else:
# deduce desired MARC tags on the basis of chosen 'f'
tl = get_field_tags(f)
if not tl:
# f index does not exist, nevermind
pass
# okay, start search:
l = [] # will hold list of recID that matched
for t in tl:
# deduce into which bibxxx table we will search:
digit1, digit2 = int(t[0]), int(t[1])
bx = "bib%d%dx" % (digit1, digit2)
bibx = "bibrec_bib%d%dx" % (digit1, digit2)
# construct and run query:
if t == "001":
res = run_sql("SELECT id FROM bibrec WHERE id %s" % query_addons,
query_params)
else:
query = "SELECT bibx.id_bibrec FROM %s AS bx LEFT JOIN %s AS bibx ON bx.id=bibx.id_bibxxx WHERE bx.value %s" % \
(bx, bibx, query_addons)
if len(t) != 6 or t[-1:]=='%':
# wildcard query, or only the beginning of field 't'
# is defined, so add wildcard character:
query += " AND bx.tag LIKE %s"
res = run_sql(query, query_params + (t + '%',))
else:
# exact query for 't':
query += " AND bx.tag=%s"
res = run_sql(query, query_params + (t,))
# fill the result set:
for id_bibrec in res:
if id_bibrec[0]:
l.append(id_bibrec[0])
# check no of hits found:
nb_hits = len(l)
# okay, return result set:
set = HitSet(l)
return set
def search_unit_in_bibrec(datetext1, datetext2, type='c'):
"""
Return hitset of recIDs found that were either created or modified
(according to 'type' arg being 'c' or 'm') from datetext1 until datetext2, inclusive.
Does not pay attention to pattern, collection, anything. Useful
to intersect later on with the 'real' query.
"""
set = HitSet()
if type.startswith("m"):
type = "modification_date"
else:
type = "creation_date" # by default we are searching for creation dates
if datetext1 == datetext2:
res = run_sql("SELECT id FROM bibrec WHERE %s LIKE %%s" % (type,),
(datetext1 + '%',))
else:
res = run_sql("SELECT id FROM bibrec WHERE %s>=%%s AND %s<=%%s" % (type, type),
(datetext1, datetext2))
for row in res:
set += row[0]
return set
def search_unit_by_times_cited(p):
"""
Return histset of recIDs found that are cited P times.
Usually P looks like '10->23'.
"""
numstr = '"'+p+'"'
#this is sort of stupid but since we may need to
#get the records that do _not_ have cites, we have to
#know the ids of all records, too
#but this is needed only if bsu_p is 0 or 0 or 0->0
allrecs = []
if p == 0 or p == "0" or \
p.startswith("0->") or p.endswith("->0"):
- allrecs = HitSet(run_sql_cached("SELECT id FROM bibrec", affected_tables=['bibrec']))
+ allrecs = HitSet(run_sql("SELECT id FROM bibrec"))
return get_records_with_num_cites(numstr, allrecs)
def search_unit_refersto(query):
"""
Search for records satisfying the query (e.g. author:ellis) and
return list of records referred to by these records.
"""
if query:
ahitset = search_pattern(p=query)
if ahitset:
return get_refersto_hitset(ahitset)
else:
return HitSet([])
else:
return HitSet([])
def search_unit_citedby(query):
"""
Search for records satisfying the query (e.g. author:ellis) and
return list of records cited by these records.
"""
if query:
ahitset = search_pattern(p=query)
if ahitset:
return get_citedby_hitset(ahitset)
else:
return HitSet([])
else:
return HitSet([])
def intersect_results_with_collrecs(req, hitset_in_any_collection, colls, ap=0, of="hb", verbose=0, ln=CFG_SITE_LANG, display_nearest_terms_box=True):
"""Return dict of hitsets given by intersection of hitset with the collection universes."""
_ = gettext_set_language(ln)
# search stage 4: intersect with the collection universe:
if verbose and of.startswith("h"):
t1 = os.times()[4]
results = {}
results_nbhits = 0
for coll in colls:
results[coll] = hitset_in_any_collection & get_collection_reclist(coll)
results_nbhits += len(results[coll])
if results_nbhits == 0:
# no hits found, try to search in Home:
results_in_Home = hitset_in_any_collection & get_collection_reclist(CFG_SITE_NAME)
if len(results_in_Home) > 0:
# some hits found in Home, so propose this search:
if of.startswith("h") and display_nearest_terms_box:
url = websearch_templates.build_search_url(req.argd, cc=CFG_SITE_NAME, c=[])
print_warning(req, _("No match found in collection %(x_collection)s. Other public collections gave %(x_url_open)s%(x_nb_hits)d hits%(x_url_close)s.") %\
{'x_collection': '<em>' + string.join([get_coll_i18nname(coll, ln, False) for coll in colls], ', ') + '</em>',
'x_url_open': '<a class="nearestterms" href="%s">' % (url),
'x_nb_hits': len(results_in_Home),
'x_url_close': '</a>'})
results = {}
else:
# no hits found in Home, recommend different search terms:
if of.startswith("h") and display_nearest_terms_box:
print_warning(req, _("No public collection matched your query. "
"If you were looking for a non-public document, please choose "
"the desired restricted collection first."))
results = {}
if verbose and of.startswith("h"):
t2 = os.times()[4]
print_warning(req, "Search stage 4: intersecting with collection universe gave %d hits." % results_nbhits)
print_warning(req, "Search stage 4: execution took %.2f seconds." % (t2 - t1))
return results
def intersect_results_with_hitset(req, results, hitset, ap=0, aptext="", of="hb"):
"""Return intersection of search 'results' (a dict of hitsets
with collection as key) with the 'hitset', i.e. apply
'hitset' intersection to each collection within search
'results'.
If the final 'results' set is to be empty, and 'ap'
(approximate pattern) is true, and then print the `warningtext'
and return the original 'results' set unchanged. If 'ap' is
false, then return empty results set.
"""
if ap:
results_ap = copy.deepcopy(results)
else:
results_ap = {} # will return empty dict in case of no hits found
nb_total = 0
for coll in results.keys():
results[coll].intersection_update(hitset)
nb_total += len(results[coll])
if nb_total == 0:
if of.startswith("h"):
print_warning(req, aptext)
results = results_ap
return results
def create_similarly_named_authors_link_box(author_name, ln=CFG_SITE_LANG):
"""Return a box similar to ``Not satisfied...'' one by proposing
author searches for similar names. Namely, take AUTHOR_NAME
and the first initial of the firstame (after comma) and look
into author index whether authors with e.g. middle names exist.
Useful mainly for CERN Library that sometimes contains name
forms like Ellis-N, Ellis-Nick, Ellis-Nicolas all denoting the
same person. The box isn't proposed if no similarly named
authors are found to exist.
"""
# return nothing if not configured:
if CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX == 0:
return ""
# return empty box if there is no initial:
if re.match(r'[^ ,]+, [^ ]', author_name) is None:
return ""
# firstly find name comma initial:
author_name_to_search = re.sub(r'^([^ ,]+, +[^ ,]).*$', '\\1', author_name)
# secondly search for similar name forms:
similar_author_names = {}
for name in author_name_to_search, strip_accents(author_name_to_search):
for tag in get_field_tags("author"):
# deduce into which bibxxx table we will search:
digit1, digit2 = int(tag[0]), int(tag[1])
bx = "bib%d%dx" % (digit1, digit2)
bibx = "bibrec_bib%d%dx" % (digit1, digit2)
if len(tag) != 6 or tag[-1:]=='%':
# only the beginning of field 't' is defined, so add wildcard character:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value LIKE %%s AND bx.tag LIKE %%s""" % bx,
(name + "%", tag + "%"))
else:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value LIKE %%s AND bx.tag=%%s""" % bx,
(name + "%", tag))
for row in res:
similar_author_names[row[0]] = 1
# remove the original name and sort the list:
try:
del similar_author_names[author_name]
except KeyError:
pass
# thirdly print the box:
out = ""
if similar_author_names:
out_authors = similar_author_names.keys()
out_authors.sort()
tmp_authors = []
for out_author in out_authors:
nbhits = get_nbhits_in_bibxxx(out_author, "author")
if nbhits:
tmp_authors.append((out_author, nbhits))
out += websearch_templates.tmpl_similar_author_names(
authors=tmp_authors, ln=ln)
return out
def create_nearest_terms_box(urlargd, p, f, t='w', n=5, ln=CFG_SITE_LANG, intro_text_p=True):
"""Return text box containing list of 'n' nearest terms above/below 'p'
for the field 'f' for matching type 't' (words/phrases) in
language 'ln'.
Propose new searches according to `urlargs' with the new words.
If `intro_text_p' is true, then display the introductory message,
otherwise print only the nearest terms in the box content.
"""
# load the right message language
_ = gettext_set_language(ln)
out = ""
nearest_terms = []
if not p: # sanity check
p = "."
if p.startswith('%') and p.endswith('%'):
p = p[1:-1] # fix for partial phrase
index_id = get_index_id_from_field(f)
# FIXME: workaround for not having phrase index yet
if f == 'fulltext':
t = 'w'
# special indexes:
if f == 'refersto':
return _("There are no records referring to %s.") % cgi.escape(p)
if f == 'citedby':
return _("There are no records cited by %s.") % cgi.escape(p)
# look for nearest terms:
if t == 'w':
nearest_terms = get_nearest_terms_in_bibwords(p, f, n, n)
if not nearest_terms:
return _("No word index is available for %s.") % \
('<em>' + cgi.escape(get_field_i18nname(get_field_name(f) or f, ln, False)) + '</em>')
else:
nearest_terms = []
if index_id:
nearest_terms = get_nearest_terms_in_idxphrase(p, index_id, n, n)
if f == 'datecreated' or f == 'datemodified':
nearest_terms = get_nearest_terms_in_bibrec(p, f, n, n)
if not nearest_terms:
nearest_terms = get_nearest_terms_in_bibxxx(p, f, n, n)
if not nearest_terms:
return _("No phrase index is available for %s.") % \
('<em>' + cgi.escape(get_field_i18nname(get_field_name(f) or f, ln, False)) + '</em>')
terminfo = []
for term in nearest_terms:
if t == 'w':
hits = get_nbhits_in_bibwords(term, f)
else:
if index_id:
hits = get_nbhits_in_idxphrases(term, f)
elif f == 'datecreated' or f == 'datemodified':
hits = get_nbhits_in_bibrec(term, f)
else:
hits = get_nbhits_in_bibxxx(term, f)
argd = {}
argd.update(urlargd)
# check which fields contained the requested parameter, and replace it.
for (px, fx) in ('p', 'f'), ('p1', 'f1'), ('p2', 'f2'), ('p3', 'f3'):
if px in argd:
argd_px = argd[px]
if t == 'w':
# p was stripped of accents, to do the same:
argd_px = strip_accents(argd_px)
if f == argd[fx] or f == "anyfield" or f == "":
if string.find(argd_px, p) > -1:
argd[px] = string.replace(argd_px, p, term)
break
else:
if string.find(argd_px, f+':'+p) > -1:
argd[px] = string.replace(argd_px, f+':'+p, f+':'+term)
break
elif string.find(argd_px, f+':"'+p+'"') > -1:
argd[px] = string.replace(argd_px, f+':"'+p+'"', f+':"'+term+'"')
break
elif string.find(argd_px, f+':\''+p+'\'') > -1:
argd[px] = string.replace(argd_px, f+':\''+p+'\'', f+':\''+term+'\'')
break
terminfo.append((term, hits, argd))
intro = ""
if intro_text_p: # add full leading introductory text
if f:
intro = _("Search term %(x_term)s inside index %(x_index)s did not match any record. Nearest terms in any collection are:") % \
{'x_term': "<em>" + cgi.escape(p.startswith("%") and p.endswith("%") and p[1:-1] or p) + "</em>",
'x_index': "<em>" + cgi.escape(get_field_i18nname(get_field_name(f) or f, ln, False)) + "</em>"}
else:
intro = _("Search term %s did not match any record. Nearest terms in any collection are:") % \
("<em>" + cgi.escape(p.startswith("%") and p.endswith("%") and p[1:-1] or p) + "</em>")
return websearch_templates.tmpl_nearest_term_box(p=p, ln=ln, f=f, terminfo=terminfo,
intro=intro)
def get_nearest_terms_in_bibwords(p, f, n_below, n_above):
"""Return list of +n -n nearest terms to word `p' in index for field `f'."""
nearest_words = [] # will hold the (sorted) list of nearest words to return
# deduce into which bibwordsX table we will search:
bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield")
if f:
index_id = get_index_id_from_field(f)
if index_id:
bibwordsX = "idxWORD%02dF" % index_id
else:
return nearest_words
# firstly try to get `n' closest words above `p':
res = run_sql("SELECT term FROM %s WHERE term<%%s ORDER BY term DESC LIMIT %%s" % bibwordsX,
(p, n_above))
for row in res:
nearest_words.append(row[0])
nearest_words.reverse()
# secondly insert given word `p':
nearest_words.append(p)
# finally try to get `n' closest words below `p':
res = run_sql("SELECT term FROM %s WHERE term>%%s ORDER BY term ASC LIMIT %%s" % bibwordsX,
(p, n_below))
for row in res:
nearest_words.append(row[0])
return nearest_words
def get_nearest_terms_in_idxphrase(p, index_id, n_below, n_above):
"""Browse (-n_above, +n_below) closest bibliographic phrases
for the given pattern p in the given field idxPHRASE table,
regardless of collection.
Return list of [phrase1, phrase2, ... , phrase_n]."""
if CFG_INSPIRE_SITE and index_id == 3: # FIXME: workaround due to new fuzzy index
return [p,]
idxphraseX = "idxPHRASE%02dF" % index_id
res_above = run_sql("SELECT term FROM %s WHERE term<%%s ORDER BY term DESC LIMIT %%s" % idxphraseX, (p, n_above))
res_above = map(lambda x: x[0], res_above)
res_above.reverse()
res_below = run_sql("SELECT term FROM %s WHERE term>=%%s ORDER BY term ASC LIMIT %%s" % idxphraseX, (p, n_below))
res_below = map(lambda x: x[0], res_below)
return res_above + res_below
def get_nearest_terms_in_idxphrase_with_collection(p, index_id, n_below, n_above, collection):
"""Browse (-n_above, +n_below) closest bibliographic phrases
for the given pattern p in the given field idxPHRASE table,
considering the collection (HitSet).
Return list of [(phrase1, hitset), (phrase2, hitset), ... , (phrase_n, hitset)]."""
idxphraseX = "idxPHRASE%02dF" % index_id
res_above = run_sql("SELECT term,hitlist FROM %s WHERE term<%%s ORDER BY term DESC LIMIT %%s" % idxphraseX, (p, n_above * 3))
res_above = [(term, HitSet(hitlist) & collection) for term, hitlist in res_above]
res_above = [(term, len(hitlist)) for term, hitlist in res_above if hitlist]
res_below = run_sql("SELECT term,hitlist FROM %s WHERE term>=%%s ORDER BY term ASC LIMIT %%s" % idxphraseX, (p, n_below * 3))
res_below = [(term, HitSet(hitlist) & collection) for term, hitlist in res_below]
res_below = [(term, len(hitlist)) for term, hitlist in res_below if hitlist]
res_above.reverse()
return res_above[-n_above:] + res_below[:n_below]
def get_nearest_terms_in_bibxxx(p, f, n_below, n_above):
"""Browse (-n_above, +n_below) closest bibliographic phrases
for the given pattern p in the given field f, regardless
of collection.
Return list of [phrase1, phrase2, ... , phrase_n]."""
## determine browse field:
if not f and string.find(p, ":") > 0: # does 'p' contain ':'?
f, p = string.split(p, ":", 1)
# FIXME: quick hack for the journal index
if f == 'journal':
return get_nearest_terms_in_bibwords(p, f, n_below, n_above)
## We are going to take max(n_below, n_above) as the number of
## values to ferch from bibXXx. This is needed to work around
## MySQL UTF-8 sorting troubles in 4.0.x. Proper solution is to
## use MySQL 4.1.x or our own idxPHRASE in the future.
index_id = get_index_id_from_field(f)
if index_id:
return get_nearest_terms_in_idxphrase(p, index_id, n_below, n_above)
n_fetch = 2*max(n_below, n_above)
## construct 'tl' which defines the tag list (MARC tags) to search in:
tl = []
if str(f[0]).isdigit() and str(f[1]).isdigit():
tl.append(f) # 'f' seems to be okay as it starts by two digits
else:
# deduce desired MARC tags on the basis of chosen 'f'
tl = get_field_tags(f)
## start browsing to fetch list of hits:
browsed_phrases = {} # will hold {phrase1: 1, phrase2: 1, ..., phraseN: 1} dict of browsed phrases (to make them unique)
# always add self to the results set:
browsed_phrases[p.startswith("%") and p.endswith("%") and p[1:-1] or p] = 1
for t in tl:
# deduce into which bibxxx table we will search:
digit1, digit2 = int(t[0]), int(t[1])
bx = "bib%d%dx" % (digit1, digit2)
bibx = "bibrec_bib%d%dx" % (digit1, digit2)
# firstly try to get `n' closest phrases above `p':
if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value<%%s AND bx.tag LIKE %%s
ORDER BY bx.value DESC LIMIT %%s""" % bx,
(p, t + "%", n_fetch))
else:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value<%%s AND bx.tag=%%s
ORDER BY bx.value DESC LIMIT %%s""" % bx,
(p, t, n_fetch))
for row in res:
browsed_phrases[row[0]] = 1
# secondly try to get `n' closest phrases equal to or below `p':
if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value>=%%s AND bx.tag LIKE %%s
ORDER BY bx.value ASC LIMIT %%s""" % bx,
(p, t + "%", n_fetch))
else:
res = run_sql("""SELECT bx.value FROM %s AS bx
WHERE bx.value>=%%s AND bx.tag=%%s
ORDER BY bx.value ASC LIMIT %%s""" % bx,
(p, t, n_fetch))
for row in res:
browsed_phrases[row[0]] = 1
# select first n words only: (this is needed as we were searching
# in many different tables and so aren't sure we have more than n
# words right; this of course won't be needed when we shall have
# one ACC table only for given field):
phrases_out = browsed_phrases.keys()
phrases_out.sort(lambda x, y: cmp(string.lower(strip_accents(x)),
string.lower(strip_accents(y))))
# find position of self:
try:
idx_p = phrases_out.index(p)
except:
idx_p = len(phrases_out)/2
# return n_above and n_below:
return phrases_out[max(0, idx_p-n_above):idx_p+n_below]
def get_nearest_terms_in_bibrec(p, f, n_below, n_above):
"""Return list of nearest terms and counts from bibrec table.
p is usually a date, and f either datecreated or datemodified.
Note: below/above count is very approximative, not really respected.
"""
col = 'creation_date'
if f == 'datemodified':
col = 'modification_date'
res_above = run_sql("""SELECT DATE_FORMAT(%s,'%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s')
FROM bibrec WHERE %s < %%s
ORDER BY %s DESC LIMIT %%s""" % (col, col, col),
(p, n_above))
res_below = run_sql("""SELECT DATE_FORMAT(%s,'%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s')
FROM bibrec WHERE %s > %%s
ORDER BY %s ASC LIMIT %%s""" % (col, col, col),
(p, n_below))
out = set([])
for row in res_above:
out.add(row[0])
for row in res_below:
out.add(row[0])
out_list = list(out)
out_list.sort()
return list(out_list)
def get_nbhits_in_bibrec(term, f):
"""Return number of hits in bibrec table. term is usually a date,
and f is either 'datecreated' or 'datemodified'."""
col = 'creation_date'
if f == 'datemodified':
col = 'modification_date'
res = run_sql("SELECT COUNT(*) FROM bibrec WHERE %s LIKE %%s" % (col,),
(term + '%',))
return res[0][0]
def get_nbhits_in_bibwords(word, f):
"""Return number of hits for word 'word' inside words index for field 'f'."""
out = 0
# deduce into which bibwordsX table we will search:
bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield")
if f:
index_id = get_index_id_from_field(f)
if index_id:
bibwordsX = "idxWORD%02dF" % index_id
else:
return 0
if word:
res = run_sql("SELECT hitlist FROM %s WHERE term=%%s" % bibwordsX,
(word,))
for hitlist in res:
out += len(HitSet(hitlist[0]))
return out
def get_nbhits_in_idxphrases(word, f):
"""Return number of hits for word 'word' inside phrase index for field 'f'."""
out = 0
# deduce into which bibwordsX table we will search:
idxphraseX = "idxPHRASE%02dF" % get_index_id_from_field("anyfield")
if f:
index_id = get_index_id_from_field(f)
if index_id:
idxphraseX = "idxPHRASE%02dF" % index_id
else:
return 0
if word:
res = run_sql("SELECT hitlist FROM %s WHERE term=%%s" % idxphraseX,
(word,))
for hitlist in res:
out += len(HitSet(hitlist[0]))
return out
def get_nbhits_in_bibxxx(p, f):
"""Return number of hits for word 'word' inside words index for field 'f'."""
## determine browse field:
if not f and string.find(p, ":") > 0: # does 'p' contain ':'?
f, p = string.split(p, ":", 1)
# FIXME: quick hack for the journal index
if f == 'journal':
return get_nbhits_in_bibwords(p, f)
## construct 'tl' which defines the tag list (MARC tags) to search in:
tl = []
if str(f[0]).isdigit() and str(f[1]).isdigit():
tl.append(f) # 'f' seems to be okay as it starts by two digits
else:
# deduce desired MARC tags on the basis of chosen 'f'
tl = get_field_tags(f)
# start searching:
recIDs = {} # will hold dict of {recID1: 1, recID2: 1, ..., } (unique recIDs, therefore)
for t in tl:
# deduce into which bibxxx table we will search:
digit1, digit2 = int(t[0]), int(t[1])
bx = "bib%d%dx" % (digit1, digit2)
bibx = "bibrec_bib%d%dx" % (digit1, digit2)
if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character:
res = run_sql("""SELECT bibx.id_bibrec FROM %s AS bibx, %s AS bx
WHERE bx.value=%%s AND bx.tag LIKE %%s
AND bibx.id_bibxxx=bx.id""" % (bibx, bx),
(p, t + "%"))
else:
res = run_sql("""SELECT bibx.id_bibrec FROM %s AS bibx, %s AS bx
WHERE bx.value=%%s AND bx.tag=%%s
AND bibx.id_bibxxx=bx.id""" % (bibx, bx),
(p, t))
for row in res:
recIDs[row[0]] = 1
return len(recIDs)
def get_mysql_recid_from_aleph_sysno(sysno):
"""Returns DB's recID for ALEPH sysno passed in the argument (e.g. "002379334CER").
Returns None in case of failure."""
out = None
res = run_sql("""SELECT bb.id_bibrec FROM bibrec_bib97x AS bb, bib97x AS b
WHERE b.value=%s AND b.tag='970__a' AND bb.id_bibxxx=b.id""",
(sysno,))
if res:
out = res[0][0]
return out
def guess_primary_collection_of_a_record(recID):
"""Return primary collection name a record recid belongs to, by
testing 980 identifier.
May lead to bad guesses when a collection is defined dynamically
via dbquery.
In that case, return 'CFG_SITE_NAME'."""
out = CFG_SITE_NAME
dbcollids = get_fieldvalues(recID, "980__a")
if dbcollids:
dbquery = "collection:" + dbcollids[0]
res = run_sql("SELECT name FROM collection WHERE dbquery=%s", (dbquery,))
if res:
out = res[0][0]
if CFG_CERN_SITE:
# dirty hack for ATLAS collections at CERN:
if out in ('ATLAS Communications', 'ATLAS Internal Notes'):
for alternative_collection in ('ATLAS Communications Physics',
'ATLAS Communications General',
'ATLAS Internal Notes Physics',
'ATLAS Internal Notes General',):
if recID in get_collection_reclist(alternative_collection):
out = alternative_collection
break
return out
_re_collection_url = re.compile('/collection/(.+)')
def guess_collection_of_a_record(recID, referer=None):
"""Return collection name a record recid belongs to, by first testing
the referer URL if provided and otherwise returning the
primary collection."""
if referer:
dummy, hostname, path, dummy, query, dummy = urlparse.urlparse(referer)
g = _re_collection_url.match(path)
if g:
name = urllib.unquote_plus(g.group(1))
if recID in get_collection_reclist(name):
return name
elif path.startswith('/search'):
query = cgi.parse_qs(query)
for name in query.get('cc', []) + query.get('c', []):
if recID in get_collection_reclist(name):
return name
return guess_primary_collection_of_a_record(recID)
def get_all_collections_of_a_record(recID):
"""Return all the collection names a record belongs to.
Note this function is O(n_collections)."""
ret = []
for name in collection_reclist_cache.cache.keys():
if recID in get_collection_reclist(name):
ret.append(name)
return ret
def get_tag_name(tag_value, prolog="", epilog=""):
"""Return tag name from the known tag value, by looking up the 'tag' table.
Return empty string in case of failure.
Example: input='100__%', output=first author'."""
out = ""
- res = run_sql_cached("SELECT name FROM tag WHERE value=%s", (tag_value,),
- affected_tables=['tag',])
+ res = run_sql("SELECT name FROM tag WHERE value=%s", (tag_value,))
if res:
out = prolog + res[0][0] + epilog
return out
def get_fieldcodes():
"""Returns a list of field codes that may have been passed as 'search options' in URL.
Example: output=['subject','division']."""
out = []
- res = run_sql_cached("SELECT DISTINCT(code) FROM field",
- affected_tables=['field',])
+ res = run_sql("SELECT DISTINCT(code) FROM field")
for row in res:
out.append(row[0])
return out
def get_field_name(code):
"""Return the corresponding field_name given the field code.
e.g. reportnumber -> report number."""
- res = run_sql_cached("SELECT name FROM field WHERE code=%s", (code, ),
- affected_tables=['field',])
+ res = run_sql("SELECT name FROM field WHERE code=%s", (code, ))
if res:
return res[0][0]
else:
return ""
def get_field_tags(field):
"""Returns a list of MARC tags for the field code 'field'.
Returns empty list in case of error.
Example: field='author', output=['100__%','700__%']."""
out = []
query = """SELECT t.value FROM tag AS t, field_tag AS ft, field AS f
WHERE f.code=%s AND ft.id_field=f.id AND t.id=ft.id_tag
ORDER BY ft.score DESC"""
res = run_sql(query, (field, ))
for val in res:
out.append(val[0])
return out
def get_fieldvalues(recIDs, tag, repetitive_values=True):
"""
Return list of field values for field TAG for the given record ID
or list of record IDs. (RECIDS can be both an integer or a list
of integers.)
If REPETITIVE_VALUES is set to True, then return all values even
if they are doubled. If set to False, then return unique values
only.
"""
out = []
if isinstance(recIDs, (int, long)):
recIDs =[recIDs,]
if not isinstance(recIDs, (list, tuple)):
return []
if len(recIDs) == 0:
return []
if tag == "001___":
# we have asked for tag 001 (=recID) that is not stored in bibXXx tables
out = [str(recID) for recID in recIDs]
else:
# we are going to look inside bibXXx tables
digits = tag[0:2]
try:
intdigits = int(digits)
if intdigits < 0 or intdigits > 99:
raise ValueError
except ValueError:
# invalid tag value asked for
return []
bx = "bib%sx" % digits
bibx = "bibrec_bib%sx" % digits
queryparam = []
for recID in recIDs:
queryparam.append(recID)
if not repetitive_values:
queryselect = "DISTINCT(bx.value)"
else:
queryselect = "bx.value"
query = "SELECT %s FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec IN (%s) " \
" AND bx.id=bibx.id_bibxxx AND bx.tag LIKE %%s " \
" ORDER BY bibx.field_number, bx.tag ASC" % \
(queryselect, bx, bibx, ("%s,"*len(queryparam))[:-1])
res = run_sql(query, tuple(queryparam) + (tag,))
for row in res:
out.append(row[0])
return out
def get_fieldvalues_alephseq_like(recID, tags_in, can_see_hidden=False):
"""Return buffer of ALEPH sequential-like textual format with fields found
in the list TAGS_IN for record RECID.
If can_see_hidden is True, just print everything. Otherwise hide fields
from CFG_BIBFORMAT_HIDDEN_TAGS.
"""
out = ""
if type(tags_in) is not list:
tags_in = [tags_in,]
if len(tags_in) == 1 and len(tags_in[0]) == 6:
## case A: one concrete subfield asked, so print its value if found
## (use with care: can mislead if field has multiple occurrences)
out += string.join(get_fieldvalues(recID, tags_in[0]),"\n")
else:
## case B: print our "text MARC" format; works safely all the time
# find out which tags to output:
dict_of_tags_out = {}
if not tags_in:
for i in range(0, 10):
for j in range(0, 10):
dict_of_tags_out["%d%d%%" % (i, j)] = 1
else:
for tag in tags_in:
if len(tag) == 0:
for i in range(0, 10):
for j in range(0, 10):
dict_of_tags_out["%d%d%%" % (i, j)] = 1
elif len(tag) == 1:
for j in range(0, 10):
dict_of_tags_out["%s%d%%" % (tag, j)] = 1
elif len(tag) < 5:
dict_of_tags_out["%s%%" % tag] = 1
elif tag >= 6:
dict_of_tags_out[tag[0:5]] = 1
tags_out = dict_of_tags_out.keys()
tags_out.sort()
# search all bibXXx tables as needed:
for tag in tags_out:
digits = tag[0:2]
try:
intdigits = int(digits)
if intdigits < 0 or intdigits > 99:
raise ValueError
except ValueError:
# invalid tag value asked for
continue
if tag.startswith("001") or tag.startswith("00%"):
if out:
out += "\n"
out += "%09d %s %d" % (recID, "001__", recID)
bx = "bib%sx" % digits
bibx = "bibrec_bib%sx" % digits
query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\
"WHERE bb.id_bibrec=%%s AND b.id=bb.id_bibxxx AND b.tag LIKE %%s"\
"ORDER BY bb.field_number, b.tag ASC" % (bx, bibx)
res = run_sql(query, (recID, str(tag)+'%'))
# go through fields:
field_number_old = -999
field_old = ""
for row in res:
field, value, field_number = row[0], row[1], row[2]
ind1, ind2 = field[3], field[4]
printme = True
#check the stuff in hiddenfields
if not can_see_hidden:
for htag in CFG_BIBFORMAT_HIDDEN_TAGS:
ltag = len(htag)
samelenfield = field[0:ltag]
if samelenfield == htag:
printme = False
if ind1 == "_":
ind1 = ""
if ind2 == "_":
ind2 = ""
# print field tag
if printme:
if field_number != field_number_old or field[:-1] != field_old[:-1]:
if out:
out += "\n"
out += "%09d %s " % (recID, field[:5])
field_number_old = field_number
field_old = field
# print subfield value
if field[0:2] == "00" and field[-1:] == "_":
out += value
else:
out += "$$%s%s" % (field[-1:], value)
return out
def record_exists(recID):
"""Return 1 if record RECID exists.
Return 0 if it doesn't exist.
Return -1 if it exists but is marked as deleted.
"""
out = 0
res = run_sql("SELECT id FROM bibrec WHERE id=%s", (recID,), 1)
if res:
recID = int(recID)
# record exists; now check whether it isn't marked as deleted:
dbcollids = get_fieldvalues(recID, "980__%")
if ("DELETED" in dbcollids) or (CFG_CERN_SITE and "DUMMY" in dbcollids):
out = -1 # exists, but marked as deleted
else:
out = 1 # exists fine
return out
def record_empty(recID):
"""
Is this record empty, e.g. has only 001, waiting for integration?
@param recID: the record identifier.
@type recID: int
@return: 1 if the record is empty, 0 otherwise.
@rtype: int
"""
record = get_record(recID)
if record is None or len(record) < 2:
return 1
else:
return 0
def record_public_p(recID):
"""Return 1 if the record is public, i.e. if it can be found in the Home collection.
Return 0 otherwise.
"""
return recID in get_collection_reclist(CFG_SITE_NAME)
def get_creation_date(recID, fmt="%Y-%m-%d"):
"Returns the creation date of the record 'recID'."
out = ""
res = run_sql("SELECT DATE_FORMAT(creation_date,%s) FROM bibrec WHERE id=%s", (fmt, recID), 1)
if res:
out = res[0][0]
return out
def get_modification_date(recID, fmt="%Y-%m-%d"):
"Returns the date of last modification for the record 'recID'."
out = ""
res = run_sql("SELECT DATE_FORMAT(modification_date,%s) FROM bibrec WHERE id=%s", (fmt, recID), 1)
if res:
out = res[0][0]
return out
def print_warning(req, msg, type='', prologue='<br />', epilogue='<br />'):
"Prints warning message and flushes output."
if req and msg:
req.write(websearch_templates.tmpl_print_warning(
msg = msg,
type = type,
prologue = prologue,
epilogue = epilogue,
))
return
def print_search_info(p, f, sf, so, sp, rm, of, ot, collection=CFG_SITE_NAME, nb_found=-1, jrec=1, rg=10,
aas=0, ln=CFG_SITE_LANG, p1="", p2="", p3="", f1="", f2="", f3="", m1="", m2="", m3="", op1="", op2="",
sc=1, pl_in_url="",
d1y=0, d1m=0, d1d=0, d2y=0, d2m=0, d2d=0, dt="",
cpu_time=-1, middle_only=0):
"""Prints stripe with the information on 'collection' and 'nb_found' results and CPU time.
Also, prints navigation links (beg/next/prev/end) inside the results set.
If middle_only is set to 1, it will only print the middle box information (beg/netx/prev/end/etc) links.
This is suitable for displaying navigation links at the bottom of the search results page."""
# sanity check:
if jrec < 1:
jrec = 1
if jrec > nb_found:
jrec = max(nb_found-rg+1, 1)
return websearch_templates.tmpl_print_search_info(
ln = ln,
collection = collection,
aas = aas,
collection_name = get_coll_i18nname(collection, ln, False),
collection_id = get_colID(collection),
middle_only = middle_only,
rg = rg,
nb_found = nb_found,
sf = sf,
so = so,
rm = rm,
of = of,
ot = ot,
p = p,
f = f,
p1 = p1,
p2 = p2,
p3 = p3,
f1 = f1,
f2 = f2,
f3 = f3,
m1 = m1,
m2 = m2,
m3 = m3,
op1 = op1,
op2 = op2,
pl_in_url = pl_in_url,
d1y = d1y,
d1m = d1m,
d1d = d1d,
d2y = d2y,
d2m = d2m,
d2d = d2d,
dt = dt,
jrec = jrec,
sc = sc,
sp = sp,
all_fieldcodes = get_fieldcodes(),
cpu_time = cpu_time,
)
def print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, collection=CFG_SITE_NAME, nb_found=-1, jrec=1, rg=10,
aas=0, ln=CFG_SITE_LANG, p1="", p2="", p3="", f1="", f2="", f3="", m1="", m2="", m3="", op1="", op2="",
sc=1, pl_in_url="",
d1y=0, d1m=0, d1d=0, d2y=0, d2m=0, d2d=0, dt="",
cpu_time=-1, middle_only=0):
"""Prints stripe with the information on 'collection' and 'nb_found' results and CPU time.
Also, prints navigation links (beg/next/prev/end) inside the results set.
If middle_only is set to 1, it will only print the middle box information (beg/netx/prev/end/etc) links.
This is suitable for displaying navigation links at the bottom of the search results page."""
out = ""
# sanity check:
if jrec < 1:
jrec = 1
if jrec > nb_found:
jrec = max(nb_found-rg+1, 1)
return websearch_templates.tmpl_print_hosted_search_info(
ln = ln,
collection = collection,
aas = aas,
collection_name = get_coll_i18nname(collection, ln, False),
collection_id = get_colID(collection),
middle_only = middle_only,
rg = rg,
nb_found = nb_found,
sf = sf,
so = so,
rm = rm,
of = of,
ot = ot,
p = p,
f = f,
p1 = p1,
p2 = p2,
p3 = p3,
f1 = f1,
f2 = f2,
f3 = f3,
m1 = m1,
m2 = m2,
m3 = m3,
op1 = op1,
op2 = op2,
pl_in_url = pl_in_url,
d1y = d1y,
d1m = d1m,
d1d = d1d,
d2y = d2y,
d2m = d2m,
d2d = d2d,
dt = dt,
jrec = jrec,
sc = sc,
sp = sp,
all_fieldcodes = get_fieldcodes(),
cpu_time = cpu_time,
)
def print_results_overview(colls, results_final_nb_total, results_final_nb, cpu_time, ln=CFG_SITE_LANG, ec=[], hosted_colls_potential_results_p=False):
"""Prints results overview box with links to particular collections below."""
out = ""
new_colls = []
for coll in colls:
new_colls.append({
'id': get_colID(coll),
'code': coll,
'name': get_coll_i18nname(coll, ln, False),
})
return websearch_templates.tmpl_print_results_overview(
ln = ln,
results_final_nb_total = results_final_nb_total,
results_final_nb = results_final_nb,
cpu_time = cpu_time,
colls = new_colls,
ec = ec,
hosted_colls_potential_results_p = hosted_colls_potential_results_p,
)
def print_hosted_results(url_and_engine, ln=CFG_SITE_LANG, of=None, req=None, no_records_found=False, search_timed_out=False, limit=CFG_EXTERNAL_COLLECTION_MAXRESULTS):
"""Prints the full results of a hosted collection"""
if of.startswith("h"):
if no_records_found:
return "<br />No results found."
if search_timed_out:
return "<br />The search engine did not respond in time."
return websearch_templates.tmpl_print_hosted_results(
url_and_engine=url_and_engine,
ln=ln,
of=of,
req=req,
limit=limit
)
def sort_records(req, recIDs, sort_field='', sort_order='d', sort_pattern='', verbose=0, of='hb', ln=CFG_SITE_LANG):
"""Sort records in 'recIDs' list according sort field 'sort_field' in order 'sort_order'.
If more than one instance of 'sort_field' is found for a given record, try to choose that that is given by
'sort pattern', for example "sort by report number that starts by CERN-PS".
Note that 'sort_field' can be field code like 'author' or MARC tag like '100__a' directly."""
_ = gettext_set_language(ln)
## check arguments:
if not sort_field:
return recIDs
if len(recIDs) > CFG_WEBSEARCH_NB_RECORDS_TO_SORT:
if of.startswith('h'):
print_warning(req, _("Sorry, sorting is allowed on sets of up to %d records only. Using default sort order.") % CFG_WEBSEARCH_NB_RECORDS_TO_SORT, "Warning")
return recIDs
sort_fields = string.split(sort_field, ",")
recIDs_dict = {}
recIDs_out = []
## first deduce sorting MARC tag out of the 'sort_field' argument:
tags = []
for sort_field in sort_fields:
if sort_field and str(sort_field[0:2]).isdigit():
# sort_field starts by two digits, so this is probably a MARC tag already
tags.append(sort_field)
else:
# let us check the 'field' table
query = """SELECT DISTINCT(t.value) FROM tag AS t, field_tag AS ft, field AS f
WHERE f.code=%s AND ft.id_field=f.id AND t.id=ft.id_tag
ORDER BY ft.score DESC"""
res = run_sql(query, (sort_field, ))
if res:
for row in res:
tags.append(row[0])
else:
if of.startswith('h'):
print_warning(req, _("Sorry, %s does not seem to be a valid sort option. Choosing title sort instead.") % cgi.escape(sort_field), "Error")
tags.append("245__a")
if verbose >= 3:
print_warning(req, "Sorting by tags %s." % cgi.escape(repr(tags)))
if sort_pattern:
print_warning(req, "Sorting preferentially by %s." % cgi.escape(sort_pattern))
## check if we have sorting tag defined:
if tags:
# fetch the necessary field values:
for recID in recIDs:
val = "" # will hold value for recID according to which sort
vals = [] # will hold all values found in sorting tag for recID
for tag in tags:
vals.extend(get_fieldvalues(recID, tag))
if sort_pattern:
# try to pick that tag value that corresponds to sort pattern
bingo = 0
for v in vals:
if v.lower().startswith(sort_pattern.lower()): # bingo!
bingo = 1
val = v
break
if not bingo: # sort_pattern not present, so add other vals after spaces
val = sort_pattern + " " + string.join(vals)
else:
# no sort pattern defined, so join them all together
val = string.join(vals)
val = strip_accents(val.lower()) # sort values regardless of accents and case
if recIDs_dict.has_key(val):
recIDs_dict[val].append(recID)
else:
recIDs_dict[val] = [recID]
# sort them:
recIDs_dict_keys = recIDs_dict.keys()
recIDs_dict_keys.sort()
# now that keys are sorted, create output array:
for k in recIDs_dict_keys:
for s in recIDs_dict[k]:
recIDs_out.append(s)
# ascending or descending?
if sort_order == 'a':
recIDs_out.reverse()
# okay, we are done
return recIDs_out
else:
# good, no sort needed
return recIDs
def print_records(req, recIDs, jrec=1, rg=10, format='hb', ot='', ln=CFG_SITE_LANG, relevances=[], relevances_prologue="(", relevances_epilogue="%%)", decompress=zlib.decompress, search_pattern='', print_records_prologue_p=True, print_records_epilogue_p=True, verbose=0, tab='', sf='', so='d', sp='', rm=''):
"""
Prints list of records 'recIDs' formatted according to 'format' in
groups of 'rg' starting from 'jrec'.
Assumes that the input list 'recIDs' is sorted in reverse order,
so it counts records from tail to head.
A value of 'rg=-9999' means to print all records: to be used with care.
Print also list of RELEVANCES for each record (if defined), in
between RELEVANCE_PROLOGUE and RELEVANCE_EPILOGUE.
Print prologue and/or epilogue specific to 'format' if
'print_records_prologue_p' and/or print_records_epilogue_p' are
True.
'sf' is sort field and 'rm' is ranking method that are passed here
only for proper linking purposes: e.g. when a certain ranking
method or a certain sort field was selected, keep it selected in
any dynamic search links that may be printed.
"""
# load the right message language
_ = gettext_set_language(ln)
# sanity checking:
if req is None:
return
# get user_info (for formatting based on user)
if isinstance(req, cStringIO.OutputType):
user_info = {}
else:
user_info = collect_user_info(req)
if len(recIDs):
nb_found = len(recIDs)
if rg == -9999: # print all records
rg = nb_found
else:
rg = abs(rg)
if jrec < 1: # sanity checks
jrec = 1
if jrec > nb_found:
jrec = max(nb_found-rg+1, 1)
# will print records from irec_max to irec_min excluded:
irec_max = nb_found - jrec
irec_min = nb_found - jrec - rg
if irec_min < 0:
irec_min = -1
if irec_max >= nb_found:
irec_max = nb_found - 1
#req.write("%s:%d-%d" % (recIDs, irec_min, irec_max))
if format.startswith('x'):
# print header if needed
if print_records_prologue_p:
print_records_prologue(req, format)
# print records
recIDs_to_print = [recIDs[x] for x in range(irec_max, irec_min, -1)]
format_records(recIDs_to_print,
format,
ln=ln,
search_pattern=search_pattern,
record_separator="\n",
user_info=user_info,
req=req)
# print footer if needed
if print_records_epilogue_p:
print_records_epilogue(req, format)
elif format.startswith('t') or str(format[0:3]).isdigit():
# we are doing plain text output:
for irec in range(irec_max, irec_min, -1):
x = print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
req.write(x)
if x:
req.write('\n')
elif format == 'excel':
recIDs_to_print = [recIDs[x] for x in range(irec_max, irec_min, -1)]
create_excel(recIDs=recIDs_to_print, req=req, ln=ln, ot=ot)
else:
# we are doing HTML output:
if format == 'hp' or format.startswith("hb_") or format.startswith("hd_"):
# portfolio and on-the-fly formats:
for irec in range(irec_max, irec_min, -1):
req.write(print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm))
elif format.startswith("hb"):
# HTML brief format:
display_add_to_basket = True
if user_info:
if user_info['email'] == 'guest':
if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS > 4:
display_add_to_basket = False
else:
if not user_info['precached_usebaskets']:
display_add_to_basket = False
req.write(websearch_templates.tmpl_record_format_htmlbrief_header(
ln = ln))
for irec in range(irec_max, irec_min, -1):
row_number = jrec+irec_max-irec
recid = recIDs[irec]
if relevances and relevances[irec]:
relevance = relevances[irec]
else:
relevance = ''
record = print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
req.write(websearch_templates.tmpl_record_format_htmlbrief_body(
ln = ln,
recid = recid,
row_number = row_number,
relevance = relevance,
record = record,
relevances_prologue = relevances_prologue,
relevances_epilogue = relevances_epilogue,
display_add_to_basket = display_add_to_basket
))
req.write(websearch_templates.tmpl_record_format_htmlbrief_footer(
ln = ln,
display_add_to_basket = display_add_to_basket))
elif format.startswith("hd"):
# HTML detailed format:
for irec in range(irec_max, irec_min, -1):
if record_exists(recIDs[irec]) == -1:
print_warning(req, _("The record has been deleted."))
continue
unordered_tabs = get_detailed_page_tabs(get_colID(guess_primary_collection_of_a_record(recIDs[irec])),
recIDs[irec], ln=ln)
ordered_tabs_id = [(tab_id, values['order']) for (tab_id, values) in unordered_tabs.iteritems()]
ordered_tabs_id.sort(lambda x,y: cmp(x[1],y[1]))
link_ln = ''
if ln != CFG_SITE_LANG:
link_ln = '?ln=%s' % ln
recid = recIDs[irec]
recid_to_display = recid # Record ID used to build the URL.
if CFG_WEBSEARCH_USE_ALEPH_SYSNOS:
try:
recid_to_display = get_fieldvalues(recid,
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG)[0]
except IndexError:
# No external sysno is available, keep using
# internal recid.
pass
citedbynum = 0 #num of citations, to be shown in the cit tab
references = -1 #num of references
citedbynum = get_cited_by_count(recid)
reftag = ""
reftags = get_field_tags("reference")
if reftags:
reftag = reftags[0]
tmprec = get_record(recid)
if reftag and len(reftag) > 4:
references = len(record_get_field_instances(tmprec, reftag[0:3], reftag[3], reftag[4]))
tabs = [(unordered_tabs[tab_id]['label'], \
'%s/record/%s/%s%s' % (CFG_SITE_URL, recid_to_display, tab_id, link_ln), \
tab_id == tab,
unordered_tabs[tab_id]['enabled']) \
for (tab_id, order) in ordered_tabs_id
if unordered_tabs[tab_id]['visible'] == True]
# load content
if tab == 'usage':
req.write(webstyle_templates.detailed_record_container_top(recIDs[irec],
tabs,
ln,
citationnum=citedbynum,
referencenum=references))
r = calculate_reading_similarity_list(recIDs[irec], "downloads")
downloadsimilarity = None
downloadhistory = None
#if r:
# downloadsimilarity = r
if CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS:
downloadhistory = create_download_history_graph_and_box(recIDs[irec], ln)
r = calculate_reading_similarity_list(recIDs[irec], "pageviews")
viewsimilarity = None
if r: viewsimilarity = r
content = websearch_templates.tmpl_detailed_record_statistics(recIDs[irec],
ln,
downloadsimilarity=downloadsimilarity,
downloadhistory=downloadhistory,
viewsimilarity=viewsimilarity)
req.write(content)
req.write(webstyle_templates.detailed_record_container_bottom(recIDs[irec],
tabs,
ln))
elif tab == 'citations':
recid = recIDs[irec]
req.write(webstyle_templates.detailed_record_container_top(recid,
tabs,
ln,
citationnum=citedbynum,
referencenum=references))
req.write(websearch_templates.tmpl_detailed_record_citations_prologue(recid, ln))
# Citing
citinglist = calculate_cited_by_list(recid)
req.write(websearch_templates.tmpl_detailed_record_citations_citing_list(recid,
ln,
citinglist,
sf=sf,
so=so,
sp=sp,
rm=rm))
# Self-cited
selfcited = get_self_cited_by(recid)
req.write(websearch_templates.tmpl_detailed_record_citations_self_cited(recid,
ln, selfcited=selfcited, citinglist=citinglist))
# Co-cited
s = calculate_co_cited_with_list(recid)
cociting = None
if s:
cociting = s
req.write(websearch_templates.tmpl_detailed_record_citations_co_citing(recid,
ln,
cociting=cociting))
# Citation history, if needed
citationhistory = None
if citinglist:
citationhistory = create_citation_history_graph_and_box(recid, ln)
#debug
if verbose > 3:
print_warning(req, "Citation graph debug: " + \
str(len(citationhistory)))
req.write(websearch_templates.tmpl_detailed_record_citations_citation_history(recid, ln, citationhistory))
req.write(websearch_templates.tmpl_detailed_record_citations_epilogue(recid, ln))
req.write(webstyle_templates.detailed_record_container_bottom(recid,
tabs,
ln))
elif tab == 'references':
req.write(webstyle_templates.detailed_record_container_top(recIDs[irec],
tabs,
ln,
citationnum=citedbynum,
referencenum=references))
req.write(format_record(recIDs[irec], 'HDREF', ln=ln, user_info=user_info, verbose=verbose))
req.write(webstyle_templates.detailed_record_container_bottom(recIDs[irec],
tabs,
ln))
elif tab == 'keywords':
from invenio.bibclassify_webinterface import \
record_get_keywords, get_sorting_options, \
generate_keywords, get_keywords_body
from invenio.webinterface_handler import wash_urlargd
form = req.form
argd = wash_urlargd(form, {
'generate': (str, 'no'),
'sort': (str, 'occurrences'),
'type': (str, 'tagcloud'),
'numbering': (str, 'off'),
})
recid = recIDs[irec]
req.write(webstyle_templates.detailed_record_container_top(recid,
tabs, ln, citationnum=citedbynum, referencenum=references))
if argd['generate'] == 'yes':
# The user asked to generate the keywords.
keywords = generate_keywords(req, recid)
else:
# Get the keywords contained in the MARC.
keywords = record_get_keywords(recid, argd)
if keywords:
req.write(get_sorting_options(argd, keywords))
elif argd['sort'] == 'related' and not keywords:
req.write('You may want to run BibIndex.')
# Output the keywords or the generate button.
get_keywords_body(keywords, req, recid, argd)
req.write(webstyle_templates.detailed_record_container_bottom(recid,
tabs, ln))
elif tab == 'plots':
req.write(webstyle_templates.detailed_record_container_top(recIDs[irec],
tabs,
ln))
content = websearch_templates.tmpl_record_plots(
recID=recIDs[irec],
ln=ln)
req.write(content)
req.write(webstyle_templates.detailed_record_container_bottom(recIDs[irec],
tabs,
ln))
else:
# Metadata tab
req.write(webstyle_templates.detailed_record_container_top(recIDs[irec],
tabs,
ln,
show_short_rec_p=False,
citationnum=citedbynum, referencenum=references))
creationdate = None
modificationdate = None
if record_exists(recIDs[irec]) == 1:
creationdate = get_creation_date(recIDs[irec])
modificationdate = get_modification_date(recIDs[irec])
content = print_record(recIDs[irec], format, ot, ln,
search_pattern=search_pattern,
user_info=user_info, verbose=verbose,
sf=sf, so=so, sp=sp, rm=rm)
content = websearch_templates.tmpl_detailed_record_metadata(
recID = recIDs[irec],
ln = ln,
format = format,
creationdate = creationdate,
modificationdate = modificationdate,
content = content)
req.write(content)
req.write(webstyle_templates.detailed_record_container_bottom(recIDs[irec],
tabs,
ln,
creationdate=creationdate,
modificationdate=modificationdate,
show_short_rec_p=False))
if len(tabs) > 0:
# Add the mini box at bottom of the page
if CFG_WEBCOMMENT_ALLOW_REVIEWS:
from invenio.webcomment import get_mini_reviews
reviews = get_mini_reviews(recid = recIDs[irec], ln=ln)
else:
reviews = ''
actions = format_record(recIDs[irec], 'HDACT', ln=ln, user_info=user_info, verbose=verbose)
files = format_record(recIDs[irec], 'HDFILE', ln=ln, user_info=user_info, verbose=verbose)
req.write(webstyle_templates.detailed_record_mini_panel(recIDs[irec],
ln,
format,
files=files,
reviews=reviews,
actions=actions))
else:
# Other formats
for irec in range(irec_max, irec_min, -1):
req.write(print_record(recIDs[irec], format, ot, ln,
search_pattern=search_pattern,
user_info=user_info, verbose=verbose,
sf=sf, so=so, sp=sp, rm=rm))
else:
print_warning(req, _("Use different search terms."))
def print_records_prologue(req, format):
"""
Print the appropriate prologue for list of records in the given
format.
"""
prologue = "" # no prologue needed for HTML or Text formats
if format.startswith('xm'):
prologue = websearch_templates.tmpl_xml_marc_prologue()
elif format.startswith('xn'):
prologue = websearch_templates.tmpl_xml_nlm_prologue()
elif format.startswith('xw'):
prologue = websearch_templates.tmpl_xml_refworks_prologue()
elif format.startswith('xr'):
prologue = websearch_templates.tmpl_xml_rss_prologue()
elif format.startswith('xe'):
prologue = websearch_templates.tmpl_xml_endnote_prologue()
elif format.startswith('xo'):
prologue = websearch_templates.tmpl_xml_mods_prologue()
elif format.startswith('x'):
prologue = websearch_templates.tmpl_xml_default_prologue()
req.write(prologue)
def print_records_epilogue(req, format):
"""
Print the appropriate epilogue for list of records in the given
format.
"""
epilogue = "" # no epilogue needed for HTML or Text formats
if format.startswith('xm'):
epilogue = websearch_templates.tmpl_xml_marc_epilogue()
elif format.startswith('xn'):
epilogue = websearch_templates.tmpl_xml_nlm_epilogue()
elif format.startswith('xw'):
epilogue = websearch_templates.tmpl_xml_refworks_epilogue()
elif format.startswith('xr'):
epilogue = websearch_templates.tmpl_xml_rss_epilogue()
elif format.startswith('xe'):
epilogue = websearch_templates.tmpl_xml_endnote_epilogue()
elif format.startswith('xo'):
epilogue = websearch_templates.tmpl_xml_mods_epilogue()
elif format.startswith('x'):
epilogue = websearch_templates.tmpl_xml_default_epilogue()
req.write(epilogue)
def get_record(recid):
"""Directly the record object corresponding to the recid."""
if CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE:
value = run_sql("SELECT value FROM bibfmt WHERE id_bibrec=%s AND FORMAT='recstruct'", (recid, ))
if value:
try:
return deserialize_via_marshal(value[0][0])
except:
### In case of corruption, let's rebuild it!
pass
return create_record(print_record(recid, 'xm'))[0]
def print_record(recID, format='hb', ot='', ln=CFG_SITE_LANG, decompress=zlib.decompress,
search_pattern=None, user_info=None, verbose=0, sf='', so='d', sp='', rm=''):
"""
Prints record 'recID' formatted according to 'format'.
'sf' is sort field and 'rm' is ranking method that are passed here
only for proper linking purposes: e.g. when a certain ranking
method or a certain sort field was selected, keep it selected in
any dynamic search links that may be printed.
"""
if format == 'recstruct':
return get_record(recID)
_ = gettext_set_language(ln)
#check from user information if the user has the right to see hidden fields/tags in the
#records as well
can_see_hidden = (acc_authorize_action(user_info, 'runbibedit')[0] == 0)
out = ""
# sanity check:
record_exist_p = record_exists(recID)
if record_exist_p == 0: # doesn't exist
return out
# New Python BibFormat procedure for formatting
# Old procedure follows further below
# We must still check some special formats, but these
# should disappear when BibFormat improves.
if not (CFG_BIBFORMAT_USE_OLD_BIBFORMAT \
or format.lower().startswith('t') \
or format.lower().startswith('hm') \
or str(format[0:3]).isdigit() \
or ot):
# Unspecified format is hd
if format == '':
format = 'hd'
if record_exist_p == -1 and get_output_format_content_type(format) == 'text/html':
# HTML output displays a default value for deleted records.
# Other format have to deal with it.
out += _("The record has been deleted.")
else:
out += call_bibformat(recID, format, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose)
# at the end of HTML brief mode, print the "Detailed record" functionality:
if format.lower().startswith('hb') and \
format.lower() != 'hb_p':
out += websearch_templates.tmpl_print_record_brief_links(ln=ln,
recID=recID,
sf=sf,
so=so,
sp=sp,
rm=rm)
return out
# Old PHP BibFormat procedure for formatting
# print record opening tags, if needed:
if format == "marcxml" or format == "oai_dc":
out += " <record>\n"
out += " <header>\n"
for oai_id in get_fieldvalues(recID, CFG_OAI_ID_FIELD):
out += " <identifier>%s</identifier>\n" % oai_id
out += " <datestamp>%s</datestamp>\n" % get_modification_date(recID)
out += " </header>\n"
out += " <metadata>\n"
if format.startswith("xm") or format == "marcxml":
# look for detailed format existence:
query = "SELECT value FROM bibfmt WHERE id_bibrec=%s AND format=%s"
res = run_sql(query, (recID, format), 1)
if res and record_exist_p == 1:
# record 'recID' is formatted in 'format', so print it
out += "%s" % decompress(res[0][0])
else:
# record 'recID' is not formatted in 'format' -- they are not in "bibfmt" table; so fetch all the data from "bibXXx" tables:
if format == "marcxml":
out += """ <record xmlns="http://www.loc.gov/MARC21/slim">\n"""
out += " <controlfield tag=\"001\">%d</controlfield>\n" % int(recID)
elif format.startswith("xm"):
out += """ <record>\n"""
out += " <controlfield tag=\"001\">%d</controlfield>\n" % int(recID)
if record_exist_p == -1:
# deleted record, so display only OAI ID and 980:
oai_ids = get_fieldvalues(recID, CFG_OAI_ID_FIELD)
if oai_ids:
out += "<datafield tag=\"%s\" ind1=\"%s\" ind2=\"%s\"><subfield code=\"%s\">%s</subfield></datafield>\n" % \
(CFG_OAI_ID_FIELD[0:3], CFG_OAI_ID_FIELD[3:4], CFG_OAI_ID_FIELD[4:5], CFG_OAI_ID_FIELD[5:6], oai_ids[0])
out += "<datafield tag=\"980\" ind1=\"\" ind2=\"\"><subfield code=\"c\">DELETED</subfield></datafield>\n"
else:
# controlfields
query = "SELECT b.tag,b.value,bb.field_number FROM bib00x AS b, bibrec_bib00x AS bb "\
"WHERE bb.id_bibrec=%s AND b.id=bb.id_bibxxx AND b.tag LIKE '00%%' "\
"ORDER BY bb.field_number, b.tag ASC"
res = run_sql(query, (recID, ))
for row in res:
field, value = row[0], row[1]
value = encode_for_xml(value)
out += """ <controlfield tag="%s" >%s</controlfield>\n""" % \
(encode_for_xml(field[0:3]), value)
# datafields
i = 1 # Do not process bib00x and bibrec_bib00x, as
# they are controlfields. So start at bib01x and
# bibrec_bib00x (and set i = 0 at the end of
# first loop)
for digit1 in range(0, 10):
for digit2 in range(i, 10):
bx = "bib%d%dx" % (digit1, digit2)
bibx = "bibrec_bib%d%dx" % (digit1, digit2)
query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\
"WHERE bb.id_bibrec=%%s AND b.id=bb.id_bibxxx AND b.tag LIKE %%s"\
"ORDER BY bb.field_number, b.tag ASC" % (bx, bibx)
res = run_sql(query, (recID, str(digit1)+str(digit2)+'%'))
field_number_old = -999
field_old = ""
for row in res:
field, value, field_number = row[0], row[1], row[2]
ind1, ind2 = field[3], field[4]
if ind1 == "_" or ind1 == "":
ind1 = " "
if ind2 == "_" or ind2 == "":
ind2 = " "
# print field tag, unless hidden
printme = True
if not can_see_hidden:
for htag in CFG_BIBFORMAT_HIDDEN_TAGS:
ltag = len(htag)
samelenfield = field[0:ltag]
if samelenfield == htag:
printme = False
if printme:
if field_number != field_number_old or field[:-1] != field_old[:-1]:
if field_number_old != -999:
out += """ </datafield>\n"""
out += """ <datafield tag="%s" ind1="%s" ind2="%s">\n""" % \
(encode_for_xml(field[0:3]), encode_for_xml(ind1), encode_for_xml(ind2))
field_number_old = field_number
field_old = field
# print subfield value
value = encode_for_xml(value)
out += """ <subfield code="%s">%s</subfield>\n""" % \
(encode_for_xml(field[-1:]), value)
# all fields/subfields printed in this run, so close the tag:
if field_number_old != -999:
out += """ </datafield>\n"""
i = 0 # Next loop should start looking at bib%0 and bibrec_bib00x
# we are at the end of printing the record:
out += " </record>\n"
elif format == "xd" or format == "oai_dc":
# XML Dublin Core format, possibly OAI -- select only some bibXXx fields:
out += """ <dc xmlns="http://purl.org/dc/elements/1.1/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://purl.org/dc/elements/1.1/
http://www.openarchives.org/OAI/1.1/dc.xsd">\n"""
if record_exist_p == -1:
out += ""
else:
for f in get_fieldvalues(recID, "041__a"):
out += " <language>%s</language>\n" % f
for f in get_fieldvalues(recID, "100__a"):
out += " <creator>%s</creator>\n" % encode_for_xml(f)
for f in get_fieldvalues(recID, "700__a"):
out += " <creator>%s</creator>\n" % encode_for_xml(f)
for f in get_fieldvalues(recID, "245__a"):
out += " <title>%s\n" % encode_for_xml(f)
for f in get_fieldvalues(recID, "65017a"):
out += " %s\n" % encode_for_xml(f)
for f in get_fieldvalues(recID, "8564_u"):
if f.split('.') == 'png':
continue
out += " %s\n" % encode_for_xml(f)
for f in get_fieldvalues(recID, "520__a"):
out += " %s\n" % encode_for_xml(f)
out += " %s\n" % get_creation_date(recID)
out += " \n"
elif len(format) == 6 and str(format[0:3]).isdigit():
# user has asked to print some fields only
if format == "001":
out += "%s\n" % (format, recID, format)
else:
vals = get_fieldvalues(recID, format)
for val in vals:
out += "%s\n" % (format, val, format)
elif format.startswith('t'):
## user directly asked for some tags to be displayed only
if record_exist_p == -1:
out += get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"], can_see_hidden)
else:
out += get_fieldvalues_alephseq_like(recID, ot, can_see_hidden)
elif format == "hm":
if record_exist_p == -1:
out += "\n" + cgi.escape(get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"], can_see_hidden)) + "
"
else:
out += "\n" + cgi.escape(get_fieldvalues_alephseq_like(recID, ot, can_see_hidden)) + "
"
elif format.startswith("h") and ot:
## user directly asked for some tags to be displayed only
if record_exist_p == -1:
out += "\n" + get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"], can_see_hidden) + "
"
else:
out += "\n" + get_fieldvalues_alephseq_like(recID, ot, can_see_hidden) + "
"
elif format == "hd":
# HTML detailed format
if record_exist_p == -1:
out += _("The record has been deleted.")
else:
# look for detailed format existence:
query = "SELECT value FROM bibfmt WHERE id_bibrec=%s AND format=%s"
res = run_sql(query, (recID, format), 1)
if res:
# record 'recID' is formatted in 'format', so print it
out += "%s" % decompress(res[0][0])
else:
# record 'recID' is not formatted in 'format', so try to call BibFormat on the fly or use default format:
out_record_in_format = call_bibformat(recID, format, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose)
if out_record_in_format:
out += out_record_in_format
else:
out += websearch_templates.tmpl_print_record_detailed(
ln = ln,
recID = recID,
)
elif format.startswith("hb_") or format.startswith("hd_"):
# underscore means that HTML brief/detailed formats should be called on-the-fly; suitable for testing formats
if record_exist_p == -1:
out += _("The record has been deleted.")
else:
out += call_bibformat(recID, format, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose)
elif format.startswith("hx"):
# BibTeX format, called on the fly:
if record_exist_p == -1:
out += _("The record has been deleted.")
else:
out += call_bibformat(recID, format, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose)
elif format.startswith("hs"):
# for citation/download similarity navigation links:
if record_exist_p == -1:
out += _("The record has been deleted.")
else:
out += '' % websearch_templates.build_search_url(recid=recID, ln=ln)
# firstly, title:
titles = get_fieldvalues(recID, "245__a")
if titles:
for title in titles:
out += "%s" % title
else:
# usual title not found, try conference title:
titles = get_fieldvalues(recID, "111__a")
if titles:
for title in titles:
out += "%s" % title
else:
# just print record ID:
out += "%s %d" % (get_field_i18nname("record ID", ln, False), recID)
out += ""
# secondly, authors:
authors = get_fieldvalues(recID, "100__a") + get_fieldvalues(recID, "700__a")
if authors:
out += " - %s" % authors[0]
if len(authors) > 1:
out += " et al"
# thirdly publication info:
publinfos = get_fieldvalues(recID, "773__s")
if not publinfos:
publinfos = get_fieldvalues(recID, "909C4s")
if not publinfos:
publinfos = get_fieldvalues(recID, "037__a")
if not publinfos:
publinfos = get_fieldvalues(recID, "088__a")
if publinfos:
out += " - %s" % publinfos[0]
else:
# fourthly publication year (if not publication info):
years = get_fieldvalues(recID, "773__y")
if not years:
years = get_fieldvalues(recID, "909C4y")
if not years:
years = get_fieldvalues(recID, "260__c")
if years:
out += " (%s)" % years[0]
else:
# HTML brief format by default
if record_exist_p == -1:
out += _("The record has been deleted.")
else:
query = "SELECT value FROM bibfmt WHERE id_bibrec=%s AND format=%s"
res = run_sql(query, (recID, format))
if res:
# record 'recID' is formatted in 'format', so print it
out += "%s" % decompress(res[0][0])
else:
# record 'recID' is not formatted in 'format', so try to call BibFormat on the fly: or use default format:
if CFG_WEBSEARCH_CALL_BIBFORMAT:
out_record_in_format = call_bibformat(recID, format, ln, search_pattern=search_pattern,
user_info=user_info, verbose=verbose)
if out_record_in_format:
out += out_record_in_format
else:
out += websearch_templates.tmpl_print_record_brief(
ln = ln,
recID = recID,
)
else:
out += websearch_templates.tmpl_print_record_brief(
ln = ln,
recID = recID,
)
# at the end of HTML brief mode, print the "Detailed record" functionality:
if format == 'hp' or format.startswith("hb_") or format.startswith("hd_"):
pass # do nothing for portfolio and on-the-fly formats
else:
out += websearch_templates.tmpl_print_record_brief_links(ln=ln,
recID=recID,
sf=sf,
so=so,
sp=sp,
rm=rm)
# print record closing tags, if needed:
if format == "marcxml" or format == "oai_dc":
out += " \n"
out += " \n"
return out
def call_bibformat(recID, format="HD", ln=CFG_SITE_LANG, search_pattern=None, user_info=None, verbose=0):
"""
Calls BibFormat and returns formatted record.
BibFormat will decide by itself if old or new BibFormat must be used.
"""
from invenio.bibformat_utils import get_pdf_snippets
keywords = []
if search_pattern is not None:
units = create_basic_search_units(None, str(search_pattern), None)
keywords = [unit[1] for unit in units if unit[0] != '-']
out = format_record(recID,
of=format,
ln=ln,
search_pattern=keywords,
user_info=user_info,
verbose=verbose)
if CFG_WEBSEARCH_FULLTEXT_SNIPPETS and user_info and \
'fulltext' in user_info['uri']:
# check snippets only if URL contains fulltext
# FIXME: make it work for CLI too, via new function arg
if keywords:
snippets = get_pdf_snippets(recID, keywords)
if snippets:
out += snippets
return out
def log_query(hostname, query_args, uid=-1):
"""
Log query into the query and user_query tables.
Return id_query or None in case of problems.
"""
id_query = None
if uid >= 0:
# log the query only if uid is reasonable
res = run_sql("SELECT id FROM query WHERE urlargs=%s", (query_args,), 1)
try:
id_query = res[0][0]
except:
id_query = run_sql("INSERT INTO query (type, urlargs) VALUES ('r', %s)", (query_args,))
if id_query:
run_sql("INSERT INTO user_query (id_user, id_query, hostname, date) VALUES (%s, %s, %s, %s)",
(uid, id_query, hostname,
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
return id_query
def log_query_info(action, p, f, colls, nb_records_found_total=-1):
"""Write some info to the log file for later analysis."""
try:
log = open(CFG_LOGDIR + "/search.log", "a")
log.write(time.strftime("%Y%m%d%H%M%S#", time.localtime()))
log.write(action+"#")
log.write(p+"#")
log.write(f+"#")
for coll in colls[:-1]:
log.write("%s," % coll)
log.write("%s#" % colls[-1])
log.write("%d" % nb_records_found_total)
log.write("\n")
log.close()
except:
pass
return
### CALLABLES
def perform_request_search(req=None, cc=CFG_SITE_NAME, c=None, p="", f="", rg=CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS, sf="", so="d", sp="", rm="", of="id", ot="", aas=0,
p1="", f1="", m1="", op1="", p2="", f2="", m2="", op2="", p3="", f3="", m3="", sc=0, jrec=0,
recid=-1, recidb=-1, sysno="", id=-1, idb=-1, sysnb="", action="", d1="",
d1y=0, d1m=0, d1d=0, d2="", d2y=0, d2m=0, d2d=0, dt="", verbose=0, ap=0, ln=CFG_SITE_LANG, ec=None, tab=""):
"""Perform search or browse request, without checking for
authentication. Return list of recIDs found, if of=id.
Otherwise create web page.
The arguments are as follows:
req - mod_python Request class instance.
cc - current collection (e.g. "ATLAS"). The collection the
user started to search/browse from.
c - collection list (e.g. ["Theses", "Books"]). The
collections user may have selected/deselected when
starting to search from 'cc'.
p - pattern to search for (e.g. "ellis and muon or kaon").
f - field to search within (e.g. "author").
rg - records in groups of (e.g. "10"). Defines how many hits
per collection in the search results page are
displayed.
sf - sort field (e.g. "title").
so - sort order ("a"=ascending, "d"=descending).
sp - sort pattern (e.g. "CERN-") -- in case there are more
values in a sort field, this argument tells which one
to prefer
rm - ranking method (e.g. "jif"). Defines whether results
should be ranked by some known ranking method.
of - output format (e.g. "hb"). Usually starting "h" means
HTML output (and "hb" for HTML brief, "hd" for HTML
detailed), "x" means XML output, "t" means plain text
output, "id" means no output at all but to return list
of recIDs found. (Suitable for high-level API.)
ot - output only these MARC tags (e.g. "100,700,909C0b").
Useful if only some fields are to be shown in the
output, e.g. for library to control some fields.
aas - advanced search ("0" means no, "1" means yes). Whether
search was called from within the advanced search
interface.
p1 - first pattern to search for in the advanced search
interface. Much like 'p'.
f1 - first field to search within in the advanced search
interface. Much like 'f'.
m1 - first matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
op1 - first operator, to join the first and the second unit
in the advanced search interface. ("a" add, "o" or,
"n" not).
p2 - second pattern to search for in the advanced search
interface. Much like 'p'.
f2 - second field to search within in the advanced search
interface. Much like 'f'.
m2 - second matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
op2 - second operator, to join the second and the third unit
in the advanced search interface. ("a" add, "o" or,
"n" not).
p3 - third pattern to search for in the advanced search
interface. Much like 'p'.
f3 - third field to search within in the advanced search
interface. Much like 'f'.
m3 - third matching type in the advanced search interface.
("a" all of the words, "o" any of the words, "e" exact
phrase, "p" partial phrase, "r" regular expression).
sc - split by collection ("0" no, "1" yes). Governs whether
we want to present the results in a single huge list,
or splitted by collection.
jrec - jump to record (e.g. "234"). Used for navigation
inside the search results.
recid - display record ID (e.g. "20000"). Do not
search/browse but go straight away to the Detailed
record page for the given recID.
recidb - display record ID bis (e.g. "20010"). If greater than
'recid', then display records from recid to recidb.
Useful for example for dumping records from the
database for reformatting.
sysno - display old system SYS number (e.g. ""). If you
migrate to CDS Invenio from another system, and store your
old SYS call numbers, you can use them instead of recid
if you wish so.
id - the same as recid, in case recid is not set. For
backwards compatibility.
idb - the same as recid, in case recidb is not set. For
backwards compatibility.
sysnb - the same as sysno, in case sysno is not set. For
backwards compatibility.
action - action to do. "SEARCH" for searching, "Browse" for
browsing. Default is to search.
d1 - first datetime in full YYYY-mm-dd HH:MM:DD format
(e.g. "1998-08-23 12:34:56"). Useful for search limits
on creation/modification date (see 'dt' argument
below). Note that 'd1' takes precedence over d1y, d1m,
d1d if these are defined.
d1y - first date's year (e.g. "1998"). Useful for search
limits on creation/modification date.
d1m - first date's month (e.g. "08"). Useful for search
limits on creation/modification date.
d1d - first date's day (e.g. "23"). Useful for search
limits on creation/modification date.
d2 - second datetime in full YYYY-mm-dd HH:MM:DD format
(e.g. "1998-09-02 12:34:56"). Useful for search limits
on creation/modification date (see 'dt' argument
below). Note that 'd2' takes precedence over d2y, d2m,
d2d if these are defined.
d2y - second date's year (e.g. "1998"). Useful for search
limits on creation/modification date.
d2m - second date's month (e.g. "09"). Useful for search
limits on creation/modification date.
d2d - second date's day (e.g. "02"). Useful for search
limits on creation/modification date.
dt - first and second date's type (e.g. "c"). Specifies
whether to search in creation dates ("c") or in
modification dates ("m"). When dt is not set and d1*
and d2* are set, the default is "c".
verbose - verbose level (0=min, 9=max). Useful to print some
internal information on the searching process in case
something goes wrong.
ap - alternative patterns (0=no, 1=yes). In case no exact
match is found, the search engine can try alternative
patterns e.g. to replace non-alphanumeric characters by
a boolean query. ap defines if this is wanted.
ln - language of the search interface (e.g. "en"). Useful
for internationalization.
ec - list of external search engines to search as well
(e.g. "SPIRES HEP").
"""
selected_external_collections_infos = None
# wash output format:
of = wash_output_format(of)
# raise an exception when trying to print out html from the cli
if of.startswith("h"):
assert req
# for every search engine request asking for an HTML output, we
# first regenerate cache of collection and field I18N names if
# needed; so that later we won't bother checking timestamps for
# I18N names at all:
if of.startswith("h"):
collection_i18nname_cache.recreate_cache_if_needed()
field_i18nname_cache.recreate_cache_if_needed()
# wash all arguments requiring special care
try:
(cc, colls_to_display, colls_to_search, hosted_colls, wash_colls_debug) = wash_colls(cc, c, sc, verbose) # which colls to search and to display?
except InvenioWebSearchUnknownCollectionError, exc:
colname = exc.colname
if of.startswith("h"):
page_start(req, of, cc, aas, ln, getUid(req),
websearch_templates.tmpl_collection_not_found_page_title(colname, ln))
req.write(websearch_templates.tmpl_collection_not_found_page_body(colname, ln))
return page_end(req, of, ln)
elif of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
else:
return page_end(req, of, ln)
p = wash_pattern(p)
f = wash_field(f)
p1 = wash_pattern(p1)
f1 = wash_field(f1)
p2 = wash_pattern(p2)
f2 = wash_field(f2)
p3 = wash_pattern(p3)
f3 = wash_field(f3)
datetext1, datetext2 = wash_dates(d1, d1y, d1m, d1d, d2, d2y, d2m, d2d)
# wash ranking method:
if not is_method_valid(None, rm):
rm = ""
_ = gettext_set_language(ln)
# backwards compatibility: id, idb, sysnb -> recid, recidb, sysno (if applicable)
if sysnb != "" and sysno == "":
sysno = sysnb
if id > 0 and recid == -1:
recid = id
if idb > 0 and recidb == -1:
recidb = idb
# TODO deduce passed search limiting criterias (if applicable)
pl, pl_in_url = "", "" # no limits by default
if action != "browse" and req and not isinstance(req, cStringIO.OutputType) \
and req.args: # we do not want to add options while browsing or while calling via command-line
fieldargs = cgi.parse_qs(req.args)
for fieldcode in get_fieldcodes():
if fieldargs.has_key(fieldcode):
for val in fieldargs[fieldcode]:
pl += "+%s:\"%s\" " % (fieldcode, val)
pl_in_url += "&%s=%s" % (urllib.quote(fieldcode), urllib.quote(val))
# deduce recid from sysno argument (if applicable):
if sysno: # ALEPH SYS number was passed, so deduce DB recID for the record:
recid = get_mysql_recid_from_aleph_sysno(sysno)
if recid is None:
recid = 0 # use recid 0 to indicate that this sysno does not exist
# deduce collection we are in (if applicable):
if recid > 0:
referer = None
if req:
referer = req.headers_in.get('Referer')
cc = guess_collection_of_a_record(recid, referer)
# deduce user id (if applicable):
try:
uid = getUid(req)
except:
uid = 0
## 0 - start output
if recid >= 0: # recid can be 0 if deduced from sysno and if such sysno does not exist
## 1 - detailed record display
title, description, keywords = \
websearch_templates.tmpl_record_page_header_content(req, recid, ln)
if req is not None and not req.header_only:
page_start(req, of, cc, aas, ln, uid, title, description, keywords, recid, tab)
# Default format is hb but we are in detailed -> change 'of'
if of == "hb":
of = "hd"
if record_exists(recid):
if recidb <= recid: # sanity check
recidb = recid + 1
if of == "id":
return [recidx for recidx in range(recid, recidb) if record_exists(recidx)]
else:
print_records(req, range(recid, recidb), -1, -9999, of, ot, ln, search_pattern=p, verbose=verbose, tab=tab, sf=sf, so=so, sp=sp, rm=rm)
if req and of.startswith("h"): # register detailed record page view event
client_ip_address = str(req.remote_ip)
register_page_view_event(recid, uid, client_ip_address)
else: # record does not exist
if of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
elif of.startswith("h"):
if req.header_only:
raise apache.SERVER_RETURN, apache.HTTP_NOT_FOUND
else:
print_warning(req, _("Requested record does not seem to exist."))
elif action == "browse":
## 2 - browse needed
of = 'hb'
page_start(req, of, cc, aas, ln, uid, _("Browse"), p=create_page_title_search_pattern_info(p, p1, p2, p3))
req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, aas, ln, p1, f1, m1, op1,
p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action))
try:
if aas == 1 or (p1 or p2 or p3):
browse_pattern(req, colls_to_search, p1, f1, rg, ln)
browse_pattern(req, colls_to_search, p2, f2, rg, ln)
browse_pattern(req, colls_to_search, p3, f3, rg, ln)
else:
browse_pattern(req, colls_to_search, p, f, rg, ln)
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
elif rm and p.startswith("recid:"):
## 3-ter - similarity search (or old-style citation search) needed
if req and not req.header_only:
page_start(req, of, cc, aas, ln, uid, _("Search Results"), p=create_page_title_search_pattern_info(p, p1, p2, p3))
if of.startswith("h"):
req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, aas, ln, p1, f1, m1, op1,
p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action))
if record_exists(p[6:]) != 1:
# record does not exist
if of.startswith("h"):
if req.header_only:
raise apache.SERVER_RETURN, apache.HTTP_NOT_FOUND
else:
print_warning(req, _("Requested record does not seem to exist."))
if of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
else:
# record well exists, so find similar ones to it
t1 = os.times()[4]
results_similar_recIDs, results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, results_similar_comments = \
rank_records(rm, 0, get_collection_reclist(cc), string.split(p), verbose)
if results_similar_recIDs:
t2 = os.times()[4]
cpu_time = t2 - t1
if of.startswith("h"):
req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, cc, len(results_similar_recIDs),
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
print_warning(req, results_similar_comments)
print_records(req, results_similar_recIDs, jrec, rg, of, ot, ln,
results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, search_pattern=p, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
elif of=="id":
return results_similar_recIDs
elif of.startswith("x"):
print_records(req, results_similar_recIDs, jrec, rg, of, ot, ln,
results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, search_pattern=p, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
else:
# rank_records failed and returned some error message to display:
if of.startswith("h"):
print_warning(req, results_similar_relevances_prologue)
print_warning(req, results_similar_relevances_epilogue)
print_warning(req, results_similar_comments)
if of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
elif p.startswith("cocitedwith:"): #WAS EXPERIMENTAL
## 3-terter - cited by search needed
page_start(req, of, cc, aas, ln, uid, _("Search Results"), p=create_page_title_search_pattern_info(p, p1, p2, p3))
if of.startswith("h"):
req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, aas, ln, p1, f1, m1, op1,
p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action))
recID = p[12:]
if record_exists(recID) != 1:
# record does not exist
if of.startswith("h"):
print_warning(req, _("Requested record does not seem to exist."))
if of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
else:
# record well exists, so find co-cited ones:
t1 = os.times()[4]
results_cocited_recIDs = map(lambda x: x[0], calculate_co_cited_with_list(int(recID)))
if results_cocited_recIDs:
t2 = os.times()[4]
cpu_time = t2 - t1
if of.startswith("h"):
req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, CFG_SITE_NAME, len(results_cocited_recIDs),
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
print_records(req, results_cocited_recIDs, jrec, rg, of, ot, ln, search_pattern=p, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
elif of=="id":
return results_cocited_recIDs
elif of.startswith("x"):
print_records(req, results_cocited_recIDs, jrec, rg, of, ot, ln, search_pattern=p, verbose=verbose, sf=sf, so=so, sp=sp, rm=rm)
else:
# cited rank_records failed and returned some error message to display:
if of.startswith("h"):
print_warning(req, "nothing found")
if of == "id":
return []
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
else:
## 3 - common search needed
query_in_cache = False
query_representation_in_cache = repr((p,f,colls_to_search))
page_start(req, of, cc, aas, ln, uid, p=create_page_title_search_pattern_info(p, p1, p2, p3))
if of.startswith("h") and verbose and wash_colls_debug:
print_warning(req, "wash_colls debugging info : %s" % wash_colls_debug)
# search into the hosted collections only if the output format is html or xml
if hosted_colls and (of.startswith("h") or of.startswith("x")) and not p.startswith("recid:"):
# hosted_colls_results : the hosted collections' searches that did not timeout
# hosted_colls_timeouts : the hosted collections' searches that timed out and will be searched later on again
(hosted_colls_results, hosted_colls_timeouts) = calculate_hosted_collections_results(req, [p, p1, p2, p3], f, hosted_colls, verbose, ln, CFG_HOSTED_COLLECTION_TIMEOUT_ANTE_SEARCH)
# successful searches
if hosted_colls_results:
hosted_colls_true_results = []
for result in hosted_colls_results:
# if the number of results is None or 0 (or False) then just do nothing
if result[1] == None or result[1] == False:
# these are the searches the returned no or zero results
if verbose:
print_warning(req, "Hosted collections (perform_search_request): %s returned no results" % result[0][1].name)
else:
# these are the searches that actually returned results on time
hosted_colls_true_results.append(result)
if verbose:
print_warning(req, "Hosted collections (perform_search_request): %s returned %s results in %s seconds" % (result[0][1].name, result[1], result[2]))
else:
if verbose:
print_warning(req, "Hosted collections (perform_search_request): there were no hosted collections results to be printed at this time")
if hosted_colls_timeouts:
if verbose:
for timeout in hosted_colls_timeouts:
print_warning(req, "Hosted collections (perform_search_request): %s timed out and will be searched again later" % timeout[0][1].name)
# we need to know for later use if there were any hosted collections to be searched even if they weren't in the end
elif hosted_colls and ((not (of.startswith("h") or of.startswith("x"))) or p.startswith("recid:")):
(hosted_colls_results, hosted_colls_timeouts) = (None, None)
else:
if verbose:
print_warning(req, "Hosted collections (perform_search_request): there were no hosted collections to be searched")
## let's define some useful boolean variables:
# True means there are actual or potential hosted collections results to be printed
hosted_colls_actual_or_potential_results_p = not (not hosted_colls or not ((hosted_colls_results and hosted_colls_true_results) or hosted_colls_timeouts))
# True means there are hosted collections timeouts to take care of later
# (useful for more accurate printing of results later)
hosted_colls_potential_results_p = not (not hosted_colls or not hosted_colls_timeouts)
# True means we only have hosted collections to deal with
only_hosted_colls_actual_or_potential_results_p = not colls_to_search and hosted_colls_actual_or_potential_results_p
if of.startswith("h"):
req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, aas, ln, p1, f1, m1, op1,
p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action))
t1 = os.times()[4]
results_in_any_collection = HitSet()
if aas == 1 or (p1 or p2 or p3):
## 3A - advanced search
try:
results_in_any_collection = search_pattern_parenthesised(req, p1, f1, m1, ap=ap, of=of, verbose=verbose, ln=ln)
if len(results_in_any_collection) == 0:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
if p2:
results_tmp = search_pattern_parenthesised(req, p2, f2, m2, ap=ap, of=of, verbose=verbose, ln=ln)
if op1 == "a": # add
results_in_any_collection.intersection_update(results_tmp)
elif op1 == "o": # or
results_in_any_collection.union_update(results_tmp)
elif op1 == "n": # not
results_in_any_collection.difference_update(results_tmp)
else:
if of.startswith("h"):
print_warning(req, "Invalid set operation %s." % cgi.escape(op1), "Error")
if len(results_in_any_collection) == 0:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
if p3:
results_tmp = search_pattern_parenthesised(req, p3, f3, m3, ap=ap, of=of, verbose=verbose, ln=ln)
if op2 == "a": # add
results_in_any_collection.intersection_update(results_tmp)
elif op2 == "o": # or
results_in_any_collection.union_update(results_tmp)
elif op2 == "n": # not
results_in_any_collection.difference_update(results_tmp)
else:
if of.startswith("h"):
print_warning(req, "Invalid set operation %s." % cgi.escape(op2), "Error")
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
else:
## 3B - simple search
if search_results_cache.cache.has_key(query_representation_in_cache):
# query is not in the cache already, so reuse it:
query_in_cache = True
results_in_any_collection = search_results_cache.cache[query_representation_in_cache]
if verbose and of.startswith("h"):
print_warning(req, "Search stage 0: query found in cache, reusing cached results.")
else:
try:
# added the display_nearest_terms_box parameter to avoid printing out the "Nearest terms in any collection"
# recommendations when there are results only in the hosted collections. Also added the if clause to avoid
# searching in case we know we only have actual or potential hosted collections results
if not only_hosted_colls_actual_or_potential_results_p:
results_in_any_collection = search_pattern_parenthesised(req, p, f, ap=ap, of=of, verbose=verbose, ln=ln, display_nearest_terms_box=not hosted_colls_actual_or_potential_results_p)
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
return page_end(req, of, ln)
if len(results_in_any_collection) == 0 and not hosted_colls_actual_or_potential_results_p:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
# store this search query results into search results cache if needed:
if CFG_WEBSEARCH_SEARCH_CACHE_SIZE and not query_in_cache:
if len(search_results_cache.cache) > CFG_WEBSEARCH_SEARCH_CACHE_SIZE:
search_results_cache.clear()
search_results_cache.cache[query_representation_in_cache] = results_in_any_collection
if verbose and of.startswith("h"):
print_warning(req, "Search stage 3: storing query results in cache.")
# search stage 4: intersection with collection universe:
try:
# added the display_nearest_terms_box parameter to avoid printing out the "Nearest terms in any collection"
# recommendations when there results only in the hosted collections. Also added the if clause to avoid
# searching in case we know since the last stage that we have no results in any collection
if len(results_in_any_collection) != 0:
results_final = intersect_results_with_collrecs(req, results_in_any_collection, colls_to_search, ap, of, verbose, ln, display_nearest_terms_box=not hosted_colls_actual_or_potential_results_p)
else:
results_final = {}
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
return page_end(req, of, ln)
if results_final == {} and not hosted_colls_actual_or_potential_results_p:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
if of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
# search stage 5: apply search option limits and restrictions:
if datetext1 != "" and results_final != {}:
if verbose and of.startswith("h"):
print_warning(req, "Search stage 5: applying time etc limits, from %s until %s..." % (datetext1, datetext2))
try:
results_final = intersect_results_with_hitset(req,
results_final,
search_unit_in_bibrec(datetext1, datetext2, dt),
ap,
aptext= _("No match within your time limits, "
"discarding this condition..."),
of=of)
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
return page_end(req, of, ln)
if results_final == {} and not hosted_colls_actual_or_potential_results_p:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
#if of.startswith("x"):
# # Print empty, but valid XML
# print_records_prologue(req, of)
# print_records_epilogue(req, of)
return page_end(req, of, ln)
if pl and results_final != {}:
pl = wash_pattern(pl)
if verbose and of.startswith("h"):
print_warning(req, "Search stage 5: applying search pattern limit %s..." % cgi.escape(pl))
try:
results_final = intersect_results_with_hitset(req,
results_final,
search_pattern_parenthesised(req, pl, ap=0, ln=ln),
ap,
aptext=_("No match within your search limits, "
"discarding this condition..."),
of=of)
except:
register_exception(req=req, alert_admin=True)
if of.startswith("h"):
req.write(create_error_box(req, verbose=verbose, ln=ln))
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
return page_end(req, of, ln)
if results_final == {} and not hosted_colls_actual_or_potential_results_p:
if of.startswith("h"):
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
if of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
return page_end(req, of, ln)
t2 = os.times()[4]
cpu_time = t2 - t1
## search stage 6: display results:
results_final_nb_total = 0
results_final_nb = {} # will hold number of records found in each collection
# (in simple dict to display overview more easily)
for coll in results_final.keys():
results_final_nb[coll] = len(results_final[coll])
#results_final_nb_total += results_final_nb[coll]
# Now let us calculate results_final_nb_total more precisely,
# in order to get the total number of "distinct" hits across
# searched collections; this is useful because a record might
# have been attributed to more than one primary collection; so
# we have to avoid counting it multiple times. The price to
# pay for this accuracy of results_final_nb_total is somewhat
# increased CPU time.
if results_final.keys() == 1:
# only one collection; no need to union them
results_final_for_all_selected_colls = results_final.values()[0]
results_final_nb_total = results_final_nb.values()[0]
else:
# okay, some work ahead to union hits across collections:
results_final_for_all_selected_colls = HitSet()
for coll in results_final.keys():
results_final_for_all_selected_colls.union_update(results_final[coll])
results_final_nb_total = len(results_final_for_all_selected_colls)
#if hosted_colls and (of.startswith("h") or of.startswith("x")):
if hosted_colls_actual_or_potential_results_p:
if hosted_colls_results:
for result in hosted_colls_true_results:
colls_to_search.append(result[0][1].name)
results_final_nb[result[0][1].name] = result[1]
results_final_nb_total += result[1]
cpu_time += result[2]
if hosted_colls_timeouts:
for timeout in hosted_colls_timeouts:
colls_to_search.append(timeout[1].name)
# use -963 as a special number to identify the collections that timed out
results_final_nb[timeout[1].name] = -963
# we continue past this point only if there is a hosted collection that has timed out and might offer potential results
if results_final_nb_total ==0 and not hosted_colls_potential_results_p:
if of.startswith("h"):
print_warning(req, "No match found, please enter different search terms.")
elif of.startswith("x"):
# Print empty, but valid XML
print_records_prologue(req, of)
print_records_epilogue(req, of)
else:
# yes, some hits found: good!
# collection list may have changed due to not-exact-match-found policy so check it out:
for coll in results_final.keys():
if coll not in colls_to_search:
colls_to_search.append(coll)
# print results overview:
if of == "id":
# we have been asked to return list of recIDs
recIDs = list(results_final_for_all_selected_colls)
if sf: # do we have to sort?
recIDs = sort_records(req, recIDs, sf, so, sp, verbose, of)
elif rm: # do we have to rank?
results_final_for_all_colls_rank_records_output = rank_records(rm, 0, results_final_for_all_selected_colls,
string.split(p) + string.split(p1) +
string.split(p2) + string.split(p3), verbose)
if results_final_for_all_colls_rank_records_output[0]:
recIDs = results_final_for_all_colls_rank_records_output[0]
return recIDs
elif of.startswith("h"):
if of not in ['hcs']:
# added the hosted_colls_potential_results_p parameter to help print out the overview more accurately
req.write(print_results_overview(colls_to_search, results_final_nb_total, results_final_nb, cpu_time, ln, ec, hosted_colls_potential_results_p=hosted_colls_potential_results_p))
selected_external_collections_infos = print_external_results_overview(req, cc, [p, p1, p2, p3], f, ec, verbose, ln)
# print number of hits found for XML outputs:
if of.startswith("x"):
req.write("\n" % results_final_nb_total)
# print records:
if of in ['hcs']:
# feed the current search to be summarized:
from invenio.search_engine_summarizer import summarize_records
summarize_records(results_final_for_all_selected_colls, 'hcs', ln, p, f, req)
else:
if len(colls_to_search)>1:
cpu_time = -1 # we do not want to have search time printed on each collection
print_records_prologue(req, of)
for coll in colls_to_search:
if results_final.has_key(coll) and len(results_final[coll]):
if of.startswith("h"):
req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, coll, results_final_nb[coll],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
results_final_recIDs = list(results_final[coll])
results_final_relevances = []
results_final_relevances_prologue = ""
results_final_relevances_epilogue = ""
if sf: # do we have to sort?
results_final_recIDs = sort_records(req, results_final_recIDs, sf, so, sp, verbose, of)
elif rm: # do we have to rank?
results_final_recIDs_ranked, results_final_relevances, results_final_relevances_prologue, results_final_relevances_epilogue, results_final_comments = \
rank_records(rm, 0, results_final[coll],
string.split(p) + string.split(p1) +
string.split(p2) + string.split(p3), verbose)
if of.startswith("h"):
print_warning(req, results_final_comments)
if results_final_recIDs_ranked:
results_final_recIDs = results_final_recIDs_ranked
else:
# rank_records failed and returned some error message to display:
print_warning(req, results_final_relevances_prologue)
print_warning(req, results_final_relevances_epilogue)
print_records(req, results_final_recIDs, jrec, rg, of, ot, ln,
results_final_relevances,
results_final_relevances_prologue,
results_final_relevances_epilogue,
search_pattern=p,
print_records_prologue_p=False,
print_records_epilogue_p=False,
verbose=verbose,
sf=sf,
so=so,
sp=sp,
rm=rm)
if of.startswith("h"):
req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, coll, results_final_nb[coll],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1))
#if hosted_colls and (of.startswith("h") or of.startswith("x")):
if hosted_colls_actual_or_potential_results_p:
if hosted_colls_results:
# TODO: add a verbose message here
for result in hosted_colls_true_results:
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, results_final_nb[result[0][1].name],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
req.write(print_hosted_results(url_and_engine=result[0], ln=ln, of=of, req=req, limit=rg))
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, results_final_nb[result[0][1].name],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1))
if hosted_colls_timeouts:
# TODO: add a verbose message here
# TODO: check if verbose messages still work when dealing with (re)calculations of timeouts
(hosted_colls_timeouts_results, hosted_colls_timeouts_timeouts) = do_calculate_hosted_collections_results(req, ln, None, verbose, None, hosted_colls_timeouts, CFG_HOSTED_COLLECTION_TIMEOUT_POST_SEARCH)
if hosted_colls_timeouts_results:
hosted_colls_timeouts_true_results = []
for result in hosted_colls_timeouts_results:
if result[1] == None or result[1] == False:
## these are the searches the returned no or zero results
## also print a nearest terms box, in case this is the only
## collection being searched and it returns no results?
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, -963,
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
req.write(print_hosted_results(url_and_engine=result[0], ln=ln, of=of, req=req, no_records_found=True, limit=rg))
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, -963,
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1))
else:
# these are the searches that actually returned results on time
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, result[1],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
req.write(print_hosted_results(url_and_engine=result[0], ln=ln, of=of, req=req, limit=rg))
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, result[0][1].name, result[1],
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1))
if hosted_colls_timeouts_timeouts:
for timeout in hosted_colls_timeouts_timeouts:
if of.startswith("h"):
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, timeout[1].name, -963,
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time))
req.write(print_hosted_results(url_and_engine=timeout[0], ln=ln, of=of, req=req, search_timed_out=True, limit=rg))
req.write(print_hosted_search_info(p, f, sf, so, sp, rm, of, ot, timeout[1].name, -963,
jrec, rg, aas, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2,
sc, pl_in_url,
d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1))
print_records_epilogue(req, of)
if f == "author" and of.startswith("h"):
req.write(create_similarly_named_authors_link_box(p, ln))
# log query:
try:
id_query = log_query(req.remote_host, req.args, uid)
if of.startswith("h") and id_query:
if not of in ['hcs']:
# display alert/RSS teaser for non-summary formats:
user_info = collect_user_info(req)
display_email_alert_part = True
if user_info:
if user_info['email'] == 'guest':
if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS > 4:
display_email_alert_part = False
else:
if not user_info['precached_usealerts']:
display_email_alert_part = False
req.write(websearch_templates.tmpl_alert_rss_teaser_box_for_query(id_query, \
ln=ln, display_email_alert_part=display_email_alert_part))
except:
# do not log query if req is None (used by CLI interface)
pass
log_query_info("ss", p, f, colls_to_search, results_final_nb_total)
# External searches
if of.startswith("h"):
if not of in ['hcs']:
perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos)
return page_end(req, of, ln)
def perform_request_cache(req, action="show"):
"""Manipulates the search engine cache."""
req.content_type = "text/html"
req.send_http_header()
req.write("")
out = ""
out += "Search Cache
"
# clear cache if requested:
if action == "clear":
search_results_cache.clear()
req.write(out)
# show collection reclist cache:
out = "Collection reclist cache
"
out += "- collection table last updated: %s" % get_table_update_time('collection')
out += "
- reclist cache timestamp: %s" % collection_reclist_cache.timestamp
out += "
- reclist cache contents:"
out += ""
for coll in collection_reclist_cache.cache.keys():
if collection_reclist_cache.cache[coll]:
out += "%s (%d)
" % (coll, len(collection_reclist_cache.cache[coll]))
out += "
"
req.write(out)
# show search results cache:
out = "Search Cache
"
out += "- search cache usage: %d queries cached (max. ~%d)" % \
(len(search_results_cache.cache), CFG_WEBSEARCH_SEARCH_CACHE_SIZE)
if len(search_results_cache.cache):
out += "
- search cache contents:"
out += ""
for query, hitset in search_results_cache.cache.items():
out += "
%s ... %s" % (query, hitset)
out += """clear search results cache""" % CFG_SITE_URL
out += "
"
req.write(out)
# show field i18nname cache:
out = "Field I18N names cache
"
out += "- fieldname table last updated: %s" % get_table_update_time('fieldname')
out += "
- i18nname cache timestamp: %s" % field_i18nname_cache.timestamp
out += "
- i18nname cache contents:"
out += ""
for field in field_i18nname_cache.cache.keys():
for ln in field_i18nname_cache.cache[field].keys():
out += "%s, %s = %s
" % (field, ln, field_i18nname_cache.cache[field][ln])
out += "
"
req.write(out)
# show collection i18nname cache:
out = "Collection I18N names cache
"
out += "- collectionname table last updated: %s" % get_table_update_time('collectionname')
out += "
- i18nname cache timestamp: %s" % collection_i18nname_cache.timestamp
out += "
- i18nname cache contents:"
out += ""
for coll in collection_i18nname_cache.cache.keys():
for ln in collection_i18nname_cache.cache[coll].keys():
out += "%s, %s = %s
" % (coll, ln, collection_i18nname_cache.cache[coll][ln])
out += "
"
req.write(out)
req.write("")
return "\n"
def perform_request_log(req, date=""):
"""Display search log information for given date."""
req.content_type = "text/html"
req.send_http_header()
req.write("")
req.write("Search Log
")
if date: # case A: display stats for a day
yyyymmdd = string.atoi(date)
req.write("Date: %d
" % yyyymmdd)
req.write("""
""")
req.write("| %s | %s | %s | %s | %s | %s |
" % ("No.", "Time", "Pattern", "Field", "Collection", "Number of Hits"))
# read file:
p = os.popen("grep ^%d %s/search.log" % (yyyymmdd, CFG_LOGDIR), 'r')
lines = p.readlines()
p.close()
# process lines:
i = 0
for line in lines:
try:
datetime, aas, p, f, c, nbhits = string.split(line,"#")
i += 1
req.write("| #%d | %s:%s:%s | %s | %s | %s | %s |
" \
% (i, datetime[8:10], datetime[10:12], datetime[12:], p, f, c, nbhits))
except:
pass # ignore eventual wrong log lines
req.write("
")
else: # case B: display summary stats per day
yyyymm01 = int(time.strftime("%Y%m01", time.localtime()))
yyyymmdd = int(time.strftime("%Y%m%d", time.localtime()))
req.write("""""")
req.write("| %s | %s |
" % ("Day", "Number of Queries"))
for day in range(yyyymm01, yyyymmdd + 1):
p = os.popen("grep -c ^%d %s/search.log" % (day, CFG_LOGDIR), 'r')
for line in p.readlines():
req.write("""| %s | %s |
""" % \
(day, CFG_SITE_URL, day, line))
p.close()
req.write("
")
req.write("")
return "\n"
def get_most_popular_field_values(recids, tags, exclude_values=None, count_repetitive_values=True):
"""
Analyze RECIDS and look for TAGS and return most popular values
and the frequency with which they occur sorted according to
descending frequency.
If a value is found in EXCLUDE_VALUES, then do not count it.
If COUNT_REPETITIVE_VALUES is True, then we count every occurrence
of value in the tags. If False, then we count the value only once
regardless of the number of times it may appear in a record.
(But, if the same value occurs in another record, we count it, of
course.)
Example:
>>> get_most_popular_field_values(range(11,20), '980__a')
(('PREPRINT', 10), ('THESIS', 7), ...)
>>> get_most_popular_field_values(range(11,20), ('100__a', '700__a'))
(('Ellis, J', 10), ('Ellis, N', 7), ...)
>>> get_most_popular_field_values(range(11,20), ('100__a', '700__a'), ('Ellis, J'))
(('Ellis, N', 7), ...)
"""
def _get_most_popular_field_values_helper_sorter(val1, val2):
"Compare VAL1 and VAL2 according to, firstly, frequency, then secondly, alphabetically."
compared_via_frequencies = cmp(valuefreqdict[val2], valuefreqdict[val1])
if compared_via_frequencies == 0:
return cmp(val1.lower(), val2.lower())
else:
return compared_via_frequencies
valuefreqdict = {}
## sanity check:
if not exclude_values:
exclude_values = []
if isinstance(tags, str):
tags = (tags,)
## find values to count:
vals_to_count = []
displaytmp = {}
if count_repetitive_values:
# counting technique A: can look up many records at once: (very fast)
for tag in tags:
vals_to_count.extend(get_fieldvalues(recids, tag))
else:
# counting technique B: must count record-by-record: (slow)
for recid in recids:
vals_in_rec = []
for tag in tags:
for val in get_fieldvalues(recid, tag, False):
vals_in_rec.append(val)
# do not count repetitive values within this record
# (even across various tags, so need to unify again):
dtmp = {}
for val in vals_in_rec:
dtmp[val.lower()] = 1
displaytmp[val.lower()] = val
vals_in_rec = dtmp.keys()
vals_to_count.extend(vals_in_rec)
## are we to exclude some of found values?
for val in vals_to_count:
if val not in exclude_values:
if valuefreqdict.has_key(val):
valuefreqdict[val] += 1
else:
valuefreqdict[val] = 1
## sort by descending frequency of values:
out = ()
vals = valuefreqdict.keys()
vals.sort(_get_most_popular_field_values_helper_sorter)
for val in vals:
tmpdisplv = ''
if displaytmp.has_key(val):
tmpdisplv = displaytmp[val]
else:
tmpdisplv = val
out += (tmpdisplv, valuefreqdict[val]),
return out
def profile(p="", f="", c=CFG_SITE_NAME):
"""Profile search time."""
import profile
import pstats
profile.run("perform_request_search(p='%s',f='%s', c='%s')" % (p, f, c), "perform_request_search_profile")
p = pstats.Stats("perform_request_search_profile")
p.strip_dirs().sort_stats("cumulative").print_stats()
return 0
## test cases:
#print wash_colls(CFG_SITE_NAME,"Library Catalogue", 0)
#print wash_colls("Periodicals & Progress Reports",["Periodicals","Progress Reports"], 0)
#print wash_field("wau")
#print print_record(20,"tm","001,245")
#print create_opft_search_units(None, "PHE-87-13","reportnumber")
#print ":"+wash_pattern("* and % doo * %")+":\n"
#print ":"+wash_pattern("*")+":\n"
#print ":"+wash_pattern("ellis* ell* e*%")+":\n"
#print run_sql("SELECT name,dbquery from collection")
#print get_index_id("author")
#print get_coll_ancestors("Theses")
#print get_coll_sons("Articles & Preprints")
#print get_coll_real_descendants("Articles & Preprints")
#print get_collection_reclist("Theses")
#print log(sys.stdin)
#print search_unit_in_bibrec('2002-12-01','2002-12-12')
#print get_nearest_terms_in_bibxxx("ellis", "author", 5, 5)
#print call_bibformat(68, "HB_FLY")
#print get_fieldvalues(10, "980__a")
#print get_fieldvalues_alephseq_like(10,"001___")
#print get_fieldvalues_alephseq_like(10,"980__a")
#print get_fieldvalues_alephseq_like(10,"foo")
#print get_fieldvalues_alephseq_like(10,"-1")
#print get_fieldvalues_alephseq_like(10,"99")
#print get_fieldvalues_alephseq_like(10,["001", "980"])
## profiling:
#profile("of the this")
#print perform_request_search(p="ellis")
diff --git a/modules/websession/lib/webgroup_dblayer.py b/modules/websession/lib/webgroup_dblayer.py
index 8ff94ba43..eb30056eb 100644
--- a/modules/websession/lib/webgroup_dblayer.py
+++ b/modules/websession/lib/webgroup_dblayer.py
@@ -1,433 +1,433 @@
# -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
""" Database related functions for groups"""
__revision__ = "$Id$"
from time import localtime
from zlib import decompress
from invenio.config import CFG_SITE_LANG
-from invenio.dbquery import run_sql, run_sql_cached, OperationalError
+from invenio.dbquery import run_sql, OperationalError
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.messages import gettext_set_language
from invenio.websession_config import CFG_WEBSESSION_GROUP_JOIN_POLICY
def get_groups_by_user_status(uid, user_status, login_method='INTERNAL'):
"""Select all the groups the user is admin of.
@param uid: user id
@return: ((id_usergroup,
group_name,
group_description, ))
"""
query = """SELECT g.id,
g.name,
g.description
FROM usergroup g, user_usergroup ug
WHERE ug.id_user=%s AND
ug.id_usergroup=g.id AND
ug.user_status=%s AND
g.login_method = %s
ORDER BY g.name"""
uid = int(uid)
- res = run_sql_cached(query, (uid, user_status, login_method), affected_tables=['usergroup', 'user_usergroup'])
+ res = run_sql(query, (uid, user_status, login_method))
return res
def get_groups_by_login_method(uid, login_method):
"""Select all the groups the user is member of selecting the login_method.
@param uid: user id
@param login_method: the login_method (>0 external)
@return: ((id_usergroup,
group_name,
group_description, ))
"""
query = """SELECT g.id,
g.name,
g.description
FROM usergroup g, user_usergroup ug
WHERE ug.id_user=%s AND
ug.id_usergroup=g.id AND
g.login_method=%s
ORDER BY g.name"""
uid = int(uid)
- res = run_sql_cached(query, (uid, login_method), affected_tables=['usergroup', 'user_usergroup'])
+ res = run_sql(query, (uid, login_method))
return res
def get_groups_with_description(uid):
"""Select all the groups the user is member of.
@param uid: user id
@return: ((id_usergroup,
group_name,
group_description, ))
"""
query = """SELECT g.id,
g.name,
g.description
FROM usergroup g, user_usergroup ug
WHERE ug.id_user=%s AND
ug.id_usergroup=g.id
ORDER BY g.name"""
uid = int(uid)
- res = run_sql_cached(query, (uid, ), affected_tables=['usergroup', 'user_usergroup'])
+ res = run_sql(query, (uid, ))
return res
def get_external_groups(uid):
"""Select all the groups the user is member of selecting the login_method.
@param uid: user id
@param login_method: the login_method (>0 external)
@return: ((id_usergroup,
group_name,
group_description, ))
"""
query = """SELECT g.id,
g.name,
g.description
FROM usergroup g, user_usergroup ug
WHERE ug.id_user=%s AND
ug.id_usergroup=g.id AND
g.login_method != 'INTERNAL'
ORDER BY g.name"""
uid = int(uid)
- res = run_sql_cached(query, (uid, ), affected_tables=['usergroup', 'user_usergroup'])
+ res = run_sql(query, (uid, ))
return res
def get_groups(uid):
"""Select all the groups id the user is member of."""
query = """SELECT g.id, g.name
FROM usergroup g, user_usergroup ug
WHERE ug.id_user=%s AND
ug.id_usergroup=g.id
"""
- res = run_sql_cached(query, (uid, ), affected_tables=['usergroup', 'user_usergroup'])
+ res = run_sql(query, (uid, ))
res = list(res)
return res
def get_group_id(group_name, login_method='INTERNAL'):
"""@return: the id of the group called group_name with given login_method."""
return run_sql("""
SELECT id FROM usergroup
WHERE login_method = %s AND name = %s""", (login_method, group_name,))
def get_login_method_groups(uid, login_method='INTERNAL'):
"""Select all the external groups of a particular login_method for which
the user is subscrided.
@return: ((group_name, group_id))
"""
return run_sql("""
SELECT g.name as name, g.id as id
FROM user_usergroup as u JOIN usergroup as g
ON u.id_usergroup = g.id
WHERE u.id_user = %s and g.login_method = %s""",
(uid, login_method,))
def get_all_login_method_groups(login_method):
"""Select all the external groups of a particular login_method.
@return: ({group_name: group_id, ...})
"""
return dict(run_sql("""
SELECT name, id
FROM usergroup
WHERE login_method = %s""",
(login_method,)))
def get_all_users_with_groups_with_login_method(login_method):
"""Select all the users that belong at least to one external group
of kind login_method.
"""
return dict(run_sql("""
SELECT DISTINCT u.email, u.id
FROM user AS u JOIN user_usergroup AS uu ON u.id = uu.id_user
JOIN usergroup AS ug ON ug.id = uu.id_usergroup
WHERE ug.login_method = %s""", (login_method,)))
def get_visible_group_list(uid, pattern=""):
"""List the group the user can join (not already member
of the group regardless user's status).
@return: groups {id : name} whose name matches pattern
"""
grpID = []
groups = {}
#list the group the user is member of"""
query = """SELECT distinct(id_usergroup)
FROM user_usergroup
WHERE id_user=%s """
uid = int(uid)
res = run_sql(query, (uid,))
map(lambda x: grpID.append(int(x[0])), res)
query2 = """SELECT id,name
FROM usergroup
WHERE (join_policy='%s' OR join_policy='%s')""" % (
CFG_WEBSESSION_GROUP_JOIN_POLICY['VISIBLEOPEN'],
CFG_WEBSESSION_GROUP_JOIN_POLICY['VISIBLEMAIL'])
if len(grpID) == 1 :
query2 += """ AND id!=%i""" % grpID[0]
elif len(grpID) > 1:
query2 += """ AND id NOT IN %s""" % str(tuple(grpID))
if pattern:
try:
res2 = run_sql(query2 + """ AND name RLIKE %s ORDER BY name""", (pattern,))
except OperationalError:
res2 = ()
else:
res2 = run_sql(query2 + """ ORDER BY name""")
map(lambda x: groups.setdefault(x[0], x[1]), res2)
return groups
def insert_new_group(uid,
new_group_name,
new_group_description,
join_policy,
login_method='INTERNAL'):
"""Create a new group and affiliate a user."""
query1 = """INSERT INTO usergroup (id, name, description, join_policy,
login_method)
VALUES (NULL,%s,%s,%s,%s)
"""
params1 = (new_group_name,
new_group_description,
join_policy,
login_method)
res1 = run_sql(query1, params1)
date = convert_datestruct_to_datetext(localtime())
uid = int(uid)
query2 = """INSERT INTO user_usergroup (id_user, id_usergroup, user_status,
user_status_date)
VALUES (%s,%s,'A',%s)
"""
params2 = (uid, res1, date)
res2 = run_sql(query2, params2)
return res1
def insert_only_new_group(new_group_name,
new_group_description,
join_policy,
login_method='INTERNAL'):
"""Create a group with no user in (yet).
@return: its id
"""
query = """INSERT INTO usergroup (name, description, join_policy, login_method)
VALUES (%s, %s, %s, %s)
"""
res = run_sql(query, (new_group_name, new_group_description, join_policy, login_method))
return res
def insert_new_member(uid,
grpID,
status):
"""Insert new member."""
query = """INSERT INTO user_usergroup (id_user, id_usergroup, user_status,
user_status_date)
VALUES (%s,%s,%s,%s)
"""
date = convert_datestruct_to_datetext(localtime())
res = run_sql(query, (uid, grpID, status, date))
return res
def get_group_infos(grpID):
"""Get group infos."""
query = """SELECT id,name,description,join_policy,login_method FROM usergroup
WHERE id = %s"""
res = run_sql(query, (grpID, ))
return res
def get_all_groups_description(login_method):
"""Get all groups description, dictionary with key name."""
query = """SELECT name, description
FROM usergroup
WHERE login_method = %s
"""
res = run_sql(query, (login_method, ))
if res:
return dict(res)
else:
return {}
def update_group_infos(grpID,
group_name,
group_description,
join_policy):
"""Update group."""
res = run_sql("""UPDATE usergroup
SET name=%s, description=%s, join_policy=%s
WHERE id=%s""",
(group_name, group_description, join_policy, grpID))
return res
def get_user_status(uid, grpID):
"""Get the status of the user for the given group."""
query = """SELECT user_status FROM user_usergroup
WHERE id_user = %s
AND id_usergroup=%s"""
uid = int(uid)
res = run_sql(query, (uid, grpID))
return res
def get_users_by_status(grpID, status, ln=CFG_SITE_LANG):
"""Get the list of users with the given status.
@return: ((id, nickname),) nickname= user # uid if
the user has no nickname
"""
_ = gettext_set_language(ln)
res = run_sql("""SELECT ug.id_user, u.nickname
FROM user_usergroup ug, user u
WHERE ug.id_usergroup = %s
AND ug.id_user=u.id
AND user_status = %s""",
(grpID, status))
users = []
if res:
for (mid, nickname) in res:
nn = nickname
if not nickname:
nn = _("user") + "#%i" % mid
users.append((mid, nn))
return tuple(users)
def delete_member(grpID, member_id):
"""Delete member."""
query = """DELETE FROM user_usergroup
WHERE id_usergroup = %s
AND id_user = %s"""
member_id = int(member_id)
res = run_sql(query, (grpID, member_id))
return res
def delete_group_and_members(grpID):
"""Delete the group and its members."""
query = """DELETE FROM usergroup
WHERE id = %s
"""
res = run_sql(query, (grpID,))
query = """DELETE FROM user_usergroup
WHERE id_usergroup = %s
"""
res = run_sql(query, (grpID,))
return res
def add_pending_member(grpID, member_id, user_status):
"""Change user status:
Pending member becomes normal member"""
date = convert_datestruct_to_datetext(localtime())
res = run_sql("""UPDATE user_usergroup
SET user_status = %s, user_status_date = %s
WHERE id_usergroup = %s
AND id_user = %s""",
(user_status, date, grpID, member_id))
return res
def leave_group(grpID, uid):
"""Remove user from the group member list."""
query = """DELETE FROM user_usergroup
WHERE id_usergroup=%s
AND id_user=%s"""
uid = int(uid)
res = run_sql(query, (grpID, uid))
return res
def drop_external_groups(userId):
"""Drops all the external groups memberships of userid."""
query = """DELETE user_usergroup FROM user_usergroup, usergroup
WHERE user_usergroup.id_user=%s
AND usergroup.id = user_usergroup.id_usergroup
AND usergroup.login_method <> 'INTERNAL'"""
return run_sql(query, (userId,))
def group_name_exist(group_name, login_method='INTERNAL'):
"""Get all group id whose name like group_name and login_method."""
query = """SELECT id
FROM usergroup
WHERE login_method=%s AND name=%s"""
res = run_sql(query, (login_method, group_name))
return res
def get_group_login_method(grpID):
"""Return the login_method of the group or None if the grpID doesn't exist."""
query = """SELECT login_method
FROM usergroup
WHERE id=%s"""
res = run_sql(query, (grpID, ))
if res:
return res[0][0]
else:
return None
def count_nb_group_user(uid, user_status):
"""
@param uid: user id
@param status: member status
@return: integer of number of groups the user belongs to
with the given status, 0 if none
"""
res = run_sql("""SELECT count(id_user)
FROM user_usergroup
WHERE id_user = %s
AND user_status = %s""",
(uid, user_status))
if res:
return int(res[0][0])
else:
return 0
def get_all_users():
"""@return: all the email:id"""
query = """SELECT UPPER(email), id
FROM user
WHERE email != ''
"""
res = run_sql(query)
if res:
return dict(res)
else:
return {}
def get_users_in_group(grpID):
"""@return: all uids of users belonging to group grpID"""
query = """SELECT id_user
FROM user_usergroup
WHERE id_usergroup = %s
"""
res = run_sql(query, (grpID, ))
return [uid[0] for uid in res]
########################## helpful functions ##################################
def __decompress_last(item):
"""private function, used to shorten code"""
item = list(item)
item[-1] = decompress(item[-1])
return item