<navbar:button id=howto_marc txt="HOWTO MARC" url="<WEBURL>/admin/howto/marc.html" hint="Admin's HOWTO Guides - How to MARC your medatada">
<navbar:button id=howto_migrate txt="HOWTO Migrate" url="<WEBURL>/admin/howto/migrate.html" hint="Admin's HOWTO Guides - How to Migrate your old documents into CDSware">
<navbar:button id=howto_run txt="HOWTO Run" url="<WEBURL>/admin/howto/run.html" hint="Admin's HOWTO Guides - How to Run your CDSware installation day-by-day">
<navbar:button id=websubmit_newdoctype txt="New Doctype" url="<WEBURL>/admin/websubmit/newDoctypeEDS.php" hint="New Type of Document">
<navbar:button id=websubmit_deldoctype txt="Remove Doctype" url="<WEBURL>/admin/websubmit/removeDoctypeEDS.php" hint="Remove Type of Document">
<navbar:button id=websubmit_listactions txt="Available Actions" url="<WEBURL>/admin/websubmit/allActionsEDS.php" hint="List Available Actions">
<navbar:button id=websubmit_listchecks txt="Available Javascript Checks" url="<WEBURL>/admin/websubmit/allChecksEDS.php" hint="List Available Javascript Checking functions">
<navbar:button id=websubmit_listelements txt="Available Element Descriptions" url="<WEBURL>/admin/websubmit/allElementsEDS.php" hint="List Available Element Descriptions">
<navbar:button id=websubmit_listfunctions txt="Available Functions" url="<WEBURL>/admin/websubmit/listFunctions.php" hint="List Available Functions">
## This part defines parameters for the CDSware OAI gateway.
## Useful if you are running CDSware as OAI data provider.
## OAIIDSCHEME -- OAI identifier scheme:
## (example: "oai")
<define-tag OAIIDSCHEME whitespace=delete>
oai
</define-tag>
## OAIIDTAG -- OAI identifier tag:
## (example: "0248_a")
<define-tag OAIIDTAG whitespace=delete>
909COo
</define-tag>
## OAISETTAG -- OAI set tag:
## (example: "0248_p")
<define-tag OAISETTAG whitespace=delete>
909COp
</define-tag>
## OAIDELETEDTAG -- OAI tag for deleted records mark. OAI record is considered deleted if the value of this field is set to "DELETED" and the OAI deleted records policy is set either to ``transient'' or ``persistent''.
AC_ARG_WITH(webdir, AC_HELP_STRING([--with-webdir], [specify where the web files go (e.g. /var/www/cdsware/)]), WEBDIR=${withval})
if test -z "$WEBDIR"; then
AC_MSG_ERROR([
You have not used the "--with-webdir" argument that should define the installation directory for Web programs and documents.
Example: "--with-webdir=/var/www/cdsware/".])
fi
## Define WEBURL URL:
AC_ARG_WITH(weburl, AC_HELP_STRING([--with-weburl], [specify URL where web files will be reached (e.g. http://cdsdev.cern.ch/cdsware/)]), WEBURL=${withval})
if test -z "$WEBURL"; then
AC_MSG_ERROR([
You have not used the "--with-weburl" argument that should define the corresponding URL where the site will be reached.
retrieves the value of the first field containing tag 'tag' and indicators 'ind1' and 'ind2'
inside record 'rec'. Returns the found value as a string. If no matching field is found
returns the empty string.
if the tag has a '%', it will retrieve the value of first field containg tag, which first characters are those before '%' in tag. The ind1, ind2 and code parameters will be ignored
retrieves the values of all the fields containing tag 'tag' and indicators 'ind1' and 'ind2'
inside record 'rec'. Returns the found values as a list. If no matching field is found
returns an empty list.
if the tag has a '%', it will retrieve the value of all fields containg tag, which first characters are those before '%' in tag. The ind1, ind2 and code parameters will be ignored
"""
tmp = []
s = tag.split('%')
if len(s) > 1:
t = s[0]
keys=rec.keys()
tags=[k for k in keys if k.startswith(t)]
for tag in tags:
fields = rec[tag]
for field in fields:
if field[3] != "":
tmp.append(field[3])
else:
for subfield in field[0]:
tmp.append(subfield[1])
else:
if rec.has_key(tag):
fields = rec[tag]
for field in fields:
if field[1]==ind1 and field[2]==ind2:
if field[3] != "":
tmp.append(field[3])
else:
for subfield in field[0]:
if subfield[0]==code:
tmp.append(subfield[1])
return tmp
def print_rec(rec,format=1):
"""prints a record
format = 1 -- XML
format = 2 -- HTML (not implemented)
"""
if format==1:
text = record_xml_output(rec)
else:
return ''
return text
def print_recs(listofrec,format=1):
"""prints a list of records
format = 1 -- XML
format = 2 -- HTML (not implemented)
if 'listofrec' is not a list it returns empty string
"""
text = ""
if type(listofrec).__name__ !='list':
return ""
else:
for rec in listofrec:
text = "%s\n%s" % (text,print_rec(rec,format))
return text
def record_xml_output(rec):
"""generates the XML for record 'rec' and returns it as a string"""
if (mk_match2 is not None) and (string.atoi(mk_match2.group('num')) == 2) and (mk_match.group('left') == mk_match2.group('left')) and (mk_match.group('right') == mk_match2.group('right')):
# Found next line:
tmpFnd = 1
elif y == len(self._content) - 1:
tmpFnd = 1
y = y + 1
if tmpFnd:
foundSect = 1
refStartLine = x
refLineMarker = mk_match.group('mark')
refLineMarkerPattern = mk_ptn
x = x - 1
if refStartLine is not None:
# Make ReferenceSectionStartPoint object with ref section start location details
refStart = ReferenceSectionStartPoint()
refStart.setLineNum(refStartLine)
refStart.setMarkerChar(refLineMarker)
refStart.setMarkerPattern(refLineMarkerPattern)
return refStart
def findReferenceSection(self):
"""Find the line number of the start of a TextDocument object's reference section. Returns a 'ReferenceSectionStartPoint'
object containing details of the reference section start line number, the reference section title & the marker char
used for each reference line or returns None if not found
"""Exception thrown if a line in the periodicals knowledge base does not comply with the expected format"""
pass
class KnowledgeBase:
"""The knowledge base of periodical titles. Consists of search & replace terms. The search terms consist of non-standard periodical titles in upper case.
These are often found in the text of documents. Replacement terms consist of standardised periodical titles in a standardised case. These will be used to
replace identified non-standard titles
"""
def __init__(self, fn = None):
self._kb = {}
self._compiledPatternsKB = {}
self._unstandardisedTitle = {}
if type(fn) is str: self._buildKB(fn)
def _buildKB(self, fn):
"""From the filename provided (fn), read the periodicals knowledge base into memory, and build a dictionary of seek/replace values to be stored in self._kb"""
"""Identify periodical titles in text line 'ln' and record information about where in the line they occur. Replace them for lower-case versions or
lowercase letter 'a's if the match was numerical. Return a Tuple containing dictionaries containing information about the substitutions, along with the new line
"""Accept a user-defined search pattern, transform it, according to some simple rules, into a regex pattern, then compile and return it as a compiled RE object
"""For a replacement index position, calculate a new (correct) replacement index, based on any spaces that have been removed before it, according to the type of the replacement"""
for strip_space in spacesKeys:
if strip_space < true_repidx:
# Spaces were removed before this replacement item should be placed. Add number of spaces removed to current replacement idx:
# All spaces removed before this URL accounted for - add it to the dictionary
uri_virtual_locations[virtual_pos] = idx
return uri_virtual_locations
class ReferenceSectionMarkupProcessor:
"""Process a reference section. Line will be cleaned, and cited items will be identified and their notation standardised. ProcessedReferenceLine will be returned"""
out = u""" <datafield tag="999" ind1="C" ind2="5">\n"""
if self._misc is not None and (type(self._misc) is unicode or type(self._misc) is str):
out += u""" <subfield code="m">"""+cgi.escape(self._misc)+u"""</subfield>\n"""
out += u""" <subfield code="r">"""+cgi.escape(self._rn)+u"""</subfield>\n"""
if xtra_subfield is not None:
out += xtra_subfield
out += u""" </datafield>\n"""
return out
class URLCitation(Citation):
def __init__(self, url, urldescr, misc=None):
self._url = url
self._urldescr = urldescr
if misc is not None: self._misc = misc.strip()
else: self._misc = misc
def getSelfMARCXML(self):
out = u""" <datafield tag="999" ind1="C" ind2="5">\n"""
if self._misc is not None and (type(self._misc) is unicode or type(self._misc) is str):
out += u""" <subfield code="m">"""+cgi.escape(self._misc)+u"""</subfield>\n"""
out += u""" <subfield code="u">"""+cgi.escape(self._url)+u"""</subfield>\n"""
out += u""" <subfield code="z">"""+cgi.escape(self._urldescr)+u"""</subfield>\n"""
out += u""" </datafield>\n"""
return out
class ProcessedReferenceLine:
"""This is a reference line that has been processed for cited items"""
def __init__(self):
self._segments = {} # Segments of reference line, each keyed by start point index. Each is a 'LineItem'.
self._nextposn = 0
def getSelfMARCXML(self):
"""Return an XML string containing this lines contents, marked up in XML MARC, as used in CDS"""
i = 0
lenline = len(self._segments)
out = u""
while i < lenline:
if isinstance(self._segments[i],TitleCitationStandard) and i < lenline-1 and isinstance(self._segments[i+1],InstitutePreprintReferenceCitation) and not self._segments[i+1].hasMisc():
# This is a $s (periodical title) reference, followed immediately by its report number ($r). Concat them both under the $s.
out += self._segments[i].getSelfMARCXML(self._segments[i+1].getRN_subfield())
i = i + 1
elif isinstance(self._segments[i],InstitutePreprintReferenceCitation) and i < lenline-1 and isinstance(self._segments[i+1],TitleCitationStandard) and not self._segments[i+1].hasMisc():
# This is a report number ($r) reference followed immediately by its periodical title ($s) reference. Concat them both under $s.
out += self._segments[i].getSelfMARCXML(self._segments[i+1].getS_subfield())
i = i + 1
else:
out += self._segments[i].getSelfMARCXML()
i = i + 1
return out
def addSection(self, newSect):
if isinstance(newSect,LineItem):
self._segments[self._nextposn] = newSect
self._nextposn += 1
def getNumberCitations(self):
numcitations = 0
numsegments = len(self._segments)
for i in range(0,numsegments):
if isinstance(self._segments[i], Citation): numcitations += 1
return numcitations
class ProcessedReferenceSection:
"""This is a reference section after it has been processed to identify cited items. It contains a list of ProcessedReferenceLines."""
def __init__(self):
self._lines = {}
self._nextline = 0
def getSelfMARCXML(self):
"""Return a unicode string of all reference lines marked up in MARC XML"""
out = u""
numlines = len(self._lines)
for i in range(0,numlines): out += self._lines[i].getSelfMARCXML()
return out
def appendLine(self, ln):
"""Add a new line to the list of processed reference lines"""
if isinstance(ln, ProcessedReferenceLine):
self._lines[self._nextline] = ln
self._nextline += 1
def getTotalNumberCitations(self):
"""Return an integer representing the total number of citations recognised (and thus marked up) in the reference section"""
numcitations = 0
numlines = len(self._lines)
for i in range(0,numlines): numcitations += self._lines[i].getNumberCitations()
return numcitations
class NumerationHandler:
"""Class whose instances identify reference numeration patterns in a text line and rearrange them into standardised numeration patterns
Returns line with numeration patterns marked up in an XML style
"""After the line has been rebuilt with marked up titles, it can be rechecked for numeration patterns because perhaps now more can be found with the aid of the recognised titles"""
# If there is no comma in the matched title, add one to the end of it before series info added. If there is already a comma present, simply discard the series info
"""Given that some more titles have been recognised within a line, reprocess that line in the hopes of recognising more numeration patterns"""
for x in self._checkAgainPtnList:
ln = x[0].sub(x[1], ln)
return self.standardise(ln)
def standardise(self, ln):
"""Accept ln (text line) as argument. Perform transformations on this line to replace non-standard numeration styles with marked-up versions in a standard format.
These recognised and marked-up numeration patterns can later be used to identify cited documents
"""
for x in self._ptnList:
ln = x[0].sub(x[1], ln)
return ln
class LineCleaner:
"""Class to enable lines to be cleaned of punctuation errors"""
def __init__(self):
self._correctionList = {}
self._setCorrectionList()
def _setCorrectionList(self):
"""Set the list of punctuation (etc) errors in a line to be corrected"""
"""Class to remove all ocurrences of multiple spaces from a line and replace them with a single space while recording information about their positioning"""
<p>This is just a formal way of describing the language, but don't
worry if you don't understand it very well because just below these lines we'll
try to describe it in a more informal way.</p>
<p>To begin with, you should know that EL is a language designed to
work with strings (a string is a collection of characters) but it has also some
logic and comparison operations. One important thing you have to be aware of is
that in EL blank spaces, tabulators or carriage returns have no more meaning
than separator for elements of the language; that means that between two basic
elements you can have as many spaces or carriage returns as you want.</p>
<p>One of the basic elements of the language is what we call <em>LITERALS</em>.
These things represent constant string values; they are delimited by a pair of
double quote (") symbols surrounding the string you want to express. Everything
you put inside the double quotes will be considered as it is, so inside a
literal several spaces or carriage have meaning (it's the only case). If you
want to express a double quote symbol inside a literal you have to <em>escape</em>
it using \.</p>
<p>Some examples of literals:</p>
<ul>
<li>If you want to represent the string <em>hello</em>,
inside the EL you'll have to use <code>"hello"</code>.</li>
<li>For the string <em>hello "big" man</em>, the representation in EL is <code>"hello \"big\" man"</code>
(notice the escape characters and that spaces have meaning).</li>
<li><em>Let's see \""</em> string has to be expressed in this
way <code>"Let's see \\\"\""</code>.</li>
</ul>
<p>Another important basic element of the language is <em>VARIABLES</em>.
These elements represent string data from the input to which you can refer
inside of the language (and is considered also as a string). Variables are defined
in advance by the administrator (or even users) so you have to know which of
them you have access to. Additionally, variables can contain <em>FIELDS</em> that
are simply other input values that are grouped under a variable because they
have some kind of relationship between them (for example, you could have a
variable for the information about the author and fields like name, born place,
etc for it). If you want to know more about variables and their correspondence
with the input you can look at the <em>Mapping the Input</em> section. The way of
expressing a variable in EL is by a dollar symbol followed by any letter,
number or underscore; variables are case-insensitive. To refer to any field of
a variable, you simply put a dot followed by the field name (which is also made
up of any character, number or underscore).</p>
<p>Some examples about variables and fields:</p>
<ul>
<li>Imagine you have a variable which contains the author
information and which is called <em>author</em>, to represent in EL you would
have to write <code>"Author: " $100.a</code>. In every place
that $author appears BibFormat will consider the value defined for it from the
current input record.</li>
<li>Then you know that the field <em>name</em> of variable <em>author</em>
contains the author full name and you want to refer to it inside an EL
statement, so you'd write <code>$author.name</code>.</li>
<li>If we speak about CDS configuration, variable and field
names correspond to MARC 21 tag & indicator names; so to refer to the main
title of a bibliographic record we should use variable <em>245</em> field <em>b</em>,
in EL terms: <code>$245.b</code>.</li>
</ul>
<p>Now that we know basic elements of the language we can start
thinking about how to combine them. The most important (and unique) string
operation is concatenation: adding strings. This operation is implicit to the
language, so we just put language elements one before another, and the representation
result will be the result of the basic elements one after another.</p>
<p>Some samples:</p>
<ul>
<li>To represent the constant string <em>Author:</em>
followed by the name of the author of the input record you should write <code>"Author: " $100.a</code> (it's supposed CDS
configuration in which MARC 21 notation is used; authors correspond to variable
100 field a).</li>
<li>You want to output the title in bold (always HTML
speaking) followed by the author in normal chars separated of the title by char
<strong>3. <a href="#3">OAI Data Providing</a></strong><br>
<a name="1"></a><h2>1. Overview</h2>
The BibHarvest module handles metadata gathering and delivery between OAI-compliant repositories. Metadata exchange is performed on top of the OAI-PMH, the Open Archives Initiative's Protocol for Metadata Harvesting.
The BibHarvest Admin Interface can be used to set up a database of OAI sources and determine the periodicity of harvesting.
<p>In order to periodically harvest metadata from one or several repositories, it is possible to organize OAI sources through the BibHarvest Admin Interface.
The interface allows the administrator to add new repositories as well as edit and delete existing ones.
Once the database has been set up thorugh the BibHarvest Admin Interface, run the oaiharvest command-line tool to run periodical harvesting</p>
<p>The first step requires the administrator to enter the baseURL of the OAI repository. This is done for validation purposes - i.e. to check that the baseURL actually points to an OAI-compliant repository.
(Note: the validation simply performs an 'Identify' query to the baseURL and parses the reply with crucial tags such as <code>OAI-PMH</code> and <code>Identify</code>).
<br>Once the baseURL is validated, the administrator is required to fill into the following fields:
<br><ul>
<li><strong>Source name</strong>: this will be the (alphanumeric) name used to refer to this OAI repository.<br><br>
<li><strong>Metadata prefix</strong>: upon validation the baseURL is checked for the available metadata formats. The administator can choose the desired format to harvest metadata in.<br><br>
<li><strong>Frequency</strong>: how often do you intend to harvest from this repository? (Note: selecting "never" excludes this source from automatic periodical harvesting)<br><br>
<li><strong>Starting date</strong>: on your first harvesting session, do you intend to harvest the whole repository (from beginning) or only newly added material (from today)? (WARNING: harvesting large collections of material may take very long!)<br><br>
<li><strong>Postprocess</strong>: how is the harvested material be dealt with after harvesting?
<ul><li>h: harvest only - metadata will be gathered and saved locally<li>h-u: harvest and upload - metadata will be automatically queued for upload (this is useful when harvested metadata is already in marcxml format)<li>h-c: harvest and convert - harvested metadata will be converted and saved locally<li>h-c-u: harvest, convert and upload - harvested metadata will be converted and queued for upload - this is useful when harvested metadata is in a format different from marcxml (e.g. oai_dc)</ul><br>
<li><strong>BibConvert configuration file</strong>: if postprocess involves conversion, the full path to a BibConvert configuration file is needed.
This file determines how the metadata conversion will take place (e.g. oai_dc to marcxml).
In order to ensure that the configuration file performs the desired conversion, please test it in advance and follow the instruction contained in the <a href="../bibconvert/guide.html">BibConvert Admin Guide</a>.
Please note that some conversion parameters previously input at the command line can now be handled by the BibConvert Configuration Language (namely record separator, file header and file footer). Please consult the <a href="../bibconvert/guide.html">BibConvert Admin Guide</a> for more information on this.
This allows fully automatised harvesting and conversion (and possibly upload) of records provided a valid, functioning, BibConvert template is provided.
</ul>
<br>
<b>Edit and delete OAI sources</b><br>
<p>Once a source has been added to the database, it will be visible in the overview page, as shown below</p>
<p>At this point it will be possible to edit the definition of this source by clicking on the appropriate action button. All the fields described in 2.2.1 can be modified (except for the Starting date).
There is no more validation at this stage, hence, please take extra care when editing important fields such as baseurl and metadataprefix.
<br>
OAI repositories can be removed from the database by clicking on the appropriate action button in the overview page.
<p>Once administrators have set up their desired OAI repositories in the database through the Admin Interface they can invoke <code>oaiharvest</code> to start up periodical harvesting.<br>
<br>
<b>Oaiharvest usage</b>
<blockquote>
<pre>
oaiharvest [options]
Specific options:
-r, --repository=REPOS_ONE,"REPOS TWO" name of the OAI repositories to be harvested (default=all)
-d, --dates=yyyy-mm-dd:yyyy-mm-dd harvest repositories between specified dates (overrides repositories' last updated timestamps)
Scheduling options:
-u, --user=USER user name to store task, password needed
-s, --sleeptime=SLEEP time after which to repeat tasks (no)
e.g.: 1s, 30m, 24h, 7d
-t, --time=TIME moment for the task to be active (now)
e.g.: +15s, 5m, 3h , 2002-10-27 13:57:26
General options:
-h, --help print this help and exit
-V, --version print version and exit
-v, --verbose=LEVEL verbose level (from 0 to 9, default 1)
</pre>
</blockquote>
<code>oaiharvest</code> performs a number of operations on the repositories listed in the database. By default <code>oaiharvest</code> considers all repositories, one by one (this gets overridden when <code>--repository</code> argument is passed).
<br> For each repository that is considered, <code>oaiharvest</code> behaves according to the arguments passed at the command line:
<ul>
<li> If the <code>--dates</code> argument is not passed, it checks whether an update from the repository is needed (Note: the update status is calculated based on the time of the last harvesting and the frequency chosen by the administrator).
<ul>
<li>If an update is needed, it launches <code>bibharvest</code> and harvests all the metadata that the repository has added since the data of the last update. When the update is finished, the last update value is set to the current time and date.
<li>If an update is not needed, the source is skipped.
</ul>
<li> If the <code>--dates</code> argument is passed, it simply harvests the metadata of the repository from/until the given dates. The last update date is left unchanged.
<li> Finally, it performs the operations indicated in the postprocess mode, i.e. convert and/or upload the harvested metadata.
</ul>
<br>
<b>Oaiharvest usage examples</b><br>
<p>In most cases, administrators will want <code>oaiharvest</code> to run in the background, i.e. run in sleep mode and wake up periodically (e.g. every 24 hours) to check whether updates are needed, e.g. <code>oaiharvest -s 24h</code><br>
<p>In other cases, administrators may want to perform periodical harvesting only on specific sources, e.g. <code>oaiharvest -r cdsweb -s 12h</code><br>
<p>Another option is that administrators may want to harvest from certain repositories within two specific dates. This will be regarded as a one-off operation and will not affect the last update value of the source, e.g. <code>oaiharvest -r cdsweb -d 2005-05-05:2005-05-30</code><br>
- 'name' *string* - The name of the value in the dd box
- 'value' *list* - The values in the textbox"""
_ = gettext_set_language(ln)
text = """<span class="adminlabel">%s""" % title
text += """</span><select name="%s" class="admin_w200">""" % name
text += """<option value="">%s</option>""" % valuenil
try:
for val in values:
intval = int(lastval)
if intval==int(val[0]): ## retrieve and display last value inputted into drop-down box
text += """<option value="%s" %s>%s</option>""" % (val[0], 'selected="selected"', str(val[1]))
else:
text += """<option value="%s">%s</option>""" % (val[0], str(val[1]))
text += """</select><br>"""
except StandardError, e:
for val in values:
if lastval==val[0]:
text += """<option value="%s" %s>%s</option>""" % (val[0], 'selected="selected"', str(val[1]))
else:
text += """<option value="%s">%s</option>""" % (val[0], str(val[1]))
text += """</select><br>"""
return text
### deprecated ###
# def tmpl_print_src_add_tips(self):
# """Outputs some tips for source adding and editing"""
# text = """<br>
# <small><strong>Frequency</strong>: how often do you intend to harvest from this repository?</small><br><br>
# <small><strong>Starting date</strong>: do you intend to harvest the whole repository (from beginning) or only newly added material (from today)? <strong>WARNING</strong>: harvesting large collections of material may take very long!</small><br><br>
# <small><strong>Postprocess</strong>: how is the harvested material be dealt with after harvesting?
# <ul><li>h: harvest only<li>h-c: harvest and convert<li>h-c-u: harvest, convert and upload</ul></small>
# <small><strong>BibConvert configuration file</strong>: if postprocess involves conversion, please include full path to a configuration file</small>
# """
# return text
### deprecated ###
# def tmpl_print_src_edit_tips(self):
# """Outputs some tips for source adding and editing"""
# text = """<br>
# <small><strong>Frequency</strong>: how often do you intend to harvest from this repository?</small><br><br>
# <small><strong>Postprocess</strong>: how is the harvested material be dealt with after harvesting?
# <ul><li>h: harvest only<li>h-c: harvest and convert<li>h-c-u: harvest, convert and upload</ul></small>
# <small><strong>BibConvert configuration file</strong>: if postprocess involves conversion, please include full path to a configuration file</small>
# """
# return text
### deprecated ###
# def tmpl_print_validate_tips(self):
# """Outputs some tips for source validation"""
# text = """<br><small><strong>Validate</strong>: to check that the baseURL of the repository is OAI-compliant</small>"""
# return text
def tmpl_print_info(self, ln, infotext):
"""Outputs some info"""
_ = gettext_set_language(ln)
text = """<br><b><span class="info">%s</span></b>""" % infotext
return text
def tmpl_print_warning(self, ln, warntext):
"""Outputs some info"""
_ = gettext_set_language(ln)
text = """<span class="warning">%s</span>""" % warntext
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""CDSware Bibharvest Administrator Interface."""
import cgi
import re
import MySQLdb
import Numeric
import os, sys, string
import ConfigParser
import time
import random
import urllib
import sre
from mod_python import apache
from cdsware.bibrankadminlib import write_outcome,modify_translations,get_def_name,get_i8n_name,get_name,get_rnk_nametypes,get_languages,check_user,is_adminuser,adderrorbox,addadminbox,tupletotable,tupletotable_onlyselected,addcheckboxes,createhiddenform,serialize_via_numeric_array_dumps,serialize_via_numeric_array_compr,serialize_via_numeric_array_escape,serialize_via_numeric_array,deserialize_via_numeric_array,serialize_via_marshal,deserialize_via_marshal
from cdsware.dbquery import run_sql
from cdsware.config import *
from cdsware.webpage import page, pageheaderonly, pagefooteronly
text = bibharvest_templates.tmpl_print_brs(cdslang, 1)
text += bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Source name", name = "oai_src_name", value = oai_src_name)
text += bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Base URL", name = "oai_src_baseurl", value = oai_src_baseurl)
text += bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Metadata prefix", name = "oai_src_prefix", value = oai_src_prefix)
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Frequency", name = "oai_src_frequency", valuenil = "- select frequency -" , values = freqs, lastval = oai_src_frequency)
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Postprocess", name = "oai_src_post", valuenil = "- select mode -" , values = posts, lastval = oai_src_post)
text += bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Bibconvert configuration file", name = "oai_src_config", value = oai_src_config)
text += bibharvest_templates.tmpl_print_brs(cdslang, 2)
output += createhiddenform(action="editsource#1",
text=text,
button="Modify",
oai_src_id=oai_src_id,
ln=ln,
confirm=1)
if confirm in [1, "1"] and not oai_src_name:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a name for the source.")
elif confirm in [1, "1"] and not oai_src_prefix:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a metadata prefix.")
elif confirm in [1, "1"] and not oai_src_baseurl:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a base url.")
elif confirm in [1, "1"] and not oai_src_frequency:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please choose a frequency of harvesting")
elif confirm in [1, "1"] and not oai_src_post:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please choose a postprocess mode")
elif confirm in [1, "1"] and (oai_src_post=="h-c" or oai_src_post=="h-c-u") and (not oai_src_config or validatefile(oai_src_config)!=0):
output += bibharvest_templates.tmpl_print_info(cdslang, "You selected a postprocess mode which involves conversion.")
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a valid full path to a bibConvert config file or change postprocess mode.")
elif oai_src_id > -1 and confirm in [1, "1"]:
if not oai_src_frequency:
oai_src_frequency = 0
if not oai_src_config:
oai_src_config = "NULL"
if not oai_src_post:
oai_src_post = "h"
res = modify_oai_src(oai_src_id, oai_src_name, oai_src_baseurl, oai_src_prefix, oai_src_frequency, oai_src_config, oai_src_post)
text = bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Source name", name = "oai_src_name", value = oai_src_name)
metadatas = findMetadataFormats(oai_src_baseurl)
prefixes = []
for value in metadatas:
prefixes.append([value, str(value)])
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Metadata prefix", name = "oai_src_prefix", valuenil = "- select prefix -" , values = prefixes, lastval = oai_src_prefix)
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Frequency", name = "oai_src_frequency", valuenil = "- select frequency -" , values = freqs, lastval = oai_src_frequency)
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Starting date", name = "oai_src_lastrun", valuenil = "- select a date -" , values = dates, lastval = oai_src_lastrun)
text += bibharvest_templates.tmpl_admin_w200_select(ln = cdslang, title = "Postprocess", name = "oai_src_post", valuenil = "- select mode -" , values = posts, lastval = oai_src_post)
text += bibharvest_templates.tmpl_admin_w200_text(ln = cdslang, title = "Bibconvert configuration file", name = "oai_src_config", value = oai_src_config)
text += bibharvest_templates.tmpl_print_brs(cdslang, 2)
output += createhiddenform(action="addsource#1",
text=text,
button="Add OAI Source",
oai_src_baseurl=oai_src_baseurl,
ln=ln,
confirm=1)
if confirm in [1, "1"] and not oai_src_name:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a name for the source.")
elif confirm in [1, "1"] and not oai_src_prefix:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a metadata prefix.")
elif confirm in [1, "1"] and not oai_src_frequency:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please choose a frequency of harvesting")
elif confirm in [1, "1"] and not oai_src_lastrun:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please choose the harvesting starting date")
elif confirm in [1, "1"] and not oai_src_post:
output += bibharvest_templates.tmpl_print_info(cdslang, "Please choose a postprocess mode")
elif confirm in [1, "1"] and (oai_src_post=="h-c" or oai_src_post=="h-c-u") and (not oai_src_config or validatefile(oai_src_config)!=0):
output += bibharvest_templates.tmpl_print_info(cdslang, "You selected a postprocess mode which involves conversion.")
output += bibharvest_templates.tmpl_print_info(cdslang, "Please enter a valid full path to a bibConvert config file or change postprocess mode.")
elif oai_src_name and confirm in [1, "1"]:
if not oai_src_frequency:
oai_src_frequency = 0
if not oai_src_lastrun:
oai_src_lastrun = 1
if not oai_src_config:
oai_src_config = "NULL"
if not oai_src_post:
oai_src_post = "h"
res = add_oai_src(oai_src_name, oai_src_baseurl, oai_src_prefix, oai_src_frequency, oai_src_lastrun, oai_src_config, oai_src_post)
"Encode special chars in string for XML-compliancy."
if strxml == None:
return strxml
else:
strxml = string.replace(strxml, '&', '&')
strxml = string.replace(strxml, '<', '<')
return strxml
def escape_space(strxml):
"Encode special chars in string for URL-compliancy."
strxml = string.replace(strxml, ' ', '%20')
return strxml
def encode_for_url(strxml):
"Encode special chars in string for URL-compliancy."
strxml = string.replace(strxml, '%', '%25')
strxml = string.replace(strxml, ' ', '%20')
strxml = string.replace(strxml, '?', '%3F')
strxml = string.replace(strxml, '#', '%23')
strxml = string.replace(strxml, '=', '%3D')
strxml = string.replace(strxml, '&', '%26')
strxml = string.replace(strxml, '/', '%2F')
strxml = string.replace(strxml, ':', '%3A')
strxml = string.replace(strxml, ';', '%3B')
strxml = string.replace(strxml, '+', '%2B')
return strxml
def oai_header(args, verb):
"Print OAI header"
out = ""
out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"
out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"
if verb:
out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))
out = out + " <%s>\n" % verb
else:
out = out + " <request>%s</request>\n" % (oaigetrequesturl(args))
return out
def oai_footer(verb):
"Print OAI footer"
out = ""
if verb:
out = "%s </%s>\n" % (out, verb)
out = out + "</OAI-PMH>\n"
return out
def oai_error_header(args, verb):
"Print OAI header"
out = ""
### out = "Content-Type: text/xml\n\n"
out = out + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + "\n"
out = out + "<OAI-PMH xmlns=\"http://www.openarchives.org/OAI/2.0/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd\">\n"
out = out + " <responseDate>" + oaigetresponsedate() + "</responseDate>\n"
out = out + " <request verb=\"%s\">%s</request>\n" % (verb, oaigetrequesturl(args))
return out
def oai_error_footer(verb):
"Print OAI footer"
out = verb
out = "</OAI-PMH>\n"
return out
def get_field(sysno, field):
"Gets list of field 'field' for the record with 'sysno' system number."
out = []
digit = field[0:2]
bibbx = "bib%sx" % digit
bibx = "bibrec_bib%sx" % digit
query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag='%s'" % (bibbx, bibx, sysno, field)
"Returns the first MySQL BIB ID for the OAI identifier 'identifier', if it exists."
sysno = None
if identifier:
query = "SELECT DISTINCT(bb.id_bibrec) FROM bib%sx AS bx, bibrec_bib%sx AS bb WHERE bx.tag='%s' AND bb.id_bibxxx=bx.id AND bx.value='%s'" % (oaiidfield[0:2], oaiidfield[0:2], oaiidfield, identifier)
res = run_sql(query)
for row in res:
sysno = row[0]
return sysno
def oaigetsysnolist(set, fromdate, untildate):
"Returns list of system numbers for the OAI set 'set', modified from 'date_from' until 'date_until'."
out_dict = {} # dict to hold list of out sysnos as its keys
if set:
query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s' AND bx.value='%s'" % (oaiidfield[0:2], oaiidfield[0:2], oaisetfield, set)
else:
query = "SELECT DISTINCT bibx.id_bibrec FROM bib%sx AS bx LEFT JOIN bibrec_bib%sx AS bibx ON bx.id=bibx.id_bibxxx LEFT JOIN bibrec AS b ON b.id=bibx.id_bibrec WHERE bx.tag='%s'" % (oaiidfield[0:2], oaiidfield[0:2], oaiidfield)
"Check if record with recid has been deleted. Return 1 if deleted."
query = "select a.id from bibrec as a left join bibrec_bib98x as b on a.id=b.id_bibrec left join bib98x as c on b.id_bibxxx=c.id where c.value='DELETED' and a.id=%s" % recid
res = run_sql(query)
for item in res:
if item == None:
return 0
else:
return 1
def oaigenresumptionToken():
"Generates unique ID for resumption token management."
return md5.new(str(time.time())).hexdigest()
def oaicachein(resumptionToken, sysnos):
"Stores or adds sysnos in cache. Input is a string of sysnos separated by commas."
out = out + oai_error("badVerb", "Illegal OAI verb")
## defined args
#
#
for param in arguments.keys():
if out_args.has_key(param):
pass
else:
out = out + oai_error("badArgument", "The request includes illegal arguments")
## unique args
#
#
for param in arguments.keys():
if (arguments[param] == "Error"):
out = out + oai_error("badArgument", "The request includes illegal arguments")
## resumptionToken exclusive
#
#
if ((arguments['from'] != "" or arguments['until'] != "" or arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "") and arguments['resumptionToken'] != ""):
out = out + oai_error("badArgument", "The request includes illegal arguments")
## datestamp formats
#
#
if arguments['from'] != "" and arguments['from'] != "":
from_length = len(arguments['from'])
if check_date(arguments['from'], "T00:00:00Z") == "":
out = out + oai_error("badArgument", "Bad datestamp format in from")
else:
from_length = 0
if arguments['until'] != "" and arguments['until'] != "":
until_length = len(arguments['until'])
if check_date(arguments['until'], "T23:59:59Z") == "":
out = out + oai_error("badArgument", "Bad datestamp format in until")
else:
until_length = 0
if from_length != 0:
if until_length != 0:
if from_length != until_length:
out = out + oai_error("badArgument", "Bad datestamp format")
if arguments['from'] != "" and arguments['until'] != "" and arguments['from'] > arguments['until']:
out = out + oai_error("badArgument", "Wrong date")
## Identify exclusive
#
#
if (arguments['verb'] =="Identify" and (arguments['metadataPrefix'] != "" or arguments['identifier'] != "" or arguments['set'] != "" or arguments['from'] != "" or arguments['until'] != "" or arguments['resumptionToken'] != "")):
out = out + oai_error("badArgument", "The request includes illegal arguments")
## parameters for GetRecord
#
#
if arguments['verb'] =="GetRecord" and arguments['identifier'] == "":
out = out + oai_error("badArgument", "Record identifier missing")
if arguments['verb'] =="GetRecord" and arguments['metadataPrefix'] == "":
out = out + oai_error("badArgument", "Missing metadataPrefix")
## parameters for ListRecords and ListIdentifiers
#
#
if (arguments['verb'] =="ListRecords" or arguments['verb'] =="ListIdentifiers") and (arguments['metadataPrefix'] == "" and arguments['resumptionToken'] == ""):
out = out + oai_error("badArgument", "Missing metadataPrefix")
## Metadata prefix defined
#
#
if arguments.has_key('metadataPrefix'):
if ((arguments['metadataPrefix'] in params['metadataPrefix']) or (params['metadataPrefix'] == "")):
pass
else:
out = out + oai_error("badArgument", "Missing metadataPrefix")
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""CDSware BibIndex Administrator Interface."""
import cgi
import re
import MySQLdb
import Numeric
import os
import urllib
import time
import random
from zlib import compress,decompress
from mod_python import apache
from cdsware.bibrankadminlib import write_outcome,modify_translations,get_def_name,get_i8n_name,get_name,get_rnk_nametypes,get_languages,check_user,is_adminuser,adderrorbox,addadminbox,tupletotable,tupletotable_onlyselected,addcheckboxes,createhiddenform,serialize_via_numeric_array_dumps,serialize_via_numeric_array_compr,serialize_via_numeric_array_escape,serialize_via_numeric_array,deserialize_via_numeric_array,serialize_via_marshal,deserialize_via_marshal
from cdsware.dbquery import run_sql
from cdsware.config import *
from cdsware.webpage import page, pageheaderonly, pagefooteronly
<td>1. <small><a href="%s/admin/bibindex/bibindexadmin.py/field?ln=%s&mtype=perform_showfieldoverview#1">Overview of logical fields</a></small></td>
<td>1. <small><a href="%s/admin/bibindex/bibindexadmin.py/editfield?fldID=%s&ln=%s&mtype=perform_modifyfield">Modify field code</a></small></td>
<td>3. <small><a href="%s/admin/bibindex/bibindexadmin.py/editfield?fldID=%s&ln=%s&mtype=perform_modifyfieldtags">Modify MARC tags</a></small></td>
<td>5. <small><a href="%s/admin/bibindex/bibindexadmin.py/editfield?fldID=%s&ln=%s&mtype=perform_showdetailsfield">Show field usage</a></small></td>
<td>1. <small><a href="%s/admin/bibindex/bibindexadmin.py/editindex?idxID=%s&ln=%s&mtype=perform_modifyindex">Modify index name / descriptor</a></small></td>
<td>3. <small><a href="%s/admin/bibindex/bibindexadmin.py/editindex?idxID=%s&ln=%s&mtype=perform_modifyindexfields">Modify index fields</a></small></td>
text += """<b><span class="info">By deleting an index, you may also loose any indexed data in the forward and reverse table for this index.</span></b><br>"""
text += """Do you want to delete the index '%s' and all its relations and definitions.""" % (idx[0][1])
query = "SELECT tag.value FROM tag, field_tag WHERE tag.id=field_tag.id_tag AND field_tag.id_field=%d ORDER BY field_tag.score DESC,tag.value ASC" % field_id
subtitle = """<a name="4"></a>3. Modify MARC tags for the logical field '%s'   <small>[<a title="See guide" href="%s/admin/bibindex/guide.html">?</a>]</small>""" % (fld_dict[int(fldID)], weburl)
output = """<dl>
<dt>Menu</dt>
<dd><a href="%s/admin/bibindex/bibindexadmin.py/addtag?fldID=%s&ln=%s#4.1">Add MARC tag</a></dd>
<dd><a href="%s/admin/bibindex/bibindexadmin.py/deletetag?fldID=%s&ln=%s#4.1">Delete unused MARC tags</a></dd>
The bibrank web interface enables you to modify the configuration of most aspects of BibRank. For full functionality, it is advised to
let the http-daemon have write/read access to your cdsware/etc/bibrank directory. If this is not wanted, you have to edit the configuration files from the console using your favourite text editor.
<a name="mi"></a><h3>3.1 Main interface</h3>
In the main interface screen, you see a list of all rank methods currently added. Each rank method is identified by the rank method code. To find out about the functionality available, check out the topics below.
</br></br><b>Explanation of concepts</b>
<blockquote>
<pre>
Rank method:
A method responsible for creating the necessary data to rank a result.
Translations:
Each rank method may have many names in many languages.
Collections:
Which collections the rank method should be visible in.
</pre>
</blockquote>
<a name="ar"></a><h3>3.2 Add rank method</h3>
When pressing the link in the upper right corner from the main interface, you will see the interface for adding a new rank method. The two available options that needs to be decided upon, are the bibrank code and the template to use, both values can be changed later. The bibrank code is used by the bibrank daemon to run the method, and should be fairly short without spaces. Which template you are using, decides how the ranking will be done, and must before used, be changed to suit your cdsware configuration. When confirming to add a new rank method, it will be added to the list of possible rank methods, and a configuration file will be created if the httpd user has proper rights to the 'cdsware/etc/bibrank' directory. If not, the file has to manually be created with the name 'bibrankcode.cfg' where bibrankcode is the same as given in the interface.
<a name="sd"></a><h3>3.3 Show details of rank method</h3>
This interface gives you an overview of the current status of the rank method, and gives direct access to the various interfaces for changing the configuration.
In the overview section, you see the bibrank code, for use with the bibrank daemon, and the date for the last run of the rank method.
In the statistics section you see how many records have been added to the rank method and other statistic data. In the collection part, the collections which the rank method is visible to is shown. The translations part shows the various translations in the languages available in cdsware. On the bottom the configuration file is shown, if accessible.
<a name="mr"></a><h3>3.4 Modify rank method</h3>
This interface gives access to modify the bibrank code given when creating the rank method and the configuration file of the rank method, if the file can be accessed. If not, it may not exist, or the httpd user doesn't have enough rights to read the file. On the bottom of the interface, it is possible to choose a template, see it, and copy it over the old rank method configuration if wanted. Remember that the values present in the template is an example, and must be changed where necessary. See this documentation for information about this, and the 'BibRank Internals' link below for additional information.
<a name="dr"></a><h3>3.5 Delete rank method</h3>
If it is necessary to delete a rank method, some precautions must be taken since the configuration of the method will be lost. When deleting a rank method, the configuration file will also be deleted ('cdsware/etc/bibrank/bibrankcode.cfg' where bibrankcode is the code of the rank method) if accessible to the httpd user. If not, the file can be deleted manually from console. Any bibrank tasks scheduled to run the deleted rank method must be modified or deleted manually.
<a name="mt"></a><h3>3.6 Modify translations</h3>
If you want to use internalisation of the rank method names, you have to add them using the 'Modify translations' interface. Below a list of all the languages used in the cdsware installation will be shown with the possibility to add the translation for each language.
If a rank method should be visible to the users of the cdsware search interface, it must be enabled for one or several collections. A rank method can be visible in the search interface of the whole site, or just one collection. The collections in the upper list box does not show the rank method in the search interface to the user. To change this select the wanted collection and press 'Enable' to enable the rank method for this collection. The collections that the method has been activated for, is shown in the lower list box. To remove a collection, select it and press the 'Disable' button to remove it from the list of collections which the rank method is enabled for.
<a name="bd"></a><h2>4. BibRank Daemon</h2>
The bibrank daemon read the necessary metadata from the cdsware database and combines the read metadata
in different ways to create the ranking data necessary at searchtime to fast be able to rank the results.
<a name="cli1"></a><h3>4.1 Command Line Interface</h3>
<blockquote>
<pre>
Usage bibrank:
bibrank -wjif -a --id=0-30000,30001-860000 --verbose=9
bibrank -wjif -d --modified='2002-10-27 13:57:26'
bibrank -wwrd --recalculate --collection=Articles
bibrank -wwrd -a -i 234-250,293,300-500 -u admin@cdsware
Ranking options:
-w, --run=r1[,r2] runs each rank method in the order given
-c, --collection=c1[,c2] select according to collection
-i, --id=low[-high] select according to doc recID
-m, --modified=from[,to] select according to modification date
-l, --lastupdate select according to last update
-a, --add add or update words for selected records
-d, --del delete words for selected records
-S, --stat show statistics for a method
-R, --recalculate recalculate weigth data, used by word frequency method
should be used if ca 1% of the document has been changed
since last time -R was used
Repairing options:
-k, --check check consistency for all records in the table(s)
check if update of ranking data is necessary
-r, --repair try to repair all records in the table(s)
Scheduling options:
-u, --user=USER user name to store task, password needed
-s, --sleeptime=SLEEP time after which to repeat tasks (no)
e.g.: 1s, 30m, 24h, 7d
-t, --time=TIME moment for the task to be active (now)
e.g.: +15s, 5m, 3h , 2002-10-27 13:57:26
General options:
-h, --help print this help and exit
-V, --version print version and exit
-v, --verbose=LEVEL verbose level (from 0 to 9, default 1)
</pre>
</blockquote>
<a name="ubd"></a><h3>4.2 Using BibRank</h3>
<h4>Step 1 - Adding the rank option to the search interface</h4>
To be able to add the needed ranking data to the database, you first have to add the rank method to the database, and
add the wished code you want to use together with it. The name of the configuration file in the next section, needs to
have the same name as the code stored in the database.
<h4>Step 2 - Get necessary external data (ex. jif values)</h4>
Find out what is necessary of data for each method. The bibrankgkb documentation below may be of assistance.
The configuration files for the different rank methods has different option, so verify that you are using the correct
configuration file and rank method. A template for each rank method exists as examples, but may not work on all configurations of CDSware.
For a description of each rank method and the configuration necessary, check section 6 below.
<h4>Step 4 - Add the ranking method as a scheduled task</h4>
When the configuration is okay, you can add the bibrank daemon to the task scheduler using the scheduling options. The daemon can then do a update of the rank method once each day or similar automatically.
<br><br><b>Example</b>
<blockquote>
<pre>
$ bibrank -wjif -r
Task #53 was successfully scheduled for execution.
</pre>
</blockquote>
It is adviced to run the BibRank daemon using no parameters, since the default settings then will be used.
<br><br><b>Example</b>
<blockquote>
<pre>
$ bibrank
Task #2505 was successfully scheduled for execution.
</pre>
</blockquote>
<h4>Step 5 - Running bibrank manually</h4>
If BibRank is scheduled without any parameters, and no records has been modified, you may get a output like shown below.
<br><br><b>Example</b>
<blockquote>
<pre>
$ bibrank 2505
2004-09-07 17:51:46 --> Task #2505 started.
2004-09-07 17:51:46 -->
2004-09-07 17:51:46 --> Running rank method: Number of downloads.
2004-09-07 17:51:47 --> No new records added since last time method was run
2004-09-07 17:52:11 --> Number of lines read from knowledgebase file: 420
2004-09-07 17:52:11 --> Number of records available in rank method: 0
2004-09-07 17:52:12 -->
2004-09-07 17:52:12 --> Running rank method: Word frequency
2004-09-07 17:52:13 --> rnkWORD01F contains 256842 words from 677912 records
2004-09-07 17:52:14 --> rnkWORD01F is in consistent state
2004-09-07 17:52:14 --> Using the last update time for the rank method
2004-09-07 17:52:14 --> No new records added. rnkWORD01F is up to date
2004-09-07 17:52:14 --> rnkWORD01F contains 256842 words from 677912 records
2004-09-07 17:52:14 --> rnkWORD01F is in consistent state
2004-09-07 17:52:14 --> Task #2505 finished.
</pre>
</blockquote>
<h4>Step 6 - Fast update of modified records</h4>
If you just want to update the latest additions or modified records, you may want to do a faster update by running the daemon without the recalculate option. (the recalculate option is off by default). This may cause lower accurancy when ranking.
<a name="brm"></a><h2>5. BibRank Methods</h2>
Each BibRank method has a configuration file which contains different parameters and sections necessary to do the ranking.
<a name="str"></a><h3>5.1 Single tag rank method</h3>
This method uses one MARC tag together with a file containing possible values for this MARC tag together with a ranking value. This data is used to create a structure containing the record id associated with the ranking value based on the content of the tag. The method can be used for various ways of ranking like ranking by Journal Impact Factor, or use it to let certain authors always appear top of a search.
The parameters needed to be configured for this method is the 'tag','kb_src' and 'check_mandatory_tags'.
##The function which is responsible for doing the work. Should not be changed
function = single_tag_rank_method
##This section must be available if the single_tag_rank_method is going to be used
[single_tag_kb]
##The tag which got the value to be searched for on the left side in the kb file (like the journal name)
tag = 909C4p
##The path to the kb file which got the content of the tag above on left side, and value on the left side
kb_src = /log/cdsware-DEMODEV/etc/bibrank/jif.kb
##Tags that must be included for a record to be added to the ranking data, to disable remove tags
check_mandatory_tags = 909C4c,909C4v,909C4y
</protect>
</blockquote>
The kb_src file must contain data on the form:
<blockquote>
Phys. Rev., D---3.838
Phys. Rev. Lett.---6.462
Phys. Lett., B---4.213
Nucl. Instrum. Methods Phys. Res., A---0.964
J. High Energy Phys.---8.664
</blockquote>
The left side must match the content of the tag mentioned in the tag variable.
</pre>
<a name="wrd"></a><h3>5.2 Word Similarity/Similar Records</h3>
The Word Similarity/Similar Records method uses the content of the tags selected to determine which records is most relevant to a query, or most similar to a selected record. This method got a lot of parameters to configure, and it may need some tweaking to get the best result. The BibRank code for this method has to be 'wrd' for it to work. For best result, it is adviced to install the stemming module mentioned in INSTALL, and use a stopword list containing stopwords in the languages the records exists in. The stemmer and stopword list is used to get better results and to limit the size of the index, thus making ranking faster and more accurate. For best result with the stemmer, it is important to mark each tag to be used with the most common language the value of the tag may be in. It is adviced to not change the 'function','table' and the parameters under [find_similar]. If the stemmer is not installed, to assure that no problems exists, the 'stem_if_avail' parameter should be set to 'no'. Each tag to be used by the method has to be given a point. The number of points describes how important one word is in this tag.
When running BibRank to update the index for this rank method, it is not necessary to recalculate each time, but when large number of records has been updated/added, it can be wise to recalculate using the recalculate parameter of BibRank.
<br><br><b>Example</b>
<pre>
<blockquote>
<protect>
[rank_method]
function = word_similarity
[word_similarity]
stemming = en
table = rnkWORD01F
stopword = True
relevance_number_output_prologue = (
relevance_number_output_epilogue = )
#relevance_number_output_prologue = <!-
#relevance_number_output_epilogue = -->
#MARC tag,tag points, tag language
tag1 = 6531_a, 2, en
tag2 = 695__a, 1, en
tag3 = 6532_a, 1, en
tag4 = 245__%, 10, en
tag5 = 246_%, 1, fr
tag6 = 250__a, 1, en
tag7 = 711__a, 1, en
tag8 = 210__a, 1, en
tag9 = 222__a, 1, en
tag10 = 520__%, 1, en
tag11 = 590__%, 1, fr
tag12 = 111__a, 1, en
tag13 = 100__%, 2, none
tag14 = 700__%, 1, none
tag15 = 721__a, 1, none
[find_similar]
max_word_occurence = 0.05
min_word_occurence = 0.00
min_word_length = 3
min_nr_words_docs = 3
max_nr_words_upper = 20
max_nr_words_lower = 10
default_min_relevance = 75
</protect>
</blockquote>
</pre>
<b>Explanation:</b>
<pre>
<blockquote>
<protect>
[rank_method]
#internal name for the bibrank program, do not modify
function = word_similarity
[word_similarity]
#if stemmer is available, default stemminglanguage should be given here. Adviced to turn off if not installed
stemming = en
#the internal table to load the index tables from.
table = rnkWORD01F
#remove stopwords?
stopword = True
#text to show before the rank value when the search result is presented. <-- to hide result
relevance_number_output_prologue = (
#text to show after the rank value when the search result is presented. --> to hide result
relevance_number_output_epilogue = )
#MARC tag,tag points, tag language
#a list of the tags to be used, together with a number describing the importance of the tag, and the
#most common language for the content. Not all languages are supported. Among the supported ones are:
tag4 = 245__%, 10, en #title, the words in the title is usually describing a record very good.
tag5 = 246_% , 1, fr #french title
tag6 = 250__a, 1, en #title
tag7 = 711__a, 1, en #title
tag8 = 210__a, 1, en #abbreviated
tag9 = 222__a, 1, en #key title
[find_similar]
#term should exist in maximum X/100% of documents
max_word_occurence = 0.05
#term should exist in minimum X/100% of documents
min_word_occurence = 0.00
#term should be atleast 3 characters long
min_word_length = 3
#term should be in atleast 3 documents or more
min_nr_words_docs = 3
#do not use more than 20 terms for "find similar"
max_nr_words_upper = 20
#if a document contains less than 10 terms, use much used terms too, if not ignore them
max_nr_words_lower = 10
#default minimum relevance value to use for find similar
default_min_relevance = 75
</protect>
</blockquote>
Tip: When executing a search using a ranking method, you can add "verbose=1" to the list of parameteres
in the URL to see which terms have been used in the ranking.
</pre>
<a name="cmb"></a><h3>5.3 Combine method</h3>
The 'Combine method' is running each method mentioned in the config file and adding the score together
based on the importance of the method given by the percentage.
<br><br><b>Example</b>
<pre>
<blockquote>
<protect>
[rank_method]
function = combine_method
[combine_method]
method1 = cern_jif,33
method2 = cern_acc,33
method3 = wrd,33
relevance_number_output_prologue = (
relevance_number_output_epilogue = )
</protect>
</blockquote>
</pre>
<b>Explanation:</b>
<pre>
<blockquote>
<protect>
[rank_method]
#tells which method to use, do not change
function = combine_method
[combine_method]
#each line tells which method to use, the code is the same as in the BibRank interface, the number describes how
#much of the total score the method should count.
method1 = jif,50
method2 = wrd,50
#text to be shown before the rank value on the search result screen.
relevance_number_output_prologue = (
#text to be shown after the rank value on the search result screen.
relevance_number_output_epilogue = )
</protect>
</blockquote>
</pre>
<a name="bt"></a><h2>6. bibrankgkb Tool</h2>
For some ranking methods, like the single_tag_rank method, a knowledge base file (kb) with the needed data in the correct format is necessary. This file can be created using the bibrankgkb tool which can read the data either from
the cdsware database, from several web pages using regular expressions, or from another file. In case one source
has another naming convention, bibrank can convert between them using a convert file.
<a name="cli2"></a><h3>6.1 Command Line Interface</h3>
-i, --input=file input file, default from /etc/bibrank/bibrankgkb.cfg
-o, --output=file output file, will be placed in current folder
General options:
-h, --help print this help and exit
-V, --version print version and exit
-v, --verbose=LEVEL verbose level (from 0 to 9, default 1)
</blockquote>
</pre>
<a name="ubt"></a><h3>6.2 Using bibrankgkb</h3>
<h4>Step 1 - Find sources</h4>
Since some of the data used for ranking purposes is not freely available, it cannot be bundled with CDSware. To get hold of the necessary data,
you may find it useful to ask your library if they have a copy of the data that can be used (like the Journal Impact Factors from the Science Citation Index), or use google to search the web for any public source.
<h4>Step 2 - Create configuration file</h4>
The default configuration file is shown below.
<protect>
<pre>
<blockquote>
##The main section
[bibrankgkb]
##The url to a web page with the data to be read, does not need to have the same name as this one, but if there
are several links, the url parameter should end with _0->
kb_3 = SELECT id_bibrec,value FROM bib93x,bibrec_bib93x WHERE tag='938__f' AND id_bibxxx=id
kb_4 = SELECT id_bibrec,value FROM bib21x,bibrec_bib21x WHERE tag='210__a' AND id_bibxxx=id
##This points to the url above (the common part of the url is 'url_' followed by a number
kb_5 = url_%s
##This is the part that will be read by the bibrankgkb tool to determine what to read.
##The first two part (separated by ,,) gives where to look for the conversion file (which convert
##the names between to formats), and the second part is the data source. A conversion file is not
##needed, as shown in create_0. If the source is from a file, url or the database, it must be
##given with file,www or db. If several create lines exists, each will be read in turn, and added
##to a common kb file.
##So this means that:
##create_0: Load from file in variable kb_1 without converting
##create_1: Load from file in variable kb_2 using convertion from file kb_2_filter
##create_3: Load from www using url in variable kb_5 and regular expression in url_regexp
##create_4: Load from database using sql statements in kb_4 and kb_5
create_0 = ,, ,,file,,%(kb_1)s
create_1 = file,,%(kb_2_filter)s,,file,,%(kb_2)s
#create_2 = ,, ,,www,,%(kb_5)s,,%(url_regexp)s
#create_3 = ,, ,,db,,%(kb_4)s,,%(kb_4)s
</pre>
</blockquote>
</protect>
When you have found a source for the data, created the configuration file, it may be necessary to
create an convertion file, but this depends on the conversions used in the available data versus
the convertion used in your cdsware installation.
<br>
The available data may look like this:
<pre>
<blockquote>
COLLOID SURFACE A---1.98
</pre>
</blockquote>
But in cdsware you are using:
<pre>
<blockquote>
Colloids Surf., A---1.98
</pre>
</blockquote>
By using a convertion file like:
<pre>
<blockquote>
COLLOID SURFACE A---Colloids Surf., A
</pre>
</blockquote>
You can convert the source to the correct naming convention.
<pre>
<blockquote>
Colloids Surf., A---1.98
</pre>
</blockquote>
<h4>Step 3 - Run tool</h4>
When ready to run the tool, you may either use the default file (/etc/bibrank/bibrankgkb.cfg), or use another one by giving it using the input variable '--input'.
If you want to test the configuration, you can use '--verbose=9' to output on screen, or if you want to save it to a file, use
'--output=filename', but remember that the file will be saved in the program directory.
This page summarizes all the information suitable to dig inside
BibRank and it's different ranking methods.
<blockquote>
<dl>
<dt><a href="single_tag_rank.html">Single tag rank method</a>
<dd>Ex. of use: Journal Impact Factor. Explains what this ranking method<br>
does to create the ranking data for use when ranking based on<br>
Journal Impact Factor or similar methods.</dd>
</dt>
<dt><a href="word_similarity.html">Word Similarity/Similar Records methods</a>
<dd>Ex. of use: Ranking based on the importance of words in the whole<br>
collection and in a single record. Explains how the ranking data is calculated.</dd>
</dt>
<dt><a href="api.html">Bibrank Record Sorter API</a>
<dd>How to call the Bibrank Record Sorter library from your python programs if necessary.</dd></dt>
<dt><a href="bibrankgkb.html">Create knowledgebase files (bibrankgkb)</a> <dd>Explains how to create knowledgebase files using the bibrankgkb tool</dd></dt>
"""Return list of field values for field tag inside record RECID."""
out = []
if tag == "001___":
out.append(str(recID))
else:
digit = tag[0:2]
bx = "bib%sx" % digit
bibx = "bibrec_bib%sx" % digit
query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag LIKE '%s'" "ORDER BY bibx.field_number, bx.tag ASC" % (bx, bibx, recID, tag)
sql_query = "select count(*) from rnkDOWNLOADS where id_bibrec=%s %s and download_time>='%s' and download_time<'%s';" % (id_bibrec, id_bibdoc_query_addition, date1, date2)
res = run_sql(sql_query)[0][0]
list_tuple.append((elem[0], res))
#list_tuple = sort_list_tuple_by_date(list_tuple)
return (list_tuple)
def sort_list_tuple_by_date(list_tuple):
"""Sort a list of tuple of the forme ("09/2004", 3)according to the
"""filter all the duplicate downloads per user for each hour intervall"""
for k in keys:
id_bibdocs = run_sql("select distinct id_bibdoc from rnkDOWNLOADS where id_bibrec=%s" % k)
for bibdoc in id_bibdocs:
values = run_sql("""select DATE_FORMAT(download_time,"%%Y-%%m-%%d %%H"), client_host from rnkDOWNLOADS where id_bibrec=%s and id_bibdoc=%s and download_time >=\"%s\";""" % (k, bibdoc[0], last_updated))
for val in values:
date_res = val[0]
date1 = "%s:00:00" % (date_res)
date2 = compute_next_hour(date_res)
duplicates = (run_sql("select count(*) from rnkDOWNLOADS where id_bibrec=%s and id_bibdoc=%s and download_time>='%s' and download_time<'%s' and client_host=%s;" % (k, bibdoc[0], date1, date2, val[1]))[0][0])-1
run_sql("delete from rnkDOWNLOADS where id_bibrec=%s and id_bibdoc=%s and download_time>='%s' and download_time<'%s' and client_host=%s limit %s;" % (k, bibdoc[0], date1, date2, val[1], duplicates))
"""filter all the duplicate downloads per user for each hour intervall"""
for k in keys:
values = run_sql("""select DATE_FORMAT(download_time,"%%Y-%%m-%%d %%H"), client_host from rnkDOWNLOADS where id_bibrec=%s and download_time >=\"%s\";""" % (k, last_updated))
for val in values:
date_res = val[0]
date1 = "%s:00:00" % (date_res)
date2 = compute_next_hour(date_res)
duplicates = (run_sql("select count(*) from rnkDOWNLOADS where id_bibrec=%s and download_time>='%s' and download_time<'%s' and client_host=%s;" % (k, date1, date2, val[1]))[0][0])-1
run_sql("delete from rnkDOWNLOADS where id_bibrec=%s and download_time>='%s' and download_time<'%s' and client_host=%s limit %s;" % (k, date1, date2, val[1], duplicates))
def compute_next_hour(date_res):
"""treat the change of the year, of (special)month etc.. and return the date in database format"""
res2 = run_sql("select id_bibrec,count(*) from rnkDOWNLOADS where client_host in %s and id_bibrec in %s and id_bibrec != %s group by id_bibrec;" % (tuple_client_host, tuple_string_id_bibrec_list, j)) #0.0023 par requete
term_rec - the number of records which contains this term
max_occ - max frequency of the term allowed
min_occ - min frequence of the term allowed
termlength - the minimum length of the terms allowed"""
try:
if is_stopword(term, 1) or (len(term) <= termlength) or ((float(term_rec) / float(col_size)) >= max_occ) or ((float(term_rec) / float(col_size)) <= min_occ):
return ""
if int(term):
return ""
except StandardError, e:
pass
return "true"
def create_rnkmethod_cache():
"""Create cache with vital information for each rank method."""
global methods
bibrank_meths = run_sql("SELECT name from rnkMETHOD")
"""Checks if a method is valid for the collection given"""
enabled_colls = dict(run_sql("SELECT id_collection, score from collection_rnkMETHOD,rnkMETHOD WHERE id_rnkMETHOD=rnkMETHOD.id AND name='%s'" % rank_method_code))
colID = int(colID)
if enabled_colls.has_key(colID):
return 1
else:
while colID:
colID = run_sql("SELECT id_dad FROM collection_collection WHERE id_son=%s" % colID)
if colID and enabled_colls.has_key(colID[0][0]):
return 1
elif colID:
colID = colID[0][0]
return 0
def get_bibrank_methods(collection, ln=cdslang):
"""Returns a list of rank methods and the name om them in the language defined by the ln parameter, if collection is given, only methods enabled for that collection is returned."""
if not globals().has_key('methods'):
create_rnkmethod_cache()
avail_methods = []
for (rank_method_code, options) in methods.iteritems():
if options.has_key("function") and is_method_valid(collection, rank_method_code):
if len(tf_values) <= methods[rank_method_code]["max_nr_words_lower"] or (len(term_recs) >= methods[rank_method_code]["min_nr_words_docs"] and (((float(len(term_recs)) / float(methods[rank_method_code]["col_size"])) <= methods[rank_method_code]["max_word_occurence"]) and ((float(len(term_recs)) / float(methods[rank_method_code]["col_size"])) >= methods[rank_method_code]["min_word_occurence"]))): #too complicated...something must be done
lwords.append((t, methods[rank_method_code]["rnkWORD_table"])) #list of terms used
(recdict, rec_termcount) = calculate_record_relevance_findsimilar((t, round(tf, 4)) , term_recs, hitset, recdict, rec_termcount, verbose, "true") #true tells the function to not calculate all unimportant terms
if len(tf_values) > methods[rank_method_code]["max_nr_words_lower"] and (len(lwords) == methods[rank_method_code]["max_nr_words_upper"] or tf < 0):
break
if len(recdict) == 0 or len(lwords) == 0:
return (None, "Could not find any similar documents, possibly because of error in ranking data.", "", voutput)
write_message("......Processing records #%s-%s" % (recids, recide))
recs = run_sql("SELECT id_bibrec,value FROM bib%sx,bibrec_bib%sx WHERE tag='%s' AND id_bibxxx=id and id_bibrec >=%s and id_bibrec<=%s" % (tag[0:2], tag[0:2], tag, recids, recide))
valid = HitSet(Numeric.ones(cfg_max_recID + 1))
for key in tags:
newset = HitSet()
newset.addlist(run_sql("SELECT id_bibrec FROM bib%sx,bibrec_bib%sx WHERE id_bibxxx=id AND tag='%s' AND id_bibxxx=id and id_bibrec >=%s and id_bibrec<=%s" % (tag[0:2], tag[0:2], key, recids, recide)))
write_message("No records specified, updating all")
min_id = run_sql("SELECT min(id) from bibrec")[0][0]
max_id = run_sql("SELECT max(id) from bibrec")[0][0]
options["recid_range"] = [[min_id, max_id]]
if options["quick"] == "no" and options["verbose"] >= 9:
write_message("Recalculate parameter not used, parameter ignored.")
if options["cmd"] == "del":
del_recids(cfg_short, options["recid_range"])
elif options["cmd"] == "add":
func_object = globals().get(cfg_function)
func_object(rank_method_code, cfg_name, config)
elif options["cmd"] == "stat":
rank_method_code_statistics(rank_method_code)
elif options["cmd"] == "check":
check_method(rank_method_code)
elif options["cmd"] == "repair":
pass
else:
write_message("Invalid command found processing %s" % rank_method_code, sys.stderr)
raise StandardError
except StandardError, e:
write_message("\nException caught: %s" % e, sys.stderr)
if options["verbose"] >= 9:
traceback.print_tb(sys.exc_info()[2])
raise StandardError
if options["verbose"]:
showtime((time.time() - startCreate))
return 1
def get_valid_range(rank_method_code):
"""Return a range of records"""
if options["verbose"] >=9:
write_message("Getting records from collections enabled for rank method.")
res = run_sql("SELECT collection.name FROM collection,collection_rnkMETHOD,rnkMETHOD WHERE collection.id=id_collection and id_rnkMETHOD=rnkMETHOD.id and rnkMETHOD.name='%s'" % rank_method_code)
#By doing this like below, characters standing alone, like c a b is not added to the inedx, but when they are together with characters like c++ or c$ they are added.
for word in split(phrase):
if options["remove_stopword"] == "True" and not is_stopword(word, 1) and check_term(word, 0):
if lang and lang !="none" and options["use_stemming"]:
word = stem(word, lang)
if not words.has_key(word):
words[word] = (0,0)
words[word] = (words[word][0] + weight, 0)
elif options["remove_stopword"] == "True" and not is_stopword(word, 1):
write_message("Exception caught: %s" % e, sys.stderr)
if options["verbose"] >= 9:
traceback.print_tb(sys.exc_info()[2])
sys.exit(1)
wordTable.report_on_table_consistency()
# We are done. State it in the database, close and quit
return 1
def get_tags(config):
"""Get the tags that should be used creating the index and each tag's parameter"""
tags = []
function = config.get("rank_method","function")
i = 1
shown_error = 0
#try:
if 1:
while config.has_option(function,"tag%s"% i):
tag = config.get(function, "tag%s" % i)
tag = string.split(tag, ",")
tag[1] = int(string.strip(tag[1]))
tag[2] = string.strip(tag[2])
#check if stemmer for language is available
if config.get(function,"stemming") and stem("information", "en") != "inform":
if shown_error == 0:
write_message("Warning: PyStemmer not found. Please read INSTALL.")
shown_error = 1
elif tag[2] and tag[2] != "none" and config.get(function,"stemming") and not is_stemmer_available_for_language(tag[2]):
write_message("Warning: Language '%s' not available in PyStemmer." % tag[2])
tags.append(tag)
i += 1
#except Exception:
# write_message("Could not read data from configuration file, please check for errors")
# raise StandardError
return tags
def get_valid_range(rank_method_code):
"""Returns which records are valid for this rank method, according to which collections it is enabled for."""
#if options["verbose"] >=9:
# write_message("Getting records from collections enabled for rank method.")
#res = run_sql("SELECT collection.name FROM collection,collection_rnkMETHOD,rnkMETHOD WHERE collection.id=id_collection and id_rnkMETHOD=rnkMETHOD.id and rnkMETHOD.name='%s'" % rank_method_code)
write_message("%s/%s---%s---%s" % (terms[i][0],terms[i][1], Gi[i][0],Gi[len(Gi) - i - 1][0]))
def update_rnkWORD(table, terms):
"""Updates rnkWORDF and rnkWORDR with Gi and Nj values. For each term in rnkWORDF, a Gi value for the term is added. And for each term in each document, the Nj value for that document is added. In rnkWORDR, the Gi value for each term in each document is added. For description on how things are computed, look in the hacking docs.
table - name of forward index to update
terms - modified terms"""
stime = time.time()
Gi = {}
Nj = {}
N = run_sql("select count(id_bibrec) from %sR" % table[:-1])[0][0]
if len(terms) == 0 and options["quick"] == "yes":
write_message("No terms to process, ending...")
return ""
elif options["quick"] == "yes": #not used -R option, fast calculation (not accurate)
write_message("Beginning post-processing of %s terms" % len(terms))
#Locating all documents related to the modified/new/deleted terms, if fast update,
#only take into account new/modified occurences
write_message("Phase 1: Finding records containing modified terms")
terms = terms.keys()
i = 0
while i < len(terms):
terms_docs = get_from_forward_index(terms, i, (i+5000), table)
for (t, hitlist) in terms_docs:
term_docs = deserialize_via_marshal(hitlist)
if term_docs.has_key("Gi"):
del term_docs["Gi"]
for (j, tf) in term_docs.iteritems():
if (options["quick"] == "yes" and tf[1] == 0) or options["quick"] == "no":
Nj[j] = 0
write_message("Phase 1: ......processed %s/%s terms" % ((i+5000>len(terms) and len(terms) or (i+5000)), len(terms)))
i += 5000
write_message("Phase 1: Finished finding records containing modified terms")
#Find all terms in the records found in last phase
write_message("Phase 2: Finding all terms in affected records")
records = Nj.keys()
i = 0
while i < len(records):
docs_terms = get_from_reverse_index(records, i, (i + 5000), table)
for (j, termlist) in docs_terms:
doc_terms = deserialize_via_marshal(termlist)
for (t, tf) in doc_terms.iteritems():
Gi[t] = 0
write_message("Phase 2: ......processed %s/%s records " % ((i+5000>len(records) and len(records) or (i+5000)), len(records)))
i += 5000
write_message("Phase 2: Finished finding all terms in affected records")
else: #recalculate
max_id = run_sql("SELECT MAX(id) FROM %s" % table)
max_id = max_id[0][0]
write_message("Beginning recalculation of %s terms" % max_id)
<dt>The rank method is currently enabled for these collections:</dt>
<dd>
"""
col_list = get_rnk_col(rnkID, ln)
if not col_list:
output += """No collections"""
else:
for (id, name) in col_list:
output += """%s, """ % name
output += """</dd>
</dl>
"""
col_list = get_def_name('', "collection")
col_rnk = dict(get_rnk_col(rnkID))
col_list = filter(lambda x: not col_rnk.has_key(x[0]), col_list)
if col_list:
text = """
<span class="adminlabel">Enable for:</span>
<select name="colID" class="admin_w200">
<option value="">- select collection -</option>
"""
for (id, name) in col_list:
text += """<option value="%s" %s>%s</option>""" % (id, (func in ["0", 0] and confirm in ["0", 0] and colID and int(colID) == int(id)) and 'selected="selected"' or '' , name)
elif confirm in ["1", 1] and func in ["0", 0] and colID:
subtitle = "Step 3 - Result"
output += write_outcome(finresult)
elif confirm not in ["0", 0] and func in ["0", 0]:
output += """<b><span class="info">Please select a collection.</span></b>"""
col_list = get_rnk_col(rnkID, ln)
if col_list:
text = """
<span class="adminlabel">Disable for:</span>
<select name="colID" class="admin_w200">
<option value="">- select collection -</option>
"""
for (id, name) in col_list:
text += """<option value="%s" %s>%s</option>""" % (id, (func in ["1", 1] and confirm in ["0", 0] and colID and int(colID) == int(id)) and 'selected="selected"' or '' , name)
"""form to add a new rank method with these values:"""
subtitle = 'Step 1 - Create new rank method'
output = """
<dl>
<dt>BibRank code:</dt>
<dd>A unique code that identifies a rank method, is used when running the bibrank daemon and used to name the configuration file for the method.
<br>The template files includes the necessary parameters for the chosen rank method, and only needs to be edited with the correct tags and paths.
<br>For more information, please go to the <a title="See guide" href="%s/admin/bibrank/guide.html">BibRank guide</a> and read the section about adding a rank method</dd>
text += """<b><span class="info"><br>Configuration file created using '%s' as template.</span></b>""" % template
else:
text += """<b><span class="info"><br>Empty configuration file created.</span></b>"""
except StandardError, e:
text += """<b><span class="info"><br>Sorry, could not create configuration file: '%s/bibrank/%s.cfg', either because it already exists, or not enough rights to create file. <br>Please create the file in the path given.</span></b>""" % (etcdir, get_rnk_code(rnkID)[0][0])
else:
text = """<b><span class="info">Sorry, could not add rank method, rank method with the same BibRank code probably exists.</span></b>"""
output += text
elif not rnkcode and confirm not in [-1, "-1"]:
output += """<b><span class="info">Sorry, could not add rank method, not enough data submitted.</span></b>"""
<dd>When changing the BibRank code of a rank method, you must also change any scheduled tasks using the old value.
<br>For more information, please go to the <a title="See guide" href="%s/admin/bibrank/guide.html">BibRank guide</a> and read the section about modifying a rank method's BibRank code.</dd>
text += """<b><span class="info">Cannot load file, either it does not exist, or not enough rights to read it: '%s/bibrank/%s.cfg'<br>Please create the file in the path given.</span></b>""" % (etcdir, get_rnk_code(rnkID)[0][0])
output += createhiddenform(action="modifyrank",
text=text,
rnkID=rnkID,
button="Modify",
confirm=1)
if rnkcode and confirm in ["1", 1] and get_rnk_code(rnkID)[0][0] != rnkcode:
oldcode = get_rnk_code(rnkID)[0][0]
result = modify_rnk(rnkID, rnkcode)
subtitle = "Step 3 - Result"
if result:
text = """<b><span class="info">Rank method modified.</span></b>"""
text = """<b><span class="info"><br>Configuration file modified: '%s/bibrank/%s.cfg'</span></b>""" % (etcdir, get_rnk_code(rnkID)[0][0])
except StandardError, e:
text = """<b><span class="info"><br>Sorry, could not modify configuration file, please check for rights to do so: '%s/bibrank/%s.cfg'<br>Please modify the file manually.</span></b>""" % (etcdir, get_rnk_code(rnkID)[0][0])
<dd>When deleting a rank method, you also deletes all data related to the rank method, like translations, which collections
it was attached to and the data necessary to rank the searchresults. Any scheduled tasks using the deleted rank method will also stop working.
<br><br>For more information, please go to the <a title="See guide" href="%s/admin/bibrank/guide.html">BibRank guide</a> and read the section regarding deleting a rank method.</dd>
text += """<br><b><span class="info">Configuration file deleted: '%s/bibrank/%s.cfg'.</span></b>""" % (etcdir, rnkcode)
except StandardError, e:
text += """<br><b><span class="info">Sorry, could not delete configuration file: '%s/bibrank/%s.cfg'.</span><br>Please delete the file manually.</span></b>""" % (etcdir, rnkcode)
else:
text = """<b><span class="info">Sorry, could not delete rank method</span></b>"""
except StandardError, e:
text = """<b><span class="info">Sorry, could not delete rank method, most likely already deleted</span></b>"""
"""add or modify translations in tables given by table
frmID - the id of the format from the format table
sel_type - the name type
langs - the languages
trans - the translations, in same order as in langs
table - the table"""
name = "name"
if table[-1:].isupper():
name = "NAME"
try:
for nr in range(0,len(langs)):
res = run_sql("SELECT value FROM %s%s WHERE id_%s=%s AND type='%s' AND ln='%s'" % (table, name, table, ID, sel_type, langs[nr][0]))
if res:
if trans[nr]:
res = run_sql("UPDATE %s%s SET value='%s' WHERE id_%s=%s AND type='%s' AND ln='%s'" % (table, name, MySQLdb.escape_string(trans[nr]), table, ID, sel_type, langs[nr][0]))
else:
res = run_sql("DELETE FROM %s%s WHERE id_%s=%s AND type='%s' AND ln='%s'" % (table, name, table, ID, sel_type, langs[nr][0]))
else:
if trans[nr]:
res = run_sql("INSERT INTO %s%s(id_%s, type, ln, value) VALUES (%s,'%s','%s','%s')" % (table, name, table, ID, sel_type, langs[nr][0], MySQLdb.escape_string(trans[nr])))
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibSched - task management, scheduling and executing system for CDSware
"""
__version__ = "$Id$"
### -- local configuration section starts here ---
cfg_valid_processes = ["bibindex","bibupload","bibreformat","webcoll","bibtaskex","bibrank","oaiharvest","oaiarchive"] # which tasks are reconized as valid?
if self.auto_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,curses.KEY_PPAGE,curses.KEY_NPAGE,ord("q"),ord("Q"),ord("a"),ord("A"),ord("1"),ord("2"))):
self.display_in_footer("in automatic mode")
self.stdscr.refresh()
elif self.move_mode and (chr not in (curses.KEY_UP,curses.KEY_DOWN,ord("m"),ord("M"),ord("q"),ord("Q"))):
self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order))
self.repaint()
ring=0
while self.running:
ring += 1
chr = -1
try:
chr = timed_out(self.stdscr.getch, 1)
if chr == 27: # escaping sequence
chr = self.stdscr.getch()
if chr == 79: # arrow
chr = self.stdscr.getch()
if chr == 65: #arrow up
chr = curses.KEY_UP
elif chr == 66: #arrow down
chr = curses.KEY_DOWN
elif chr == 72:
chr = curses.KEY_PPAGE
elif chr == 70:
chr = curses.KEY_NPAGE
elif chr == 91:
chr = self.stdscr.getch()
if chr == 53:
chr = self.stdscr.getch()
if chr == 126:
chr = curses.KEY_HOME
except TimedOutExc:
chr = -1
self.handle_keys(chr)
if ring == 4:
if self.display == 1:
where = "and status='DONE'"
order = "DESC"
else:
where = "and status!='DONE'"
order = "ASC"
self.rows = run_sql("SELECT id,proc,user,runtime,sleeptime,status,progress,arguments FROM schTASK WHERE status NOT LIKE '%%DELETED%%' %s ORDER BY runtime %s" % (where,order))
ring = 0
self.repaint()
class BibSched:
def __init__(self):
self.helper_modules = cfg_valid_processes
self.running = {}
self.sleep_done = {}
self.sleep_sent ={}
self.stop_sent = {}
self.suicide_sent = {}
def set_status(self, id, status):
return run_sql("UPDATE schTASK set status=%s WHERE id=%s", (status, id))
def can_run( self, proc ):
return len( self.running.keys() ) == 0
def get_running_processes(self):
row = None
res = run_sql("SELECT id,proc,user,UNIX_TIMESTAMP(runtime),sleeptime,arguments,status FROM schTASK "\
" WHERE status='RUNNING' or status='CONTINUING' LIMIT 1")
global $maxfieldno,$index,$options,$xml,$nbTags,$nbUpdatedRecords,$nbInsertedRecords,$nbErrorRecords, $nbRecords,$currentfield,$i,$stopstate,$sleepstate,$errorfp,$validrec;
// read task options:
$task_id = $options["TASKRUN"]; // remember task id
$options = array();
$options = task_get_options($task_id); // get options from BibSched task table
$options["TASKRUN"] = $task_id;
if (task_get_state() != "WAITING") {
$progress = task_get_progress();
}
else {
$progress = 0;
}
task_update_state("RUNNING");
$starttime = getmicrotime();
$vals = array();
$nbRecords = 0;
$totalrecs = 0;
foreach ($options['recordfiles'] as $recordfile) {
//open xml file
$fp = @fopen("$recordfile","r");
if (!$fp) {
fwrite($errorfp, "Error: cannot localize and/or open file $recordfile for reading.\n");
raise elmsubmitSubmissionError("Submission does not contain the required fields for document type %s. Required fields: %s" % (doctype, elmsubmit_config.required_fields))
# Check that the fields we have been given validate OK:
# Messages we need to send to the user, before we've identified the
# correct language to talk to them in (so we assume English!):
nolangmsgs = {'bad_email': 'Your email could not be parsed correctly to discover a submission. Please check your email client is functioning correctly.',
'bad_submission': 'The submission data that you have provided could not be parsed correctly. Please visit <http://pcdh23.cern.ch> for a description of the correct format.',
'missing_type': 'The submission data that you have provided does not contain a TYPE field. This is mandatory for all submissions.',
'unsupported_type': 'The TYPE field of your submission does not contain a recognized value.',
'missing_fields_1': 'Your submission of type',
'missing_fields_2': 'does not contain all the required fields:',
'bad_field': 'This field does not validate correctly:',
'correct_format': 'It must be formatted as follows:',
'missing_attachment': 'We could not find the following file attached to your submission email:',
'temp_problem': 'There is a temporary problem with CDSware\'s email submission interface. Please retry your submission again shortly.'}
servers = {'smtp': 'localhost'}
people = {'admin': 'cds.support@cern.ch'}
# fields required in the submission mail
required_fields = ['title',
'author',
'date',
'files']
# defines the mapping of metadata fields to their marc codes
# mapping code as a list means the first element is mapped to the first element
# of the list, and the rest to the second
marc_mapping = {'author': ['100__a','700__a'],
'title': '245__a',
'subtitle': '245__b',
'photocaption': '246__b',
'subject': '65017a',
'secondary_subject': '65027a',
'email': '8560_f',
'files': ['FFT__a','FFT__a'],
'affiliation': ['100__u', '700__u'],
'language': '041__a',
'abstract': '520__a',
'keywords': '6531_a',
'OAIid': '909COo',
'PrimaryReportNumber': '037__a',
'AdditionalReportNumber': '088__a',
'series': ['490__a','490__v'],
'year': '260__a',
'note': '500__a',
#test tags used in test cases
'test1': '111__a',
'test2': '111__b',
'test3': '111__c',
'test4': '111__d',
'test5': '111__e'
}
# the list of the fields determines which subfields should be joined into a
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
parse_submission takes text like this:
--------------
cdson:::
_language_
eng
_type_
test
_title_
Blathering on About More Scientific Crap
_author_
Owen, R
_num_
69
_date_
01/01/2004
_keywords_
science forgery vitriol
_abstract_
This is possibly the best document to come out of space in a long
time. Aliens have really improved their scientific writing abilites.
Brillig things about this document:
One: I wrote it.
Two: It smells of cheese.
Three: Fnah!
_note_
Not musical, but informative.
_refnums_
AA11x-madeup_ref
_files_
info.pdf
foo.txt
cdsoff:::
Dear Sir,
Here is the rest of the email.
Sincerely,
Jonathon Bloggs.
--
Tel: 555 111 234567
IT-UDS CERN
--------------
This is turned into a 2-tuple. The first entry in the tuple is a
Python dictionary containing the submission info. The second entry is
the trailing text that follows the submission (the submission MUST be
at the top of the text).
({'abstract': 'This is possibly the best document to come out of space in a long\ntime. Aliens have really improved their scientific writing abilites.\n\nBrillig things about this document:\nOne: I wrote it.\nTwo: It smells of cheese.\nThree: Fnah!',
'author': 'Owen, R',
'date': '01/01/2004',
'files': 'info.pdf\nfoo.txt',
'keywords': 'science forgery vitriol',
'language': 'eng',
'note': 'Not musical, but informative.',
'num': '69',
'refnums': 'AA11x-madeup_ref',
'title': 'Blathering on About More Scientific Crap',
'type': 'test'},
'\nDear Sir,\n\nHere is the rest of the email.\n\nSincerely, \nJonathon Bloggs.\n\n--\nTel: 555 111 234567\nIT-UDS CERN')
It is fairly robust when treating misformatted submissions, so should
hopefully protect against misformatting due to evil smtp servers /
conversion from HTML email producing clients, etc. For example, we can
process the following OK:
----------------
cdson:::
_language_
eng
_type_
test
_title_
Blathering on About More Scientific Crap
__author__
Owen, R
_num_
69
_date_
01/01/2004
_keywords_
science forgery vitriol
cdsoff:::
---------------------
"""
# text values determining begin and end of submission
@param param: tuple of string params to insert in the query
(see notes below)
@param n: number of tuples in result (0 for unbounded)
@param with_desc: if true, will return a
DB API 7-tuple describing columns in query
@return: if SELECT, SHOW, DESCRIBE statements: tuples of data, followed
by description if parameter
provided
if INSERT: last row id.
else: SQL result as provided by database
</pre>
<h2>3. <a name="notes">Notes</a></h2>
<h3>3.1 Params</h3>
As said before, params must be of type string. Trying to pass non character data
may lead to an error. Developers should consider converting to string, or render
string before sending, giving no param:
<pre>
initial_params=(1234,)
query = "SELECT id FROM user where id>%s"
params = (str(initial_params[0]),)
run_sql(query, params)
</pre>
or
<pre>
query = "SELECT id FROM user where id>%d" % 1234
run_sql(query)
</pre>
<p>When using this last technique, be careful with SQL injection problem. One should
use the <code>MySQLdb.escape_string()</code> method.
<h3>3.2 Dates</h3>
<p>Switching from MySQLdb 0.9 to MySQLdb 1.2, while using Python 2.2 or 2.3
led to discovery of incompatibilities</p>
<p>If a date field is in the received tuple, its format will be:</p>
<ul>
<li>string with MySQLdb 0.9</li>
<li>datetime with MySQLdb 1.2</li>
</ul>
<p>As Python 2.2 doesn't provide <code>datetime</code> class, handling of this
problem should be done for backwards compatibility reasons. The
solution is to force MySQL to convert date to a textual format:</p>
<pre>
SELECT DATE_FORMAT(date_field,'%%Y-%%m-%%d %%H:%%i:%%s') FROM table
</pre>
<p>This conversion will return a datetext format as described in <a href="./dateutils.html">dateutils library</a><code>(YEAR-MONTH-DAY HOUR:MINUTE:SECOND)</code></p>
INSERT INTO flxFORMATS VALUES ('_DEFAULT_TITLE','$245.a \r\nif($245.b!=\"\"){ \" : \" $245.b } \r\nif($250.a!=\"\"){ \" ; \" $250.a } ','HTML for displaying the title in brief formats','O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"245\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"A\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:4;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:8;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"245\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"B\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:3:\" : \";s:4:\"sons\";a:0:{}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"245\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"B\";s:4:\"sons\";a:0:{}}}}}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:4;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:8;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"250\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"A\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:3:\" ; \";s:4:\"sons\";a:0:{}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"250\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"A\";s:4:\"sons\";a:0:{}}}}}}}}}}');
INSERT INTO flxFORMATS VALUES ('_DEFAULT_ABSTRACT_FIRST_SENTENCE','limw_l($520.a, \".\") \". [...]\"','This is the default subformat to yield the first sentence of the abstract.','O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:2;s:3:\"lex\";s:6:\"LIMW_L\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:3:\"520\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"A\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:1:\".\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:7:\". [...]\";s:4:\"sons\";a:0:{}}}}');
INSERT INTO flxFORMATS VALUES ('DEFAULT_HTML_DETAILED','format(\"_full_topbanner\") \r\nformat(\"_full_title\") \r\n\"<p><center>\" \r\nformat(\"_full_author\")\r\nformat(\"_full_affiliation\")\r\nformat(\"_full_datedoc\") \"<br>\"\r\nformat(\"_full_imprint\")\r\n\"</center>\"\r\n\r\n\"<p style=\\\"margin-left: 15%; width: 70%\\\">\"\r\n\r\nformat(\"_full_abstract\")\r\nformat(\"_full_daterec\")\r\n\r\nformat(\"_full_keyword\")\r\n\r\nformat(\"_full_note\") \r\nformat(\"_full_publiinfo\")\r\n\"<br>\"\r\nformat(\"_full_url\")\r\nformat(\"_full_citedby\")\r\n\"<p>\" \r\nformat(\"_full_references\")\r\n','This is the default HTML detailed format.','O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:15:\"_full_topbanner\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:11:\"_full_title\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:11:\"<p><center>\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:12:\"_full_author\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:17:\"_full_affiliation\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:13:\"_full_datedoc\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:4:\"<br>\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:13:\"_full_imprint\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:9:\"</center>\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:40:\"<p style=\"margin-left: 15%; width: 70%\">\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:14:\"_full_abstract\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:13:\"_full_daterec\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:13:\"_full_keyword\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:10:\"_full_note\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:15:\"_full_publiinfo\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:4:\"<br>\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:9:\"_full_url\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:13:\"_full_citedby\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:3:\"<p>\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:13;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:16:\"_full_references\";s:4:\"sons\";a:0:{}}}}}}');
INSERT INTO flxFORMATS VALUES ('_DEFAULT_URL','\"<a class=\\\"note\\\" href=\\\"\" $8564.u \"\\\">\" $8564.u \"</a>\"','This is the default format for formatting URLs.','O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:11;s:3:\"lex\";s:0:\"\";s:4:\"sons\";a:2:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:22:\"<a class=\"note\" href=\"\";s:4:\"sons\";a:0:{}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:4:\"8564\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"U\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:2:\"\">\";s:4:\"sons\";a:0:{}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:0;s:3:\"lex\";s:4:\"8564\";s:4:\"sons\";a:1:{i:0;O:7:\"aelnode\":3:{s:3:\"cat\";i:12;s:3:\"lex\";s:1:\"U\";s:4:\"sons\";a:0:{}}}}}}i:1;O:7:\"aelnode\":3:{s:3:\"cat\";i:1;s:3:\"lex\";s:4:\"</a>\";s:4:\"sons\";a:0:{}}}}');
INSERT INTO flxBEHAVIORCONDITIONSACTIONS VALUES ('HX',0,0,'','format(\"_FULL_BIBTEX\")');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('DEFAULT',100,'\"\"=\"\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HB',0,'$980.a=\"PICTURE\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HB',100,'\"\"=\"\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HP',0,'\"\"=\"\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HC',0,'\"\"=\"\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HD',100,'\"\"=\"\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HD',90,'$980.a=\"PICTURE\"');
INSERT INTO flxBEHAVIORCONDITIONS VALUES ('HX',0,'\"\"=\"\"');
INSERT INTO flxBEHAVIORS VALUES ('DEFAULT','IENRICH','Creates <b>DEFAULT</b> formats and includes them in the input XML OAI MARC record in the \"FMT\" element');
INSERT INTO flxBEHAVIORS VALUES ('HB','NORMAL','Produces HTML brief format. Useful for reformatting records existing in the database.');
INSERT INTO flxBEHAVIORS VALUES ('HP','NORMAL','PRODUCES HTML PORTFOLIO FORMAT. USEFUL FOR REFORMATTING RECORDS EXISTING IN THE DATABASE.');
INSERT INTO flxBEHAVIORS VALUES ('HC','NORMAL','PRODUCES HTML CAPTIONS FORMAT. USEFUL FOR REFORMATTING RECORDS\r\nEXISTING IN THE DATABASE.');
INSERT INTO flxBEHAVIORS VALUES ('HD','NORMAL','PRODUCES HTML DETAILED FORMAT. USEFUL FOR REFORMATTING RECORDS EXISTING IN THE DATABASE.');
INSERT INTO flxBEHAVIORS VALUES ('HX','NORMAL','PRODUCES HTML BibTeX format.');
INSERT INTO flxUDFS VALUES ('URLENCODE','return urlencode($url);','STRING',NULL);
INSERT INTO flxUDFS VALUES ('UPPER','return strtoupper($s);','STRING','<b>Purpose</b>: Returns characters of string <i>s</i> into uppercase characters\r\n<b>Example</b>: upper(\"test12\") gives as result \"TEST12\"');
INSERT INTO flxUDFS VALUES ('SUBSTR','return substr($s, $start, $end-$start+1);','STRING','<b>Purpose</b>: Returns substring between position <i>start</i> and <i>end</i> from the string <i>s</i>\r\n<b>Example</b>: substr(\"test\", 2, 3) gives as result \"st\"');
INSERT INTO flxUDFS VALUES ('ADD','if($prefix!=\"\")\r\n $value=$prefix.$value;\r\nif($postfix!=\"\")\r\n $value=$value.$postfix;\r\nreturn $value;\r\n','STRING','<b>Purpose</b>: This function adds <i>prefix</i> at the begining and <i>postfix</i> at the end, of <i>value</i>\r\n<b>Example</b>: add(\"test\",\"pre\",\"post\") gives as result \"pretestpost\"\r\n<b>Note</b>: Defined to keep compatibility with uploader\'s languaje');
INSERT INTO flxUDFS VALUES ('MINFO','return \"<a href=\\\"http://weblib.cern.ch/cgi-bin/showfull?id=$sysno\\\">More Info</a>\";','STRING','');
INSERT INTO flxUDFS VALUES ('RNETC','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('EXT','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('MARK','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('SEPARATOR','if(!$LAST_ITERATION)\r\n return $str;\r\nreturn \"\";','STRING',NULL);
INSERT INTO flxUDFS VALUES ('TIONE','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('BRBER','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('MINFA','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('INDIS','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('INHLD','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('LKR','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('NIUCO','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('HOL','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('SNLNK','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('AUFU','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('LKRFU','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('SP','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('HO','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('SREF','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('QUOT','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('MARKF','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('LATEX','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('DI','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('REP_PREFIX','if($FIRST_ITERATION) return $str; return \"\";','STRING',NULL);
INSERT INTO flxUDFS VALUES ('XML_TEXT','return htmlspecialchars($str,ENT_QUOTES); \r\n\r\n','STRING','');
INSERT INTO flxUDFS VALUES ('REP','return str_replace($search, $replacement, $str); \r\n\r\n','STRING','<b>Purpose</b>: Replaces any occurence of string <i>search</i> inside string <i>str</i> by the \r\n<i>replacement</i> one.\r\n<b>Example</b>: rep(\"test12test\", \"12\", \"-\") gives as result \"test-test\"');
INSERT INTO flxUDFS VALUES ('FORMAT_EXIST','$format_name=strtoupper(trim($format_name)); \r\n$res=\"\"; \r\n$db=mysql_pconnect(\"cdsdb.cern.ch\", \"flexelink\", \"2flexible\"); \r\nmysql_selectdb(\"flexelink\"); \r\n$qry=\"select name from FORMATS where name=\'$format_name\'\"; \r\n$qh=mysql_query($qry); \r\nif(mysql_num_rows($qh)==1) \r\n $res=\"Y\"; \r\nreturn $res; ','STRING','<b>Purpose</b>: Tells if a format exists in the FlexElink formats database. Returns non-empty string when a format called <i>format_name</i> doesn\'t exist in the format db, empty string in\r\n other case\r\n<b>Note</b>: To be migrated to FlexElink EL internal fuction');
INSERT INTO flxUDFS VALUES ('TOC','return \"\";','STRING','Temporary defined for \"automatic format migration\" compatibility. Gives empty string.');
INSERT INTO flxUDFS VALUES ('DATE','return date(\"j M Y H:i:s\");','STRING','<b>Purpose</b>: Gives the current date in the following format \"day month_name year2k hh:mm:ss\"');
INSERT INTO flxUDFS VALUES ('FILTER_PAGENR','$temp=explode(\"-\", $str);\r\nreturn $temp[0];','STRING','');
INSERT INTO flxUDFS VALUES ('LOWER','return strtolower($str);','STRING','<b>Purpose</b>: Returns characters of string <i>s</i> into lowercase characters \r\n<b>Example</b>: lower(\"TEsT12\") gives as result \"test12\"');
INSERT INTO flxUDFS VALUES ('SUBSTR_END','return substr($str, $start);','STRING','<b>Purpose</b>: Returns a substring from the string <i>s</i>, from position <i>start</i> to the end of the string. \r\nNote that characters into a string are always numerated starting in 0\r\n<b>Example</b>: substr(\"test\", 2) gives as result \"st\"');
INSERT INTO flxUDFS VALUES ('LIMW_L','$temp=explode($token, $str, 2);\r\nreturn $temp[0];','STRING','<b>Purpose</b>: Returns a substring from the string <i>str</i>, containing all characters found in it\r\nbefore the first occurrence of string <i>token</i>.\r\n<b>Example</b>: substr_bef_token(\"test1@test2@test3\", \"@\") gives as result \"test1\"');
INSERT INTO flxUDFS VALUES ('LIMW_R','$temp=explode($token, $str, 2); \r\nreturn $temp[1];','STRING','<b>Purpose</b>: Returns a substring from the string <i>str</i>, containing all characters found in it\r\nafter the first acourrence of string <i>token</i>.\r\n<b>Example</b>: substr_bef_token(\"test1@test2@test3\", \"@\") gives as result \"test2@test3\"');
INSERT INTO flxUDFS VALUES ('FILE_EXISTS','$fh=@fopen($url, \"r\");\r\n$res=\"\";\r\nif($fh)\r\n $res=$url;\r\nreturn $res;','STRING','<b>Purpose</b>: Checks if the given <i>url</i> exists. If so it returns the <i>url</i>, if not it gives back an empty string.\r\n<b>Example</b>: file_exists(\"/not/exists\") gives \"\" as far as file \"/not/exists\" doesn\'t exist');
INSERT INTO flxUDFS VALUES ('CUT_FILE','$fh=@fopen($url, \"r\");\r\n$res=\"\";\r\nif($fh)\r\n{\r\n $found=false;\r\n if($start_token==\"\")\r\n {\r\n $found=true;\r\n }\r\n while(!feof($fh))\r\n {\r\n if(!$found)\r\n {\r\n $line=fgets($fh, 4096);\r\n $pos=strpos($line, $start_token);\r\n if($pos === false)\r\n continue;\r\n $res=substr($line, $pos, strlen($start_tag));\r\n $line=substr($line, $pos+strlen($start_tag));\r\n $found=true;\r\n }\r\n else\r\n {\r\n $pos=strpos($line, $end_token);\r\n if($pos === false)\r\n {\r\n $res.=$line;\r\n $line=fgets($fh, 4096);\r\n }\r\n else\r\n {\r\n $res.=substr($line, 0, $pos+strlen($end_token));\r\n break;\r\n }\r\n }\r\n }\r\n fclose($fh);\r\n}\r\n\r\nreturn $res;','STRING','<b>Purpose</b>: This function opens the file pointed by the URL <i>url</i>. If it\'s accessible, it parses it and \r\nreturns all characters between first occurence of string <i>start_token</i> and first occurrence of \r\nstring <i>end_token</i>.\r\n<b>Example</b>: cut_file(\"/local/pp.txt\", \"PEPE\", \"JOSE\") and \"/local/pp.txt\" file contains the\r\n following text \r\n\"In spanish PEPE means JOSE and PACO means FRANCISCO\", \r\nwould give as result \"PEPE means JOSE\"');
INSERT INTO flxUDFS VALUES ('FILTER_CDS_ID','$temp=explode(\"-\", $str, 2); \r\nreturn $temp[0];','STRING','');
INSERT INTO flxUDFS VALUES ('COPY','return substr($str, $start, $length);','STRING','<b>Purpose</b>: Returns a substring from string <i>str</i>, that starts in position <i>start</i>\r\n and that is <i>length</i> characters long from there\r\n<b>Example</b>: copy(\"test\", \"1\", \"2\") gives as result \"es\"');
INSERT INTO flxUDFS VALUES ('TERMINATOR','if($LAST_ITERATION)\r\n return $str;','STRING','');
INSERT INTO flxUDFS VALUES ('SUM','return $v1+$v2;','STRING','<b>Purpose</b>: Gives the result of adding <i>v1</i> to <i>v2</i> (<i>v1</i>+<i>v2</i>)\r\n<b>Example</b>: add(\"3\", \"4\") gives as result \"7\"');
INSERT INTO flxUDFS VALUES ('GT','if($v1>$v2)\r\n return \"1\";\r\nreturn \"\";','STRING','<b>Purpose</b>: \"Greater than\" function; returns \"1\" if <i>v1</i>><i>v2</i>, \"\" in other case\r\n<b>Example</b>: gt(\"3\", \"4\") gives as result \"\"');
INSERT INTO flxUDFS VALUES ('EXTRACT_EMAIL','if(ereg(\"([^ <\\t\\n]+@[^ >\\t\\n]+)\", $str, $res))\r\n{\r\n return $res[1];\r\n}\r\nelse\r\n return \"\";\r\n','STRING','<b>Purpose</b>: Returns the first valid (filtered) email address found in string <i>str</i>. \r\nIf there\'s no valid email address in it, it returns \"\"\r\n<b>Example</b>: extract_email(\"Hector Sanchez <Hector.Sanchez@cern.ch>\") gives \"Hector.Sanchez@cern.ch\".');
INSERT INTO flxUDFS VALUES ('GE','if($v1>=$v2)\r\n return \"1\";\r\nreturn \"\";','STRING','<b>Purpose</b>: \"Greater or equal than\" function; returns \"1\" if <i>v1</i>>=<i>v2</i>, \"\" in other case\r\n<b>Example</b>: ge(\"3\", \"4\") gives as result \"\"');
INSERT INTO flxUDFS VALUES ('LE','if($v1<=$v2)\r\n return \"1\";\r\nreturn \"\";','STRING','<b>Purpose</b>: \"Lower or equal than\" function; returns \"1\" if <i>v1</i><=<i>v2</i>, \"\" in other case\r\n<b>Example</b>: ge(\"3\", \"4\") gives as result \"\"');
INSERT INTO flxUDFS VALUES ('ISO_NUM','$iso=substr($str, strlen($str_iso));\r\n$temp=explode(\"-\", $iso, 2);\r\n$isonum=$temp[0]; \r\nreturn $isonum;\r\n','STRING','');
INSERT INTO flxUDFS VALUES ('STRIP_SPACES','return str_replace(\" \", \"\", $str);','STRING','');
INSERT INTO flxUDFS VALUES ('LFILL','if(strlen($char)==0) return $str;\r\nif(strlen($str)>$digits) \r\n $str=substr($str, strlen($str)-$digits);\r\nelse\r\n $str=sprintf(\"%0\".$digits.\"u\", $str);\r\nreturn $str;','STRING','<b>Purpose</b>: Returns a string consisting in the string <i>str</i> with the character <i>char</i> \r\nadded to its left the numer of times needed until the lenght <i>digits</i> is reached. If the length\r\nof <i>str</i> is bigger than <i>digits</i>, a left truncated string to length <i>digit</i> is returned.\r\n<b>Example</b>: lfill(\"12\", \"0\", \"5\") gives as result \"00012\", while lfill(\"12345\", \"0\", \"3\") gives as result \"345\", while');
INSERT INTO flxUDFS VALUES ('STR_BUFFER','return add_text($action, $str_to_add, $remove_end_before_print, $min_length_for_print);\r\n\r\nfunction add_text($action, $str_to_add, $remove_end_before_print, $min_length_for_print) {\r\n static $whole_str_buffer = \'\';\r\n\r\n if($action === \'print\') {\r\n if (strlen($whole_str_buffer) > $min_length_for_print){ \r\n $retrn_val = substr($whole_str_buffer, 0, strlen($whole_str_buffer) - $remove_end_before_print) . $str_to_add;\r\n $whole_str_buffer = \'\';\r\n \r\n return $retrn_val;\r\n }\r\n $whole_str_buffer = \'\';\r\n return \'\';\r\n }elseif($action === \'add\'){\r\n $whole_str_buffer .= $str_to_add;\r\n return \'\';\r\n }elseif($action ===\'reset\'){\r\n $whole_str_buffer = \'\';\r\n return \'\';\r\n }else{\r\n return \'\';\r\n }\r\n}','','<b>Purpose</b>:Concatenates strings into static buffer without printing it out before you say so.\r\n<b>Parameters</b>:\r\n<i>action</i>: If action is \"add\", then it adds the string to the buffer, if the action is \"print\", then it first removes the end of the string, then adds str_to_add, then prints out. If action is \"reset\", then it resets the buffer to \"\". When you do a \"print\", it also resets the buffer.\r\n<i>str_to_add</i>: The string you want to add to the already stored string. If action is \"print\" then this part will be added after the function has taken away the end of the string.\r\n<i>remove_end_before_print</i>: Only works when action is \"print\", then it removes the number of chars you have specified before adding the rest of the string, and prints out.\r\n<i>min_length_for_print</i>: If string is shorter then this when action is \"print\", then it will return \"\". This parameter is only in use when action is \"print\".\r\n<b>Example</b>:\r\nSTR_BUFFER(\"add\", \"\\\"Ãyvind and \", \"0\", \"0\");\r\nSTR_BUFFER(\"print\", \"\\\"\", \"5\", \"3\");\r\noutputs: \"Ãyvind\"');
INSERT INTO flxUDFS VALUES ('GET_NAME','$pos=strpos($names, \',\');\r\n\r\nif($pos === false){\r\n $name_ar = preg_split(\"/[\\s+]+/\", $names);\r\n $names = \"\";\r\n\r\n for($i = 0; $i < count($name_ar); $i++){\r\n if(strlen($name_ar[$i]) >= strlen($names)){\r\n $names = $name_ar[$i];\r\n }\r\n }\r\n return $names;\r\n}\r\n\r\n$newname = substr($names, 0, $pos);\r\n\r\nreturn str_replace(\" \", \"\", $newname);','','<b>Purpose</b>: Tryes to find the last name in a string, and returns it.\r\n<b>Example</b>: GET_NAME(\"Ãstlund, Ãyvind B\"); returns \"Ãstlund\".');
INSERT INTO flxUDFS VALUES ('SPLIT_LINE','$start_index = 0;\r\n$result_string;\r\n$done = false;\r\n$firstline = true;\r\n\r\nwhile (!$done){\r\n\r\n if($start_index + $str_len >= strlen($split_str)){\r\n $done = true;\r\n $split = strlen($split_str) - $start_index;\r\n $result_string .= substr($split_str, $start_index);\r\n }else{\r\n $found = false;\r\n\r\n if($firstline == true){\r\n $split = $str_len + $indentation;\r\n $firstline = false;\r\n }else{\r\n $split = $str_len;\r\n }\r\n\r\n while ($split > 0 && $found == false){\r\n if($split_str[$start_index + $split] == \' \'){\r\n $found = true;\r\n }else{\r\n $split --;\r\n }\r\n }\r\n\r\n $result_string .= substr($split_str, $start_index, $split) . \'<br>\' . str_repeat(\' \', $indentation);\r\n $start_index += $split + 1;\r\n }\r\n}\r\n\r\n\r\nreturn $result_string;\r\n\r\n\r\n\r\n','','<b>Purpose</b>: Splits a line after a certain amount of letters. Does not split a word in two, but finds the nearest word break. \r\nNotice: Indentation is only for line 2->n, not line 1.\r\nNotice: First line will be the length of str_len + indentation.\r\nNotice: If there is HTML tags in the string, they will be counted too.\r\n<b>Example</b>: SPLIT_LINE(\"title = Nuclear matter with off-shell propagation\", \"35\", \"15\");\r\nReturns:<pre>title = Nuclear matter with\r\n off-shell propagation</pre>');
INSERT INTO flxUDFS VALUES ('GET_YEAR','if(ereg(\'[[:digit:]][[:digit:]][[:digit:]][[:digit:]]\', $date, $res)){\r\n return $res[0];\r\n}elseif(ereg(\'[[:digit:]][[:digit:]][[:digit:]][[:digit:]]\', $date, $res)){\r\n return $res[0];\r\n}','','<b>Purpose</b>: Takes a string as input, and returns a 4 digit year if the string contains it.');
INSERT INTO flxUDFPARAMS VALUES ('URLENCODE','url',0);
INSERT INTO flxUDFPARAMS VALUES ('ADD','postfix',2);
INSERT INTO flxUDFPARAMS VALUES ('SUBSTR','end',2);
INSERT INTO flxUDFPARAMS VALUES ('UPPER','s',0);
INSERT INTO flxUDFPARAMS VALUES ('SUBSTR','s',0);
INSERT INTO flxUDFPARAMS VALUES ('SUBSTR','start',1);
INSERT INTO flxUDFPARAMS VALUES ('MINFO','sysno',0);
INSERT INTO flxUDFPARAMS VALUES ('SEPARATOR','str',0);
INSERT INTO flxUDFPARAMS VALUES ('REP_PREFIX','str',0);
INSERT INTO flxUDFPARAMS VALUES ('XML_TEXT','str',0);
INSERT INTO flxUDFPARAMS VALUES ('REP','search',1);
INSERT INTO flxUDFPARAMS VALUES ('REP','str',0);
INSERT INTO flxUDFPARAMS VALUES ('SAVETOFILE','filename',0);
INSERT INTO flxUDFPARAMS VALUES ('SAVETOFILE','code',1);
INSERT INTO flxUDFPARAMS VALUES ('REP','replacement',2);
INSERT INTO flxUDFPARAMS VALUES ('FORMAT_EXIST','format_name',0);
INSERT INTO flxUDFPARAMS VALUES ('FILTER_PAGENR','str',0);
INSERT INTO flxUDFPARAMS VALUES ('LOWER','str',0);
INSERT INTO flxUDFPARAMS VALUES ('ADD','value',0);
INSERT INTO flxUDFPARAMS VALUES ('ADD','prefix',1);
INSERT INTO flxUDFPARAMS VALUES ('STRPOS','substr',1);
INSERT INTO flxUDFPARAMS VALUES ('STRPOS','str',0);
INSERT INTO flxUDFPARAMS VALUES ('CUT_FILE','end_token',2);
INSERT INTO flxUDFPARAMS VALUES ('CUT_FILE','start_token',1);
INSERT INTO flxUDFPARAMS VALUES ('CUT_FILE','url',0);
INSERT INTO flxUDFPARAMS VALUES ('FILE_EXISTS','url',0);
INSERT INTO flxUDFPARAMS VALUES ('SUBSTR_END','str',0);
INSERT INTO flxUDFPARAMS VALUES ('SUBSTR_END','start',1);
INSERT INTO flxUDFPARAMS VALUES ('LIMW_R','str',0);
INSERT INTO flxUDFPARAMS VALUES ('LIMW_L','str',0);
INSERT INTO flxUDFPARAMS VALUES ('LIMW_L','token',1);
INSERT INTO flxUDFPARAMS VALUES ('LIMW_R','token',1);
INSERT INTO flxUDFPARAMS VALUES ('GET_FROM_SETLINKURL','url',0);
INSERT INTO flxUDFPARAMS VALUES ('GET_FROM_SETLINKURL','field',1);
INSERT INTO flxUDFPARAMS VALUES ('FILTER_CDS_ID','str',0);
INSERT INTO flxUDFPARAMS VALUES ('COPY','str',0);
INSERT INTO flxUDFPARAMS VALUES ('COPY','start',1);
INSERT INTO flxUDFPARAMS VALUES ('TERMINATOR','str',0);
INSERT INTO flxUDFPARAMS VALUES ('SUM','v1',0);
INSERT INTO flxUDFPARAMS VALUES ('GT','v1',0);
INSERT INTO flxUDFPARAMS VALUES ('SUM','v2',1);
INSERT INTO flxUDFPARAMS VALUES ('EXTRACT_EMAIL','str',0);
INSERT INTO flxUDFPARAMS VALUES ('GT','v2',1);
INSERT INTO flxUDFPARAMS VALUES ('GE','v2',1);
INSERT INTO flxUDFPARAMS VALUES ('GE','v1',0);
INSERT INTO flxUDFPARAMS VALUES ('LE','v1',0);
INSERT INTO flxUDFPARAMS VALUES ('LE','v2',1);
INSERT INTO flxUDFPARAMS VALUES ('ISO_NUM','str_iso',0);
INSERT INTO flxUDFPARAMS VALUES ('ISO_NUM','str',1);
INSERT INTO flxUDFPARAMS VALUES ('ISO_PART','str_iso',0);
INSERT INTO flxUDFPARAMS VALUES ('ISO_PART','str',1);
INSERT INTO flxUDFPARAMS VALUES ('ISO_NUMPART_LINK','str_iso',0);
INSERT INTO flxUDFPARAMS VALUES ('ISO_NUMPART_LINK','str',1);
INSERT INTO flxUDFPARAMS VALUES ('STRIP_SPACES','str',0);
INSERT INTO flxUDFPARAMS VALUES ('LFILL','digits',2);
INSERT INTO flxUDFPARAMS VALUES ('LFILL','char',1);
INSERT INTO flxUDFPARAMS VALUES ('LFILL','str',0);
INSERT INTO flxUDFPARAMS VALUES ('COPY','length',2);
INSERT INTO flxUDFPARAMS VALUES ('STR_BUFFER','action',0);
INSERT INTO flxUDFPARAMS VALUES ('STR_BUFFER','str_to_add',1);
INSERT INTO flxUDFPARAMS VALUES ('STR_BUFFER','min_length_for_print',3);
INSERT INTO flxUDFPARAMS VALUES ('STR_BUFFER','remove_end_before_print',2);
INSERT INTO flxUDFPARAMS VALUES ('GET_YEAR','date',0);
INSERT INTO flxUDFPARAMS VALUES ('GET_MONTH','date',0);
INSERT INTO flxUDFPARAMS VALUES ('GET_NAME','names',0);
INSERT INTO flxUDFPARAMS VALUES ('SPLIT_LINE','indentation',2);
INSERT INTO flxUDFPARAMS VALUES ('SPLIT_LINE','split_str',0);
INSERT INTO flxUDFPARAMS VALUES ('SPLIT_LINE','str_len',1);
INSERT INTO flxUSERS VALUES (1);
INSERT INTO sbmACTION VALUES ('Submit New Record','SBI','running','1998-08-17','2001-08-08','','Submit New Record');
INSERT INTO sbmACTION VALUES ('Modify Record','MBI','modify','1998-08-17','2001-11-07','','Modify Record');
INSERT INTO sbmACTION VALUES ('Submit New File','SRV','revise','0000-00-00','2001-11-07','','Submit New File');
INSERT INTO sbmACTION VALUES ('Approve Record','APP','approve','2001-11-08','2002-06-11','','Approve Record');
INSERT INTO sbmALLFUNCDESCR VALUES ('CaseEDS','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Create_Modify_Interface',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Create_Recid',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Finish_Submission','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Get_Info','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Get_Report_Number',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Get_Sysno',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Insert_Modify_Record','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Insert_Record',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Is_Original_Submitter','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Is_Referee','This function checks whether the logged user is a referee for the current document');
INSERT INTO sbmALLFUNCDESCR VALUES ('Mail_Submitter',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Make_Modify_Record',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Make_Record','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Move_From_Pending','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Move_to_Done',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Move_to_Pending',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Print_Success','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Print_Success_APP','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Print_Success_MBI',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Print_Success_SRV',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Report_Number_Generation',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Send_Approval_Request',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Send_APP_Mail','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Send_Modify_Mail',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Send_SRV_Mail',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Test_Status','');
INSERT INTO sbmALLFUNCDESCR VALUES ('Update_Approval_DB',NULL);
INSERT INTO sbmALLFUNCDESCR VALUES ('Upload_Files','');
INSERT INTO sbmCHECKS VALUES ('AUCheck','function AUCheck(txt) {\r\n var res=1;\r\n tmp=txt.indexOf(\"\\015\");\r\n while (tmp != -1) {\r\n left=txt.substring(0,tmp);\r\n right=txt.substring(tmp+2,txt.length);\r\n txt=left + \"\\012\" + right;\r\n tmp=txt.indexOf(\"\\015\");\r\n }\r\n tmp=txt.indexOf(\"\\012\");\r\n if (tmp==-1){\r\n line=txt;\r\n txt=\'\';}\r\n else{\r\n line=txt.substring(0,tmp);\r\n txt=txt.substring(tmp+1,txt.length);}\r\n while (line != \"\"){\r\n coma=line.indexOf(\",\");\r\n left=line.substring(0,coma);\r\n right=line.substring(coma+1,line.length);\r\n coma2=right.indexOf(\",\");\r\n space=right.indexOf(\" \");\r\n if ((coma==-1)||(left==\"\")||(right==\"\")||(space!=0)||(coma2!=-1)){\r\n res=0;\r\n error_log=line;\r\n }\r\n tmp=txt.indexOf(\"\\012\");\r\n if (tmp==-1){\r\n line=txt;\r\n txt=\'\';}\r\n else{\r\n line=txt.substring(0,tmp-1);\r\n txt=txt.substring(tmp+1,txt.length);}\r\n }\r\n if (res == 0){\r\n alert(\"This author name cannot be managed \\: \\012\\012\" + error_log + \" \\012\\012It is not in the required format!\\012Put one author per line and a comma (,) between the name and the firstname initial letters. \\012The name is going first, followed by the firstname initial letters.\\012Don\'t forget the whitespace after the comma!!!\\012\\012Example \\: Put\\012\\012Le Meur, J Y \\012Baron, T \\012\\012for\\012\\012Le Meur Jean-Yves & Baron Thomas.\");\r\n return 0;\r\n } \r\n return 1; \r\n}','1998-08-18','0000-00-00','','');
INSERT INTO sbmCHECKS VALUES ('DatCheckNew','function DatCheckNew(txt) {\r\n var res=1;\r\n if (txt.length != 10){res=0;}\r\n if (txt.indexOf(\"/\") != 2){res=0;}\r\n if (txt.lastIndexOf(\"/\") != 5){res=0;}\r\n tmp=parseInt(txt.substring(0,2),10);\r\n if ((tmp > 31)||(tmp < 1)||(isNaN(tmp))){res=0;}\r\n tmp=parseInt(txt.substring(3,5),10);\r\n if ((tmp > 12)||(tmp < 1)||(isNaN(tmp))){res=0;}\r\n tmp=parseInt(txt.substring(6,10),10);\r\n if ((tmp < 1)||(isNaN(tmp))){res=0;}\r\n if (txt.length == 0){res=1;}\r\n if (res == 0){\r\n alert(\"Please enter a correct Date \\012Format: dd/mm/yyyy\");\r\n return 0;\r\n }\r\n return 1; \r\n}','0000-00-00','0000-00-00','','');
INSERT INTO sbmFORMATEXTENSION VALUES ('WORD','.doc');
INSERT INTO sbmFORMATEXTENSION VALUES ('PostScript','.ps');
INSERT INTO sbmFORMATEXTENSION VALUES ('PDF','.pdf');
INSERT INTO sbmFORMATEXTENSION VALUES ('JPEG','.jpg');
INSERT INTO sbmFORMATEXTENSION VALUES ('JPEG','.jpeg');
INSERT INTO sbmFORMATEXTENSION VALUES ('GIF','.gif');
INSERT INTO sbmFORMATEXTENSION VALUES ('PPT','.ppt');
INSERT INTO sbmFORMATEXTENSION VALUES ('HTML','.htm');
INSERT INTO sbmFORMATEXTENSION VALUES ('HTML','.html');
INSERT INTO sbmFORMATEXTENSION VALUES ('Latex','.tex');
INSERT INTO sbmFORMATEXTENSION VALUES ('Compressed PostScript','.ps.gz');
INSERT INTO sbmFORMATEXTENSION VALUES ('Tarred Tex (.tar)','.tar');
INSERT INTO sbmFORMATEXTENSION VALUES ('Text','.txt');
INSERT INTO sbmFUNDESC VALUES ('Get_Report_Number','edsrn');
INSERT INTO sbmFUNDESC VALUES ('Send_Modify_Mail','addressesMBI');
INSERT INTO sbmFUNDESC VALUES ('Send_Modify_Mail','sourceDoc');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','edsrn');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','autorngen');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','rnin');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','counterpath');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','rnformat');
INSERT INTO sbmFUNDESC VALUES ('Report_Number_Generation','yeargen');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','authorfile');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','status');
INSERT INTO sbmFUNDESC VALUES ('Send_Approval_Request','authorfile');
INSERT INTO sbmFUNDESC VALUES ('Create_Modify_Interface','fieldnameMBI');
INSERT INTO sbmFUNDESC VALUES ('Send_Modify_Mail','fieldnameMBI');
INSERT INTO sbmFUNDESC VALUES ('Update_Approval_DB','categformatDAM');
INSERT INTO sbmFUNDESC VALUES ('Send_SRV_Mail','categformatDAM');
INSERT INTO sbmFUNDESC VALUES ('Send_SRV_Mail','addressesSRV');
INSERT INTO sbmFUNDESC VALUES ('Send_Approval_Request','directory');
INSERT INTO sbmFUNDESC VALUES ('Send_Approval_Request','categformatDAM');
INSERT INTO sbmFUNDESC VALUES ('Send_Approval_Request','addressesDAM');
INSERT INTO sbmFUNDESC VALUES ('Send_Approval_Request','titleFile');
INSERT INTO sbmFUNDESC VALUES ('Send_APP_Mail','edsrn');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','titleFile');
INSERT INTO sbmFUNDESC VALUES ('Send_Modify_Mail','emailFile');
INSERT INTO sbmFUNDESC VALUES ('Get_Info','authorFile');
INSERT INTO sbmFUNDESC VALUES ('Get_Info','emailFile');
INSERT INTO sbmFUNDESC VALUES ('Get_Info','titleFile');
INSERT INTO sbmFUNDESC VALUES ('Make_Modify_Record','modifyTemplate');
INSERT INTO sbmFUNDESC VALUES ('Send_APP_Mail','addressesAPP');
INSERT INTO sbmFUNDESC VALUES ('Send_APP_Mail','categformatAPP');
INSERT INTO sbmFUNDESC VALUES ('Send_APP_Mail','newrnin');
INSERT INTO sbmFUNDESC VALUES ('CaseEDS','casevariable');
INSERT INTO sbmFUNDESC VALUES ('CaseEDS','casevalues');
INSERT INTO sbmFUNDESC VALUES ('CaseEDS','casesteps');
INSERT INTO sbmFUNDESC VALUES ('CaseEDS','casedefault');
INSERT INTO sbmFUNDESC VALUES ('Send_SRV_Mail','noteFile');
INSERT INTO sbmFUNDESC VALUES ('Send_SRV_Mail','emailFile');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','emailFile');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','edsrn');
INSERT INTO sbmFUNDESC VALUES ('Mail_Submitter','newrnin');
INSERT INTO sbmFUNDESC VALUES ('Upload_Files','maxsize');
INSERT INTO sbmFUNDESC VALUES ('Upload_Files','minsize');
INSERT INTO sbmFUNDESC VALUES ('Upload_Files','iconsize');
INSERT INTO sbmFUNDESC VALUES ('Upload_Files','type');
INSERT INTO sbmFUNDESC VALUES ('Make_Record','sourceTemplate');
INSERT INTO sbmFUNDESC VALUES ('Make_Record','createTemplate');
INSERT INTO sbmFUNDESC VALUES ('Print_Success','edsrn');
INSERT INTO sbmFUNDESC VALUES ('Print_Success','newrnin');
INSERT INTO sbmFUNDESC VALUES ('Print_Success','status');
INSERT INTO sbmFUNDESC VALUES ('Make_Modify_Record','sourceTemplate');
INSERT INTO sbmGFILERESULT VALUES ('HTML','HTML document');
INSERT INTO sbmGFILERESULT VALUES ('WORD','data');
INSERT INTO sbmGFILERESULT VALUES ('PDF','PDF document');
INSERT INTO sbmGFILERESULT VALUES ('PostScript','PostScript document');
INSERT INTO sbmGFILERESULT VALUES ('PostScript','data ');
INSERT INTO sbmGFILERESULT VALUES ('PostScript','HP Printer Job Language data');
INSERT INTO sbmGFILERESULT VALUES ('jpg','JPEG image');
INSERT INTO sbmGFILERESULT VALUES ('Compressed PostScript','gzip compressed data');
INSERT INTO sbmGFILERESULT VALUES ('Tarred Tex (.tar)','tar archive');
INSERT INTO sbmGFILERESULT VALUES ('JPEG','JPEG image');
INSERT INTO sbmGFILERESULT VALUES ('GIF','GIF');
INSERT INTO accROLE VALUES (1,'superadmin','superuser with all rights');
INSERT INTO accROLE VALUES (2,'photoadmin','Photo collection administrator');
INSERT INTO accROLE VALUES (3,'webaccessadmin','WebAccess administrator');
INSERT INTO user_accROLE VALUES (1,1);
INSERT INTO user_accROLE VALUES (1,3);
INSERT INTO accACTION VALUES (1,'cfgwebsearch','configure WebSearch','','no');
INSERT INTO accACTION VALUES (2,'cfgbibformat','configure BibFormat','','no');
INSERT INTO accACTION VALUES (3,'runbibindex','run BibIndex','','no');
INSERT INTO accACTION VALUES (4,'runbibupload','run BibUpload','','no');
INSERT INTO accACTION VALUES (5,'runwebcoll','run webcoll','collection','yes');
INSERT INTO accACTION VALUES (6,'runbibformat','run BibFormat','format','yes');
INSERT INTO accACTION VALUES (7,'cfgwebaccess','configure WebAccess','','no');
INSERT INTO accACTION VALUES (8,'accdelegaterole','delegate subroles inside WebAccess','role','no');
INSERT INTO accACTION VALUES (9,'runbibtaskex','run BibTaskEx example', '','no');
INSERT INTO accACTION VALUES (11,'submit','use webSubmit','doctype,act','no');
INSERT INTO accACTION VALUES (12,'cfgwebsubmit','configure webSubmit','','no');
INSERT INTO accACTION VALUES (13,'referee','referee document type doctype','doctype,categ','yes');
INSERT INTO accACTION VALUES (14,'runbibrank','run BibRank','','no');
INSERT INTO accACTION VALUES (15,'cfgbibrank','configure BibRank','','no');
INSERT INTO accACTION VALUES (16,'cfgbibindex','configure BibIndex','','no');
INSERT INTO accACTION VALUES (17,'cfgbibharvest','configure BibHarvest','','no');
INSERT INTO accACTION VALUES (18,'runoaiharvest','run BibHarvest periodical harvesting','','no');
INSERT INTO accACTION VALUES (19,'cfgwebcomment','configure WebComment','','no');
INSERT INTO accACTION VALUES (20,'runoaiarchive','run OAI archive','','no');
INSERT INTO accARGUMENT VALUES (1,'collection','Pictures');
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,1, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,2, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,3, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,4, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,6,-1,-1);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,7, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (2,5, 1, 1);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (3,7, 0, 0);
INSERT INTO accROLE_accACTION_accARGUMENT VALUES (1,9, 0, 0);
argstr += (argstr and ',' or '') + str(id_argument)
# check if equal authorization exists
for (id_trav, ) in run_sql("""SELECT DISTINCT argumentlistid FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = '%s' AND id_accACTION = '%s' """% (id_role, id_action)):
listlength = run_sql("""SELECT COUNT(*) FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = '%s' AND id_accACTION = '%s' AND argumentlistid = '%s' AND
id_accARGUMENT IN (%s) """ % (id_role, id_action, id_trav, argstr))[0][0]
notlist = run_sql("""SELECT COUNT(*) FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = '%s' AND id_accACTION = '%s' AND argumentlistid = '%s' AND
id_accARGUMENT NOT IN (%s) """ % (id_role, id_action, id_trav, argstr))[0][0]
# this means that a duplicate already exists
if not notlist and listlength == len(id_arguments): return []
# find new arglistid, highest + 1
try: arglistid = 1 + run_sql("""SELECT MAX(argumentlistid) FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND id_accACTION = %s """
% (id_role, id_action))[0][0]
except (IndexError, TypeError): arglistid = 1
if arglistid <= 0: arglistid = 1
# insert
for id_argument in id_arguments:
run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT values (%s, %s, %s, %s) """
if not run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND id_accACTION = %s AND argumentlistid = %s AND id_accARGUMENT = %s"""
% (id_role, id_action, 0, 0)):
if verbose: print 'ids: try to insert'
result = run_sql("""INSERT INTO accROLE_accACTION_accARGUMENT values (%s, %s, %s, %s)""" % (id_role, id_action, 0, 0))
return ((id_role, id_action, 0, 0), )
else:
if verbose: print 'ids: already existed'
return 0
else:
if verbose: print 'ids: arguments exist'
argstr = ''
# check that the argument exists, and that it is a valid key
if verbose: print 'ids: checking all the arguments'
for id_argument in id_arguments:
res_arg = run_sql("""SELECT * FROM accARGUMENT WHERE id = %s""" % (id_argument, ))
if not res_arg or res_arg[0][1] not in allowedkeys:
return 0
else:
if argstr: argstr += ','
argstr += '%s' % (id_argument, )
# arglistid = -1 means that the user wants a new group
if verbose: print 'ids: find arglistid'
if arglistid < 0:
# check if such single group already exists
for (id_trav, ) in run_sql("""SELECT DISTINCT argumentlistid FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = '%s' AND id_accACTION = '%s' """
% (id_role, id_action)):
listlength = run_sql("""SELECT COUNT(*) FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = '%s' AND id_accACTION = '%s' AND argumentlistid = '%s' AND
id_accARGUMENT IN (%s) """ % (id_role, id_action, id_trav, argstr))[0][0]
notlist = run_sql("""SELECT COUNT(*) FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = '%s' AND id_accACTION = '%s' AND argumentlistid = '%s' AND
id_accARGUMENT NOT IN (%s) """ % (id_role, id_action, id_trav, argstr))[0][0]
# this means that a duplicate already exists
if not notlist and listlength == len(id_arguments): return 0
# find new arglistid
try:
arglistid = run_sql("""SELECT MAX(argumentlistid) FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND id_accACTION = %s """ %
(id_role, id_action))[0][0] + 1
except ProgrammingError: return 0
except (IndexError, TypeError): arglistid = 1
if arglistid <= 0: arglistid = 1
if verbose: print 'ids: insert all the entries'
# all references are valid, insert: one entry in raa for each argument
for id_argument in id_arguments:
if not run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND id_accACTION = %s AND id_accARGUMENT = %s AND argumentlistid = %s""" %
if run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT WHERE id_accROLE = %s AND id_accACTION = %s AND id_accARGUMENT = 0 AND argumentlistid = 0""", (id_role, id_action)):
return [['#', 'argument keyword'], ['0', 'action without arguments']]
# tuples into lists
res2, arglistids = [], {}
for res in res1:
res2.append([])
for r in res: res2[-1].append(r)
res2.sort()
# create multilevel dictionary
for res in res2:
a, kw, value = res # rolekey, argumentlistid, keyword, value
if kw not in keywords: continue
if not arglistids.has_key(a):
arglistids[a] = {}
# fill dictionary
if not arglistids[a].has_key(kw): arglistids[a][kw] = [value]
elif not value in arglistids[a][kw]: arglistids[a][kw] = arglistids[a][kw] + [value]
# fill list with all possible combinations
res3 = []
# rolekeys = roles2.keys(); rolekeys.sort()
for a in arglistids.keys(): # argumentlistids
# fill a list with the new entries, shortcut and copying first keyword list
next_arglistid = []
for row in arglistids[a][keywords[0]]: next_arglistid.append([a, row[:] ])
# run through the rest of the keywords
for kw in keywords[1:]:
if not arglistids[a].has_key(kw): arglistids[a][kw] = ['optional value']
new_list = arglistids[a][kw][:]
new_len = len(new_list)
# duplicate the list
temp_list = []
for row in next_arglistid:
for i in range(new_len): temp_list.append(row[:])
# append new values
for i in range(len(temp_list)):
new_item = new_list[i % new_len][:]
temp_list[i].append( new_item )
next_arglistid = temp_list[:]
res3.extend(next_arglistid)
res3.sort()
# if optional allowed, put on top
opt = run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE = %s AND
id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1""" % (id_role, id_action))
if opt: res3.insert(0, [-1] + ['optional value'] * len(keywords))
except (ProgrammingError, IndexError): return (3, "%s %s" % (cfg_webaccess_warning_msgs[3] % name_action, (called_from and cfg_webaccess_msgs[1] or "")))
defkeys = aallowedkeywords.split(',')
for key in arguments.keys():
if key not in defkeys: return (8, "%s %s" % (cfg_webaccess_warning_msgs[8], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or ""))) #incorrect arguments?
# -------------------------------------------
# TASK 1: check if user is a superadmin
# we know the action exists. no connection with role is necessary
# passed arguments must have allowed keywords
# no check to see if the argument exists
if verbose: print 'task 1 - is user %s' % (SUPERADMINROLE, )
# TASK 2: check if user exists and find all the user's roles and create or-string
if verbose: print 'task 2 - find user and userroles'
try:
query2 = """SELECT email, note from user where id=%s""" % id_user
res2 = run_sql(query2)
if not res2:
raise Exception
if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS >= 1 and res2[0][1] not in [1, "1"]:
if res[0][1]:
return (9, "%s %s" % (cfg_webaccess_warning_msgs[9] % res[0][1], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or "")))
else:
raise Exception
query2 = """SELECT ur.id_accROLE FROM user_accROLE ur WHERE ur.id_user=%s ORDER BY ur.id_accROLE """ % id_user
res2 = run_sql(query2)
except Exception: return (6, "%s %s" % (cfg_webaccess_warning_msgs[6], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or "")))
if not res2: return (2, "%s %s" % (cfg_webaccess_warning_msgs[2], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or ""))) #user has no roles
# -------------------------------------------
# create role string (add default value? roles='(raa.id_accROLE='def' or ')
str_roles = ''
for (role, ) in res2:
if str_roles: str_roles += ','
str_roles += '%s' % (role, )
# TASK 3: authorizations with no arguments given
if verbose: print 'task 3 - checks with no arguments'
if not arguments:
# 3.1
if optional == 'no':
if verbose: print ' - action with zero arguments'
connection = run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE IN (%s) AND
id_accACTION = %s AND
argumentlistid = 0 AND
id_accARGUMENT = 0 """ % (str_roles, id_action))
if connection and 1:
return (0, cfg_webaccess_warning_msgs[0])
else:
return (1, "%s %s" % (cfg_webaccess_warning_msgs[1], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or "")))
# 3.2
if optional == 'yes':
if verbose: print ' - action with optional arguments'
connection = run_sql("""SELECT * FROM accROLE_accACTION_accARGUMENT
WHERE id_accROLE IN (%s) AND
id_accACTION = %s AND
id_accARGUMENT = -1 AND
argumentlistid = -1 """ % (str_roles, id_action))
if connection and 1:
return (0, cfg_webaccess_warning_msgs[0])
else:
return (1, "%s %s" % (cfg_webaccess_warning_msgs[1], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or "")))
# none of the zeroargs tests succeded
if verbose: print ' - not authorization without arguments'
return (5, "%s %s" % (cfg_webaccess_warning_msgs[5], (called_from and "%s" % (cfg_webaccess_msgs[1] or ""))))
# TASK 4: create list of keyword and values that satisfy part of the authentication and create or-string
if verbose: print 'task 4 - create keyword=value pairs'
# create dictionary with default values and replace entries from input arguments
defdict = {}
for key in defkeys:
try: defdict[key] = arguments[key]
except KeyError: return (5, "%s %s" % (cfg_webaccess_warning_msgs[5], (called_from and "%s" % (cfg_webaccess_msgs[1] or "")))) # all keywords must be present
# except KeyError: defdict[key] = 'x' # default value, this is not in use...
except ProgrammingError: return (3, "%s %s" % (cfg_webaccess_warning_msgs[3], (called_from and "%s" % (cfg_webaccess_msgs[1] or ""))))
if not res4: return (1, "%s %s" % (cfg_webaccess_warning_msgs[1], (called_from and "%s %s" % (cfg_webaccess_msgs[0] % name_action[3:], cfg_webaccess_msgs[1]) or ""))) # no entries at all
res5 = []
for res in res4:
res5.append(res)
res5.sort()
# USER AUTHENTICATED TO PERFORM ACTION WITH ONE ARGUMENT
if len(defdict) == 1: return (0, cfg_webaccess_warning_msgs[0])
# CHECK WITH MORE THAN 1 ARGUMENT
# TASK 6: run through the result and try to satisfy authentication
if verbose: print 'task 6 - combine results and try to satisfy'
cur_role = cur_action = cur_arglistid = 0
booldict = {}
for key in defkeys: booldict[key] = 0
# run through the results
for (role, action, arglistid, arg, keyword, val) in res5 + [(-1, -1, -1, -1, -1, -1)]:
# not the same role or argumentlist (authorization group), i.e. check if thing are satisfied
# if cur_arglistid != arglistid or cur_role != role or cur_action != action:
if (cur_arglistid, cur_role, cur_action) != (arglistid, role, action):
if verbose: print ' : checking new combination',
# test if all keywords are satisfied
for value in booldict.values():
if not value: break
else:
if verbose: print '-> found satisfying combination'
return (0, cfg_webaccess_warning_msgs[0]) # USER AUTHENTICATED TO PERFORM ACTION
if verbose: print '-> not this one'
# assign the values for the current tuple from the query
output += """%s %s<br>""" % (system, (CFG_EXTERNAL_AUTHENTICATION[system][1] and "(Default)" or ""))
output += "<br><b>Changing the settings:</b><br>"
output += "Currently, all changes must be done using your favourite editor, and the webserver restarted for changes to take effect. For the settings to change, either look in the guide or in access_control_config.py ."
output += "Remember that you will be logged out as the current user."
#baskets = run_sql("SELECT basket.id, basket.name, basket.public FROM basket, user_basket WHERE id_user=%s and user_basket.id_basket=basket.id" % userID)
#output += "<table><tr>"
#for (id, name, public) in baskets:
# output += "<tr><td>%s<br>Public: %s</td></tr>" % (name, (public=="y" and "Yes" or "No"))
# basket_records = run_sql("SELECT id_record, nb_order FROM basket_record WHERE id_basket=%s" % id)
# for (id_record, nb_order) in basket_records:
# output += "<tr><td></td><td>"
# output += print_record(id_record)
# output += "</td></tr>"
#
#output += "</tr></table>"
else:
output += '<b><span class="info">The account id given does not exist.</span></b>'
res = run_sql("SELECT id, email, password FROM user WHERE id=%s" % userID)
output = ""
if res:
user_pref = get_user_preferences(userID)
if confirm in [1, "1"]:
if login_method:
user_pref['login_method'] = login_method
set_user_preferences(userID, user_pref)
output += "Select default login method:<br>"
text = ""
methods = CFG_EXTERNAL_AUTHENTICATION.keys()
methods.sort()
for system in methods:
text += """<input type="radio" name="login_method" value="%s" %s>%s<br>""" % (system, (user_pref['login_method'] == system and "checked" or ""), system)
</select><br>""" % ((limit_to=="all" and "selected" or ""), (limit_to=="enabled" and "selected" or ""), (limit_to=="disabled" and "selected" or ""))
text += """<span class="adminlabel">Accounts per page:</span>
<select name="maxpage" class="admin_wvar">
<option value="25" %s>25</option>
<option value="50" %s>50</option>
<option value="100" %s>100</option>
<option value="250" %s>250</option>
<option value="500" %s>500</option>
<option value="1000" %s>1000</option>
</select><br>""" % ((maxpage==25 and "selected" or ""), (maxpage==50 and "selected" or ""), (maxpage==100 and "selected" or ""), (maxpage==250 and "selected" or ""), (maxpage==500 and "selected" or ""), (maxpage==1000 and "selected" or ""))
# 0 = production, nothing on the console, email sent
# 1 = messages on the console, email sent
# 2 = messages on the console, but no email sent
# 3 = many messages on the console, no email sent
# 4 = many messages on the console, email sent to DEVELOPERADDR
DEBUGLEVEL = 0
def update_date_lastrun(alert):
return run_sql('update user_query_basket set date_lastrun=%s where id_user=%s and id_query=%s and id_basket=%s;', (strftime("%Y-%m-%d"), alert[0], alert[1], alert[2],))
def get_alert_queries(frequency):
return run_sql('select distinct id, urlargs from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.frequency=%s and uqb.date_lastrun <= now();', (frequency,))
def get_alert_queries_for_user(uid):
return run_sql('select distinct id, urlargs, uqb.frequency from query q, user_query_basket uqb where q.id=uqb.id_query and uqb.id_user=%s and uqb.date_lastrun <= now();', (uid,))
def get_alerts(query, frequency):
r = run_sql('select id_user, id_query, id_basket, frequency, date_lastrun, alert_name, notification from user_query_basket where id_query=%s and frequency=%s;', (query['id_query'], frequency,))
""" account_list_searches: list the searches of the user
input: the user id
output: resume of the searches"""
out =""
# first detect number of queries:
nb_queries_total = 0
nb_queries_distinct = 0
id_queries_distinct = []
res = run_sql("SELECT COUNT(*),COUNT(DISTINCT(id_query)) FROM user_query WHERE id_user=%s", (uid,), 1)
try:
nb_queries_total = res[0][0]
nb_queries_distinct = res[0][1]
except:
pass
# load the right message language
_ = gettext_set_language(ln)
out += _("You have made %(number)s queries. A %(detailed_list)s is available with a posibility to (a) view search results and (b) subscribe for automatic email alerting service for these queries") % {
out += """<P>""" + _("You have performed <B>%(number)d</B> searches (<strong>%(different)d</strong> different questions) during the last 30 days or so.""") % {
'number' : nb_queries_total,
'different' : nb_queries_distinct
} + """</P>"""
else:
# permanent="y"
out += """<P>Here are listed the <B>%s</B> most popular searches.</P>""" % len(query_result)
# display the list of searches
out += """<TABLE border="1" cellspacing="0" cellpadding="3" width="100%%">
"""return html warning box for non registered users"""
_ = gettext_set_language(ln)
message = _("You are logged in as a guest user, so your baskets will disappear at the end of the current session. If you wish you can %slogin or register here%s.")
subject = "An error report has been sent from a user"
body = '''
The following comment has been reported a total of %(cmt_reported)s times.
Author: nickname = %(nickname)s
email = %(user_email)s
user_id = %(uid)s
This user has:
total number of reports = %(user_nb_abuse_reports)s
%(votes)s
Comment: comment_id = %(cmt_id)s
record_id = %(id_bibrec)s
date written = %(cmt_date)s
nb reports = %(cmt_reported)s
%(review_stuff)s
body =
---start body---
%(cmt_body)s
---end body---
Please go to the Comments Admin Panel %(comment_admin_link)s to delete this message if necessary. A warning will be sent to the user in question.''' % \
A brief description of things we strive at, more or less unsuccessfully.
1. Packaging
We use the classical GNU Autoconf/Automake approach, for tutorial
see e.g. <a href="http://www.amath.washington.edu/~lf/tutorials/autoconf/tutorial_toc.html">Learning the GNU development tools</a> or the <a href="http://sources.redhat.com/autobook/autobook/autobook_toc.html">AutoBook</a>.
2. Modules
CDSware started as a set of pretty independent modules developed by
independent people with independent styles. This was even more
pronounced by the original use of many different languages
(e.g. Python, PHP, Perl). Now the CDSware code base is striving to
use Python everywhere, except in speed-critical parts when a
compiled language such as Common Lisp may come to the rescue in the
near future.
When modifying an existing module, we propose to strictly continue
using whatever coding style the module was originally written into.
When writing new modules, we propose to stick to the
below-mentioned standards.
The code integration across modules is happening, but is slow.
Therefore, don't be surprised to see that there is a lot of room to
refactor.
3. WML/ePerl/etc
This is not so important, because not many lines-of-code were
written in WML/ePerl. We prefer to loosely follow the GNU way, as
always.
4. Python
We aim at following recommendations from <a
href="http://www.python.org/peps/pep-0008.html">PEP 8</a>, although
the existing code surely do not fulfil them here and there.
The most easily notable exception from PEP 8 is perhaps the line
width, that usually exceeds 72 characters per line in our sources,
as we tend to use fullscreen editors. The code indentation is done
via spaces only, please do not use tabs. One tab counts as four
spaces. Emacs users can look into our <a
href="cdsware.el">cdsware.el</a> for inspiration.
All the Python code should be extensively documented via
docstrings, so you can always try to run pydoc file.py to
peruse a module's documentation in one go.
Do not forget to run pylint on your code to check errors like
uninitialized variables and to improve its quality and conformance
to coding standards. The typical incantation may be:
$ cd <LIBDIR>/python/cdsware
$ pylint --max-line-length=160 oai_repository.py | most
Read and implement pylint suggestions.
Do not hardcode magic constants in your code. Every magic string or
a number should be put into accompanying file_config.py with
symbol name beginning by cfg_modulename_*.
Clearly separate interfaces from implementation. Document your
interfaces. Do not expose to other modules anything that does not
have to be exposed. Apply principle of least information.
Create as few new library files as possible. Do not create many
nested files in nested modules; rather put all the lib files in one
dir with bibindex_foo and bibindex_bar names.
Use imperative/functional paradigm rather then OO. If you do use
OO, then stick to as simple class hierarchy as possible. Recall
that method calls and exception handling in Python are quite
expensive.
Use rather the good old foo_bar naming convention for symbols (both
variables and function names) instead of fooBar CaMelCaSe
convention. (Except for Class names where UppercaseSymbolNames are
to be used.)
Pay special attention to name your symbols descriptively. Your
code is going to be read and work with by others and its symbols
should be self-understandable without any comments and without
studying other parts of the code. For example, use proper English
words, not abbreviations that can be misspelled in many a way; use
words that go in pair (e.g. create/destroy, start/stop; never
create/stop); use self-understandable symbol names
(e.g. list_of_file_extensions rather than list2); never misname
symbols (e.g. score_list should hold the list of scores and nothing
else - if in the course of development you change the semantics of
what the symbol holds then change the symbol name too). Do not be
afraid to use long descriptive names; good editors such as Emacs
can tab-complete symbols for you.
When hacking module A, pay close attention to ressemble existing
coding convention in A, even if it is legacy-weird and even if we
use a different technique elsewhere. (Unless the whole module A is
going to be refactored, of course.)
All core code should come with a suitable set of test cases, see
the <a href="testsuite.html">test suite strategy</a> for details.
Speed-critical parts should be profiled with pyprof. Do not forget
to use tricks like psyco.
For more hints, thoughts, and other ruminations, see <a
href="http://simko.info/vademecum/">Vademecum and Essays</a>.
5. PHP
We are moving slowly away out of PHP so that there may be several
practices in place with the PHP code present in CDSware. Usually
this is consistent within modules but inconsistent across modules.
For example, some old code used Emacs' perl-mode, following
traditional K&R C style, while some other old code tried to stick
to <a href="http://pear.php.net/manual/en/standards.php">PEAR recommendations</a>.
6. MySQL
Table naming policy is, roughly and briefly:
- "foo": table names in lowercase, without prefix, used by me
for WebSearch
- "foo_bar": underscores represent M:N relationship between
"foo" and "bar", to tie the two tables together
- "bib*": many tables to hold the metadata and relationships
between them
- "idx*": idx is the table name prefix used by BibIndex
- "rnk*": rnk is the table name prefix used by BibRank
- "flx*": flx is the table name prefix used by FlexElink (also known as
BibFormat)
- "sbm*": sbm is the table name prefix used by WebSubmit
- "sch*": sch is the table name prefix used by BibSched
- "collection*": many tables to describe collections and search
interface pages
- "user*" : many tables to describe personal features (baskets,
"""Splits search pattern and search field into a list of independently searchable units.
- A search unit consists of '(operator, pattern, field, type, hitset)' tuples where
'operator' is set union (|), set intersection (+) or set exclusion (-);
'pattern' is either a word (e.g. muon*) or a phrase (e.g. 'nuclear physics');
'field' is either a code like 'title' or MARC tag like '100__a';
'type' is the search type ('w' for word file search, 'a' for access file search).
- Optionally, the function accepts the match type argument 'm'.
If it is set (e.g. from advanced search interface), then it
performs this kind of matching. If it is not set, then a guess is made.
'm' can have values: 'a'='all of the words', 'o'='any of the words',
'p'='phrase/substring', 'r'='regular expression',
'e'='exact value'.
- Warnings are printed on req (when not None) in case of HTML output formats."""
opfts = [] # will hold (o,p,f,t,h) units
## check arguments: if matching type phrase/string/regexp, do we have field defined?
if (m=='p' or m=='r' or m=='e') and not f:
m = 'a'
if of.startswith("h"):
print_warning(req, "This matching type cannot be used within <em>any field</em>. I will perform a word search instead." )
print_warning(req, "If you want to phrase/substring/regexp search in a specific field, e.g. inside title, then please choose <em>within title</em> search option.")
## is desired matching type set?
if m:
## A - matching type is known; good!
if m == 'e':
# A1 - exact value:
opfts.append(['+',p,f,'a']) # '+' since we have only one unit
elif m == 'p':
# A2 - phrase/substring:
opfts.append(['+',"%"+p+"%",f,'a']) # '+' since we have only one unit
elif m == 'r':
# A3 - regular expression:
opfts.append(['+',p,f,'r']) # '+' since we have only one unit
elif m == 'a' or m == 'w':
# A4 - all of the words:
p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed
for word in get_words_from_pattern(p):
opfts.append(['+',word,f,'w']) # '+' in all units
elif m == 'o':
# A5 - any of the words:
p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed
for word in get_words_from_pattern(p):
if len(opfts)==0:
opfts.append(['+',word,f,'w']) # '+' in the first unit
else:
opfts.append(['|',word,f,'w']) # '|' in further units
else:
if of.startswith("h"):
print_warning(req, "Matching type '%s' is not implemented yet." % m, "Warning")
opfts.append(['+',"%"+p+"%",f,'a'])
else:
## B - matching type is not known: let us try to determine it by some heuristics
if f and p[0]=='"' and p[-1]=='"':
## B0 - does 'p' start and end by double quote, and is 'f' defined? => doing ACC search
opfts.append(['+',p[1:-1],f,'a'])
elif f and p[0]=="'" and p[-1]=="'":
## B0bis - does 'p' start and end by single quote, and is 'f' defined? => doing ACC search
opfts.append(['+','%'+p[1:-1]+'%',f,'a'])
elif f and p[0]=="/" and p[-1]=="/":
## B0ter - does 'p' start and end by a slash, and is 'f' defined? => doing regexp search
opfts.append(['+',p[1:-1],f,'r'])
elif f and string.find(p, ',') >= 0:
## B1 - does 'p' contain comma, and is 'f' defined? => doing ACC search
opfts.append(['+',p,f,'a'])
elif f and str(f[0:2]).isdigit():
## B2 - does 'f' exist and starts by two digits? => doing ACC search
opfts.append(['+',p,f,'a'])
else:
## B3 - doing WRD search, but maybe ACC too
# search units are separated by spaces unless the space is within single or double quotes
# so, let us replace temporarily any space within quotes by '__SPACE__'
p = sre_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", p)
p = sre_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", p)
p = sre_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", p)
# wash argument:
p = sre_equal.sub(":", p)
p = sre_logical_and.sub(" ", p)
p = sre_logical_or.sub(" |", p)
p = sre_logical_not.sub(" -", p)
p = sre_operators.sub(r' \1', p)
for pi in split(p): # iterate through separated units (or items, as "pi" stands for "p item")
pi = sre_pattern_space.sub(" ", pi) # replace back '__SPACE__' by ' '
# firstly, determine set operator
if pi[0] == '+' or pi[0] == '-' or pi[0] == '|':
oi = pi[0]
pi = pi[1:]
else:
# okay, there is no operator, so let us decide what to do by default
oi = '+' # by default we are doing set intersection...
# secondly, determine search pattern and field:
if string.find(pi, ":") > 0:
fi, pi = split(pi, ":", 1)
else:
fi, pi = f, pi
# look also for old ALEPH field names:
if fi and cfg_fields_convert.has_key(string.lower(fi)):
fi = cfg_fields_convert[string.lower(fi)]
# wash 'pi' argument:
if sre_quotes.match(pi):
# B3a - quotes are found => do ACC search (phrase search)
if fi:
if pi[0] == '"' and pi[-1] == '"':
pi = string.replace(pi, '"', '') # remove quote signs
opfts.append([oi,pi,fi,'a'])
elif pi[0] == "'" and pi[-1] == "'":
pi = string.replace(pi, "'", "") # remove quote signs
opfts.append([oi,"%"+pi+"%",fi,'a'])
else: # unbalanced quotes, so do WRD query:
opfts.append([oi,pi,fi,'w'])
else:
# fi is not defined, look at where we are doing exact or subphrase search (single/double quotes):
if pi[0]=='"' and pi[-1]=='"':
opfts.append([oi,pi[1:-1],"anyfield",'a'])
if of.startswith("h"):
print_warning(req, "Searching for an exact match inside any field may be slow. You may want to search for words instead, or choose to search within specific field.")
else:
# nope, subphrase in global index is not possible => change back to WRD search
pi = strip_accents(pi) # strip accents for 'w' mode, FIXME: delete when not needed
for pii in get_words_from_pattern(pi):
# since there may be '-' and other chars that we do not index in WRD
opfts.append([oi,pii,fi,'w'])
if of.startswith("h"):
print_warning(req, "The partial phrase search does not work in any field. I'll do a boolean AND searching instead.")
print_warning(req, "If you want to do a partial phrase search in a specific field, e.g. inside title, then please choose 'within title' search option.", "Tip")
print_warning(req, "If you want to do exact phrase matching, then please use double quotes.", "Tip")
elif fi and str(fi[0]).isdigit() and str(fi[0]).isdigit():
# B3b - fi exists and starts by two digits => do ACC search
opfts.append([oi,pi,fi,'a'])
elif fi and not get_index_id(fi):
# B3c - fi exists but there is no words table for fi => try ACC search
opfts.append([oi,pi,fi,'a'])
elif fi and pi.startswith('/') and pi.endswith('/'):
# B3d - fi exists and slashes found => try regexp search
opfts.append([oi,pi[1:-1],fi,'r'])
else:
# B3e - general case => do WRD search
pi = strip_accents(pi) # strip accents for 'w' mode, FIXME: delete when not needed
# speed up HitSet operations by ~20% if Psyco is installed:
try:
import psyco
psyco.bind(HitSet)
except:
pass
def escape_string(s):
"Escapes special chars in string. For MySQL queries."
s = MySQLdb.escape_string(s)
return s
def wash_colls(cc, c, split_colls=0):
"""Wash collection list by checking whether user has deselected
anything under 'Narrow search'. Checks also if cc is a list or not.
Return list of cc, colls_to_display, colls_to_search since the list
of collections to display is different from that to search in.
This is because users might have chosen 'split by collection'
functionality.
The behaviour of "collections to display" depends solely whether
user has deselected a particular collection: e.g. if it started
from 'Articles and Preprints' page, and deselected 'Preprints',
then collection to display is 'Articles'. If he did not deselect
anything, then collection to display is 'Articles & Preprints'.
The behaviour of "collections to search in" depends on the
'split_colls' parameter:
* if is equal to 1, then we can wash the colls list down
and search solely in the collection the user started from;
* if is equal to 0, then we are splitting to the first level
of collections, i.e. collections as they appear on the page
we started to search from;
"""
colls_out = []
colls_out_for_display = []
# check what type is 'cc':
if type(cc) is list:
for ci in cc:
if collection_reclist_cache.has_key(ci):
# yes this collection is real, so use it:
cc = ci
break
else:
# check once if cc is real:
if not collection_reclist_cache.has_key(cc):
cc = cdsname # cc is not real, so replace it with Home collection
# check type of 'c' argument:
if type(c) is list:
colls = c
else:
colls = [c]
# remove all 'unreal' collections:
colls_real = []
for coll in colls:
if collection_reclist_cache.has_key(coll):
colls_real.append(coll)
colls = colls_real
# check if some real collections remain:
if len(colls)==0:
colls = [cc]
# then let us check the list of non-restricted "real" sons of 'cc' and compare it to 'coll':
query = "SELECT c.name FROM collection AS c, collection_collection AS cc, collection AS ccc WHERE c.id=cc.id_son AND cc.id_dad=ccc.id AND ccc.name='%s' AND cc.type='r' AND c.restricted IS NULL" % MySQLdb.escape_string(cc)
res = run_sql(query)
l_cc_nonrestricted_sons = []
l_c = colls
for row in res:
l_cc_nonrestricted_sons.append(row[0])
l_c.sort()
l_cc_nonrestricted_sons.sort()
if l_cc_nonrestricted_sons == l_c:
colls_out_for_display = [cc] # yep, washing permitted, it is sufficient to display 'cc'
else:
colls_out_for_display = colls # nope, we need to display all 'colls' successively
# remove duplicates:
colls_out_for_display_nondups=filter(lambda x, colls_out_for_display=colls_out_for_display: colls_out_for_display[x-1] not in colls_out_for_display[x:], range(1, len(colls_out_for_display)+1))
"""Sort records in 'recIDs' list according sort field 'sort_field' in order 'sort_order'.
If more than one instance of 'sort_field' is found for a given record, try to choose that that is given by
'sort pattern', for example "sort by report number that starts by CERN-PS".
Note that 'sort_field' can be field code like 'author' or MARC tag like '100__a' directly."""
_ = gettext_set_language(ln)
## check arguments:
if not sort_field:
return recIDs
if len(recIDs) > cfg_nb_records_to_sort:
if of.startswith('h'):
print_warning(req, _("Sorry, sorting is allowed on sets of up to %d records only. Using default sort order (\"latest first\").") % cfg_nb_records_to_sort, "Warning")
return recIDs
sort_fields = string.split(sort_field, ",")
recIDs_dict = {}
recIDs_out = []
## first deduce sorting MARC tag out of the 'sort_field' argument:
tags = []
for sort_field in sort_fields:
if sort_field and str(sort_field[0:2]).isdigit():
# sort_field starts by two digits, so this is probably a MARC tag already
tags.append(sort_field)
else:
# let us check the 'field' table
query = """SELECT DISTINCT(t.value) FROM tag AS t, field_tag AS ft, field AS f
WHERE f.code='%s' AND ft.id_field=f.id AND t.id=ft.id_tag
ORDER BY ft.score DESC""" % sort_field
res = run_sql(query)
if res:
for row in res:
tags.append(row[0])
else:
if of.startswith('h'):
print_warning(req, _("Sorry, '%s' does not seem to be a valid sort option. Choosing title sort instead.") % sort_field, "Error")
tags.append("245__a")
if verbose >= 3:
print_warning(req, "Sorting by tags %s." % tags)
if sort_pattern:
print_warning(req, "Sorting preferentially by %s." % sort_pattern)
## check if we have sorting tag defined:
if tags:
# fetch the necessary field values:
for recID in recIDs:
val = "" # will hold value for recID according to which sort
vals = [] # will hold all values found in sorting tag for recID
for tag in tags:
vals.extend(get_fieldvalues(recID, tag))
if sort_pattern:
# try to pick that tag value that corresponds to sort pattern
bingo = 0
for v in vals:
if v.startswith(sort_pattern): # bingo!
bingo = 1
val = v
break
if not bingo: # sort_pattern not present, so add other vals after spaces
val = sort_pattern + " " + string.join(vals)
else:
# no sort pattern defined, so join them all together
res = run_sql("SHOW TABLE STATUS LIKE 'collection'")
out += "- collection table last updated: %s" % str(res[0][11])
out += "<br>- reclist cache timestamp: %s" % collection_reclist_cache_timestamp
out += "<br>- reclist cache contents:"
out += "<blockquote>"
for coll in collection_reclist_cache.keys():
if collection_reclist_cache[coll]:
out += "%s (%d)<br>" % (coll, get_collection_reclist(coll)._nbhits)
out += "</blockquote>"
# show search cache:
out += "<h3>Search Cache</h3>"
out += "<blockquote>"
if len(search_cache):
out += """<table border="=">"""
out += "<tr><td><strong>%s</strong></td><td><strong>%s</strong></td><td><strong>%s</strong></td><td><strong>%s</strong></td></tr>" % ("Pattern","Field","Collection","Number of Hits")
for search_cache_key in search_cache.keys():
p, f, c = string.split(search_cache_key, "@", 2)
# find out about length of cached data:
l = 0
for coll in search_cache[search_cache_key]:
l += search_cache[search_cache_key][coll]._nbhits
out += "<tr><td>%s</td><td>%s</td><td>%s</td><td>%d</td></tr>" % (p, f, c, l)
out += "</table>"
else:
out += "<p>Search cache is empty."
out += "</blockquote>"
out += """<p><a href="%s/search.py/cache?action=clear">clear cache</a>""" % weburl
# show field i18nname cache:
out += "<h3>Field I18N names cache</h3>"
res = run_sql("SHOW TABLE STATUS LIKE 'fieldname'")
out += "- fieldname table last updated: %s" % str(res[0][11])
out += "<br>- i18nname cache timestamp: %s" % field_i18nname_cache_timestamp
out += "<br>- i18nname cache contents:"
out += "<blockquote>"
for field in field_i18nname_cache.keys():
for ln in field_i18nname_cache[field].keys():
out += "%s, %s = %s<br>" % (field, ln, field_i18nname_cache[field][ln])
out += "</blockquote>"
# show collection i18nname cache:
out += "<h3>Collection I18N names cache</h3>"
res = run_sql("SHOW TABLE STATUS LIKE 'collectionname'")
out += "- collectionname table last updated: %s" % str(res[0][11])
out += "<br>- i18nname cache timestamp: %s" % collection_i18nname_cache_timestamp
out += "<br>- i18nname cache contents:"
out += "<blockquote>"
for coll in collection_i18nname_cache.keys():
for ln in collection_i18nname_cache[coll].keys():
out += "%s, %s = %s<br>" % (coll, ln, collection_i18nname_cache[coll][ln])
out += "</blockquote>"
req.write(out)
return "\n"
def perform_request_log(req, date=""):
"""Display search log information for given date."""
'founds' : _("<strong>Results overview:</strong> Found <strong>%s</strong> records in %.2f seconds.") % (self.tmpl_nice_number(results_final_nb_total, ln), cpu_time)
}
# then print hits per collection:
for coll in colls:
if results_final_nb.has_key(coll['code']) and results_final_nb[coll['code']] > 0:
out += """<strong><a href="#%(coll)s">%(coll_name)s</a></strong>,
<a href="#%(coll)s">%(number)s</a><br>""" % {
'coll' : urllib.quote(coll['code']),
'coll_name' : coll['name'],
'number' : _("<strong>%s</strong> records found") % self.tmpl_nice_number(results_final_nb[coll['code']], ln)
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""CDSware WebSearch Administrator Interface."""
import cgi
import re
import MySQLdb
import Numeric
import os
import urllib
import random
import marshal
import time
from zlib import compress,decompress
from mod_python import apache
from cdsware.bibrankadminlib import write_outcome,modify_translations,get_def_name,get_i8n_name,get_name,get_rnk_nametypes,get_languages,check_user,is_adminuser,adderrorbox,addadminbox,tupletotable,tupletotable_onlyselected,addcheckboxes,createhiddenform,serialize_via_numeric_array_dumps,serialize_via_numeric_array_compr,serialize_via_numeric_array_escape,serialize_via_numeric_array,deserialize_via_numeric_array,serialize_via_marshal,deserialize_via_marshal
from cdsware.dbquery import run_sql
from cdsware.config import *
from cdsware.webpage import page, pageheaderonly, pagefooteronly
text += """<option value="%s" %s>%s</option>""" % (id, (func in ["0", 0] and confirm in ["0", 0] and int(rnkID) == int(id)) and 'selected="selected"' or '' , name)
if confirm in ["1", 1] and func in ["0", 0] and int(rnkID) != -1:
output += write_outcome(finresult)
elif confirm not in ["0", 0] and func in ["0", 0]:
output += """<b><span class="info">Please select a rank method.</span></b>"""
coll_list = get_col_rnk(colID, ln)
if coll_list:
text = """
<span class="adminlabel">Disable:</span>
<select name="rnkID" class="admin_w200">
<option value="-1">- select rank method-</option>
"""
for (id, name) in coll_list:
text += """<option value="%s" %s>%s</option>""" % (id, (func in ["1", 1] and confirm in ["0", 0] and int(rnkID) == int(id)) and 'selected="selected"' or '' , name)
if confirm not in [-1, "-1"] and not (add_son and add_dad and rtype):
output2 += """<b><span class="info">All fields must be filled.</span></b><br><br>
"""
elif add_son and add_dad and rtype:
add_son = int(add_son)
add_dad = int(add_dad)
if confirm not in [-1, "-1"]:
if add_son == add_dad:
output2 += """<b><span class="info">Cannot add a collection as a pointer to itself.</span></b><br><br>
"""
elif check_col(add_dad, add_son):
res = add_col_dad_son(add_dad, add_son, rtype)
output2 += write_outcome(res)
if res[0] == 1:
output2 += """<b><span class="info"><br> The collection will appear on your website after the next webcoll run. You can either run it manually or wait until bibsched does it for you.</span></b><br><br>
"""
else:
output2 += """<b><span class="info">Cannot add the collection '%s' as a %s subcollection of '%s' since it will either create a loop, or the association already exists.</span></b><br><br>
""" % (col_dict[add_son], (rtype=="r" and 'regular' or 'virtual'), col_dict[add_dad])
res = run_sql("SELECT dbquery FROM collection WHERE id=%s" % colID)
dbquery = res[0][0]
if not dbquery:
dbquery = ''
reg_sons = len(get_col_tree(colID, 'r'))
vir_sons = len(get_col_tree(colID, 'v'))
if reg_sons > 1:
if dbquery:
output += "Warning: This collection got subcollections, and should because of this not have a collection query, for further explanation, check the WebSearch Guide<br>"
elif reg_sons <= 1:
if not dbquery:
output += "Warning: This collection does not have any subcollections, and should because of this have a collection query, for further explanation, check the WebSearch Guide<br>"
if switch and switch_col_treescore(tree[move_up], tree[switch]):
output += """<b><span class="info">Moved the %s collection '%s' up and '%s' down.</span></b><br><br>
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]], col_dict[tree[switch][0]])
else:
output += """<b><span class="info">Could not move the %s collection '%s' up and '%s' down.</span></b><br><br>
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]], col_dict[tree[switch][0]])
elif move_down:
move_down = int(move_down)
switch = find_next(tree, move_down)
if switch and switch_col_treescore(tree[move_down], tree[switch]):
output += """<b><span class="info">Moved the %s collection '%s' down and '%s' up.</span></b><br><br>
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_down][0]], col_dict[tree[switch][0]])
else:
output += """<b><span class="info">Could not move the %s collection '%s' up and '%s' down.</span></b><br><br>
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]],col_dict[tree[switch][0]])
elif delete:
delete = int(delete)
if confirm in [0, "0"]:
if col_dict[tree[delete][0]] != col_dict[tree[delete][3]]:
text = """<b>Do you want to remove the %s collection '%s' and its subcollections in the %s collection '%s'.</b>
""" % ((tree[delete][4]=="r" and 'regular' or 'virtual'), col_dict[tree[delete][0]], (rtype=="r" and 'regular' or 'virtual'), col_dict[tree[delete][3]])
else:
text = """<b>Do you want to remove all subcollections of the %s collection '%s'.</b>
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[delete][3]])
if move_from_id == move_to_id and move_from_rtype==move_to_rtype:
output += """<b><span class="info">Cannot move to itself.</span></b><br><br>
"""
elif tree_from[move_from_id][3] == tree_to[move_to_id][0] and move_from_rtype==move_to_rtype:
output += """<b><span class="info">The collection is already there.</span></b><br><br>
"""
elif check_col(tree_to[move_to_id][0], tree_from[move_from_id][0]) or (tree_to[move_to_id][0] == 1 and tree_from[move_from_id][3] == tree_to[move_to_id][0] and move_from_rtype != move_to_rtype):
text = """<b>Move %s collection '%s' to the %s collection '%s'.</b>
""" % ((tree_from[move_from_id][4]=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (tree_to[move_to_id][4]=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
output += """<b><span class="info">Cannot move the collection '%s' and set it as a subcollection of '%s' since it will create a loop.</span></b><br><br>
if (move_to_id != 0 and move_col_tree(tree_from[move_from_id], tree_to[move_to_id])) or (move_to_id == 0 and move_col_tree(tree_from[move_from_id], tree_to[move_to_id], move_to_rtype)):
output += """<b><span class="info">Moved %s collection '%s' to the %s collection '%s'.</span></b><br><br>
""" % ((move_from_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (move_to_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
else:
output += """<b><span class="info">Could not move %s collection '%s' to the %s collection '%s'.</span></b><br><br>
""" % ((move_from_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (move_to_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
text = """Do you want to remove the %s '%s' %s from the collection '%s'.""" % (field, fld_dict[fldID], (fldvID not in["", "None"] and "with value '%s'" % fldv_dict[fldvID] or ''), col_dict[colID])
run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s" % (vscore, colID, fldID, fldvID))
vscore -= 1
output += write_outcome((1, ""))
else:
output += write_outcome((0, (0, "No values to order")))
if cdsname != run_sql("SELECT name from collection WHERE id=1")[0][0]:
res = run_sql("update collection set name='%s' where id=1" % cdsname)
if res:
fin_output += """<b><span class="info">The name of the root collection has been modified to be the same as the cdsware installation name given prior to installing cdsware.</span><b><br>"""
<td>1. <small><a href="%s/admin/websearch/websearchadmin.py?colID=%s&ln=%s&mtype=perform_addcollection">Create new collection</a></small></td>
<td>2. <small><a href="%s/admin/websearch/websearchadmin.py?colID=%s&ln=%s&mtype=perform_addcollectiontotree">Attach collection to tree</a></small></td>
text += """</td><td></td><td></td><td></td><td><table border="0" cellspacing="0" cellpadding="0"><tr><td>
"""
if i == 0:
tstack.append((id_son, dad, 1))
else:
tstack.append((id_son, dad, tables))
if up == 1 and edit:
text += """<a href="%s/admin/websearch/websearchadmin.py/modifycollectiontree?colID=%s&ln=%s&move_up=%s&rtype=%s#%s"><img border="0" src="%s/img/smallup.gif" title="Move collection up"></a>""" % (weburl, colID, ln, i, rtype, tree[i][0], weburl)
else:
text += """ """
text += "</td><td>"
if down == 1 and edit:
text += """<a href="%s/admin/websearch/websearchadmin.py/modifycollectiontree?colID=%s&ln=%s&move_down=%s&rtype=%s#%s"><img border="0" src="%s/img/smalldown.gif" title="Move collection down"></a>""" % (weburl, colID, ln, i, rtype, tree[i][0], weburl)
else:
text += """ """
text += "</td><td>"
if edit:
if move_from and move_to:
tmove_from = move_from
move_from = ''
if not (move_from == "" and i == 0) and not (move_from != "" and int(move_from[1:len(move_from)]) == i and rtype == move_from[0]):
check = "true"
if move_from:
#if tree_from[move_from_id][0] == tree_to[i][0] or not check_col(tree_to[i][0], tree_from[move_from_id][0]):
# check = ''
#elif not check_col(tree_to[i][0], tree_from[move_from_id][0]):
# check = ''
#if not check and (tree_to[i][0] == 1 and tree_from[move_from_id][3] == tree_to[i][0] and move_from_rtype != rtype):
# check = "true"
if check:
text += """<a href="%s/admin/websearch/websearchadmin.py/modifycollectiontree?colID=%s&ln=%s&move_from=%s&move_to=%s%s&rtype=%s#tree"><img border="0" src="%s/img/move_to.gif" title="Move '%s' to '%s'"></a>
text += """<a href="%s/admin/websearch/websearchadmin.py/modifycollectiontree?colID=%s&ln=%s&move_from=%s%s&rtype=%s#%s"><img border="0" src="%s/img/move_from.gif" title="Move '%s' from this location."></a>""" % (weburl, colID, ln, rtype, i, rtype, tree[i][0], weburl, col_dict[tree[i][0]])
except KeyError:
pass
else:
text += """<img border="0" src="%s/img/white_field.gif">
""" % weburl
else:
text += """<img border="0" src="%s/img/white_field.gif">
""" % weburl
text += """
</td>
<td>"""
if edit:
try:
text += """<a href="%s/admin/websearch/websearchadmin.py/modifycollectiontree?colID=%s&ln=%s&delete=%s&rtype=%s#%s"><img border="0" src="%s/img/iconcross.gif" title="Remove colletion from tree"></a>""" % (weburl, colID, ln, i, rtype, tree[i][0], weburl)
except KeyError:
pass
elif i != 0:
text += """<img border="0" src="%s/img/white_field.gif">
""" % weburl
text += """</td><td>
"""
if tmove_from:
move_from = tmove_from
try:
text += """<a name="%s"></a>%s<a href="%s/admin/websearch/websearchadmin.py/editcollection?colID=%s&ln=%s" title="Edit collection">%s</a>%s%s%s""" % (tree[i][0], (reltype=="v" and '<i>' or ''), weburl, tree[i][0], ln, col_dict[id_son], (move_to=="%s%s" %(rtype,i) and ' <img border="0" src="%s/img/move_to.gif">' % weburl or ''), (move_from=="%s%s" % (rtype,i) and ' <img border="0" src="%s/img/move_from.gif">' % weburl or ''), (reltype=="v" and '</i>' or ''))
<dd>When deleting a collection, you also deletes all data related to the collection like translations, relations to other collections and information about which rank methods to use.
<br>For more information, please go to the <a title="See guide" href="%s/admin/websearch/guide.html">WebSearch guide</a> and read the section regarding deleting a collection.</dd>
</dl>
</strong>
</span>
""" % weburl
col_dict = dict(get_def_name('', "collection"))
if colID !=1 and colID and col_dict.has_key(int(colID)):
output = """<b><span class="info">Can not delete a collection that is a part of the collection tree, remove collection from the tree and try again.</span></b>"""
else:
subtitle = """4. Delete collection"""
output = """<b><span class="info">Not possible to delete the root collection</span></b>"""
if collection_table_update_time > collection_web_update_time:
output += """<br><b><span class="info">Warning: The collections has been modified since last time Webcoll was executed, to process the changes, Webcoll must be executed.</span></b><br>"""
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='webcoll' and runtime< now() ORDER by runtime")
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='bibindex' and runtime< now() ORDER by runtime")
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='webcoll' and runtime > now() ORDER by runtime")
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='bibindex' and runtime > now() ORDER by runtime")
output += """<br><b><span class="info">Warning: Webcoll is not scheduled for a future run by bibsched, any updates to the collection will not be processed.</span></b><br>"""
if bibindex_future == "":
output += """<br><b><span class="info">Warning: Bibindex is not scheduled for a future run by bibsched, any updates to the records will not be processed.</span></b><br>"""
tree = tree[0:ssize] + ntree + tree[ssize:len(tree)]
return tree
except StandardError, e:
return ()
def add_col_dad_son(add_dad, add_son, rtype):
"""Add a son to a collection (dad)
add_dad - add to this collection id
add_son - add this collection id
rtype - either regular or virtual"""
try:
res = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s ORDER BY score ASC" % add_dad)
highscore = 0
for score in res:
if int(score[0]) > highscore:
highscore = int(score[0])
highscore += 1
res = run_sql("INSERT INTO collection_collection(id_dad,id_son,score,type) values(%s,%s,%s,'%s')" % (add_dad, add_son, highscore, rtype))
return (1, highscore)
except StandardError, e:
return (0, e)
def compare_on_val(first, second):
"""Compare the two values"""
return cmp(first[1], second[1])
def get_col_fld(colID=-1, type = '', id_field=''):
"""Returns either all portalboxes associated with a collection, or based on either colID or language or both.
colID - collection id
ln - language id"""
sql = "SELECT id_field,id_fieldvalue,type,score,score_fieldvalue FROM collection_field_fieldvalue, field WHERE id_field=field.id"
try:
if colID > -1:
sql += " AND id_collection=%s" % colID
if id_field:
sql += " AND id_field=%s" % id_field
if type:
sql += " AND type='%s'" % type
sql += " ORDER BY type, score desc, score_fieldvalue desc"
res = run_sql(sql)
return res
except StandardError, e:
return ""
def get_col_pbx(colID=-1, ln='', position = ''):
"""Returns either all portalboxes associated with a collection, or based on either colID or language or both.
colID - collection id
ln - language id"""
sql = "SELECT id_portalbox, id_collection, ln, score, position, title, body FROM collection_portalbox, portalbox WHERE id_portalbox = portalbox.id"
try:
if colID > -1:
sql += " AND id_collection=%s" % colID
if ln:
sql += " AND ln='%s'" % ln
if position:
sql += " AND position='%s'" % position
sql += " ORDER BY position, ln, score desc"
res = run_sql(sql)
return res
except StandardError, e:
return ""
def get_col_fmt(colID=-1):
"""Returns all formats currently associated with a collection, or for one specific collection
colID - the id of the collection"""
try:
if colID not in [-1, "-1"]:
res = run_sql("SELECT id_format, id_collection, code, score FROM collection_format, format WHERE id_format = format.id AND id_collection=%s ORDER BY score desc" % colID)
else:
res = run_sql("SELECT id_format, id_collection, code, score FROM collection_format, format WHERE id_format = format.id ORDER BY score desc")
return res
except StandardError, e:
return ""
def get_col_rnk(colID, ln):
""" Returns a list of the rank methods the given collection is attached to
colID - id from collection"""
try:
res1 = dict(run_sql("SELECT id_rnkMETHOD, '' FROM collection_rnkMETHOD WHERE id_collection=%s" % colID))
res2 = get_def_name('', "rnkMETHOD")
result = filter(lambda x: res1.has_key(x[0]), res2)
return result
except StandardError, e:
return ()
def get_pbx():
"""Returns all portalboxes"""
try:
res = run_sql("SELECT id, title, body FROM portalbox ORDER by title,body")
return res
except StandardError, e:
return ""
def get_fld_value(fldvID = ''):
"""Returns fieldvalue"""
try:
sql = "SELECT id, name, value FROM fieldvalue"
if fldvID:
sql += " WHERE id=%s" % fldvID
sql += " ORDER BY name"
res = run_sql(sql)
return res
except StandardError, e:
return ""
def get_pbx_pos():
"""Returns a list of all the positions for a portalbox"""
position = {}
position["rt"] = "Right Top"
position["lt"] = "Left Top"
position["te"] = "Title Epilog"
position["tp"] = "Title Prolog"
position["ne"] = "Narrow by coll epilog"
position["np"] = "Narrow by coll prolog"
return position
def get_sort_nametypes():
"""Return a list of the various translationnames for the fields"""
type = {}
type['soo'] = 'Sort options'
type['seo'] = 'Search options'
type['sew'] = 'Search within'
return type
def get_fmt_nametypes():
"""Return a list of the various translationnames for the output formats"""
type = []
type.append(('ln', 'Long name'))
return type
def get_fld_nametypes():
"""Return a list of the various translationnames for the fields"""
type = []
type.append(('ln', 'Long name'))
return type
def get_col_nametypes():
"""Return a list of the various translationnames for the collections"""
type = []
type.append(('ln', 'Long name'))
return type
def find_last(tree, start_son):
"""Find the previous collection in the tree with the same father as start_son"""
id_dad = tree[start_son][3]
while start_son > 0:
start_son -= 1
if tree[start_son][3] == id_dad:
return start_son
def find_next(tree, start_son):
"""Find the next collection in the tree with the same father as start_son"""
id_dad = tree[start_son][3]
while start_son < len(tree):
start_son += 1
if tree[start_son][3] == id_dad:
return start_son
def remove_col_subcol(id_son, id_dad, type):
"""Remove a collection as a son of another collection in the tree, if collection isn't used elsewhere in the tree, remove all registered sons of the id_son.
id_son - collection id of son to remove
id_dad - the id of the dad"""
try:
if id_son != id_dad:
tree = get_col_tree(id_son)
res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s" % (id_son, id_dad))
else:
tree = get_col_tree(id_son, type)
res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s and type='%s'" % (id_son, id_dad, type))
if not run_sql("SELECT * from collection_collection WHERE id_son=%s and type='%s'" % (id_son, type)):
for (id, up, down, dad, rtype) in tree:
res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s" % (id, dad))
return (1, "")
except StandardError, e:
return (0, e)
def check_col(add_dad, add_son):
"""Check if the collection can be placed as a son of the dad without causing loops.
add_dad - collection id
add_son - collection id"""
try:
stack = [add_dad]
res = run_sql("SELECT id_dad FROM collection_collection WHERE id_dad=%s AND id_son=%s" % (add_dad,add_son))
if res:
raise StandardError
while len(stack) > 0:
colID = stack.pop()
res = run_sql("SELECT id_dad FROM collection_collection WHERE id_son=%s" % colID)
for id in res:
if int(id[0]) == int(add_son):
raise StandardError
else:
stack.append(id[0])
return (1, "")
except StandardError, e:
return (0, e)
def attach_rnk_col(colID, rnkID):
"""attach rank method to collection
rnkID - id from rnkMETHOD table
colID - id of collection, as in collection table """
try:
res = run_sql("INSERT INTO collection_rnkMETHOD(id_collection, id_rnkMETHOD) values (%s,%s)" % (colID, rnkID))
return (1, "")
except StandardError, e:
return (0, e)
def detach_rnk_col(colID, rnkID):
"""detach rank method from collection
rnkID - id from rnkMETHOD table
colID - id of collection, as in collection table """
try:
res = run_sql("DELETE FROM collection_rnkMETHOD WHERE id_collection=%s AND id_rnkMETHOD=%s" % (colID, rnkID))
return (1, "")
except StandardError, e:
return (0, e)
def switch_col_treescore(col_1, col_2):
try:
res1 = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s and id_son=%s" % (col_1[3], col_1[0]))
res2 = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s and id_son=%s" % (col_2[3], col_2[0]))
res = run_sql("UPDATE collection_collection SET score=%s WHERE id_dad=%s and id_son=%s" % (res2[0][0], col_1[3], col_1[0]))
res = run_sql("UPDATE collection_collection SET score=%s WHERE id_dad=%s and id_son=%s" % (res1[0][0], col_2[3], col_2[0]))
score - decides which portalbox is the most important
position - position on page the portalbox should appear."""
try:
if score:
res = run_sql("INSERT INTO collection_portalbox(id_portalbox, id_collection, ln, score, position) values (%s,%s,'%s',%s,'%s')" % (pbxID, colID, ln, score, position))
else:
res = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and ln='%s' and position='%s' ORDER BY score desc, ln, position" % (colID, ln, position))
if res:
score = int(res[0][0])
else:
score = 0
res = run_sql("INSERT INTO collection_portalbox(id_portalbox, id_collection, ln, score, position) values (%s,%s,'%s',%s,'%s')" % (pbxID, colID, ln, (score + 1), position))
return (1, "")
except StandardError, e:
return (0, e)
def add_col_fmt(colID, fmtID, score=''):
"""Add a output format to the collection.
colID - the id of the collection involved
fmtID - the id of the format.
score - the score of the format, decides sorting, if not given, place the format on top"""
try:
if score:
res = run_sql("INSERT INTO collection_format(id_format, id_collection, score) values (%s,%s,%s)" % (fmtID, colID, score))
else:
res = run_sql("SELECT score FROM collection_format WHERE id_collection=%s ORDER BY score desc" % colID)
if res:
score = int(res[0][0])
else:
score = 0
res = run_sql("INSERT INTO collection_format(id_format, id_collection, score) values (%s,%s,%s)" % (fmtID, colID, (score + 1)))
return (1, "")
except StandardError, e:
return (0, e)
def add_col_fld(colID, fldID, type, fldvID=''):
"""Add a sort/search/field to the collection.
colID - the id of the collection involved
fldID - the id of the field.
fldvID - the id of the fieldvalue.
type - which type, seo, sew...
score - the score of the format, decides sorting, if not given, place the format on top"""
try:
if fldvID and fldvID not in [-1, "-1"]:
run_sql("DELETE FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type='%s' and id_fieldvalue is NULL" % (colID, fldID, type))
res = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type='%s' ORDER BY score desc" % (colID, fldID, type))
if res:
score = int(res[0][0])
res = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type='%s' ORDER BY score_fieldvalue desc" % (colID, fldID, type))
else:
res = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and type='%s' ORDER BY score desc" % (colID, type))
if res:
score = int(res[0][0]) + 1
else:
score = 1
res = run_sql("SELECT * FROM collection_field_fieldvalue where id_field=%s and id_collection=%s and type='%s' and id_fieldvalue=%s" % (fldID, colID, type, fldvID))
if not res:
run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=score_fieldvalue+1 WHERE id_field=%s AND id_collection=%s and type='%s'" % (fldID, colID, type))
res = run_sql("INSERT INTO collection_field_fieldvalue(id_field, id_fieldvalue, id_collection, type, score, score_fieldvalue) values (%s,%s,%s,'%s',%s,%s)" % (fldID, fldvID, colID, type, score, 1))
else:
return (0, (1, "Already exists"))
else:
res = run_sql("SELECT * FROM collection_field_fieldvalue WHERE id_collection=%s AND type='%s' and id_field=%s and id_fieldvalue is NULL" % (colID, type, fldID))
if res:
return (0, (1, "Already exists"))
else:
run_sql("UPDATE collection_field_fieldvalue SET score=score+1")
res = run_sql("INSERT INTO collection_field_fieldvalue(id_field, id_collection, type, score,score_fieldvalue) values (%s,%s,'%s',%s, 0)" % (fldID, colID, type, 1))
return (1, "")
except StandardError, e:
return (0, e)
def modify_restricted(colID, rest):
"""Modify which apache group is allowed to use the collection.
pbxID - the id of the portalbox that should be modified
sel_ln - the language of the portalbox that should be modified
title - the title
body - the content
score - if several portalboxes in one position, who should appear on top.
position - position on page"""
try:
if title:
res = run_sql("UPDATE portalbox SET title='%s' WHERE id=%s" % (MySQLdb.escape_string(title), pbxID))
if body:
res = run_sql("UPDATE portalbox SET body='%s' WHERE id=%s" % (MySQLdb.escape_string(body), pbxID))
if score:
res = run_sql("UPDATE collection_portalbox SET score='%s' WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (score, colID, pbxID, sel_ln))
if position:
res = run_sql("UPDATE collection_portalbox SET position='%s' WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (position, colID, pbxID, sel_ln))
return (1, "")
except Exception, e:
return (0, e)
def switch_fld_score(colID, id_1, id_2):
"""Switch the scores of id_1 and id_2 in collection_field_fieldvalue
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s" % (colID, id_1))
res2 = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s" % (colID, id_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
else:
res = run_sql("UPDATE collection_field_fieldvalue SET score=%s WHERE id_collection=%s and id_field=%s" % (res2[0][0], colID, id_1))
res = run_sql("UPDATE collection_field_fieldvalue SET score=%s WHERE id_collection=%s and id_field=%s" % (res1[0][0], colID, id_2))
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s" % (colID, id_1, fldvID_1))
res2 = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s" % (colID, id_1, fldvID_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
else:
res = run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s" % (res2[0][0], colID, id_1, fldvID_1))
res = run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s" % (res1[0][0], colID, id_1, fldvID_2))
return (1, "")
except Exception, e:
return (0, e)
def switch_pbx_score(colID, id_1, id_2, sel_ln):
"""Switch the scores of id_1 and id_2 in the table given by the argument.
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (colID, id_1, sel_ln))
res2 = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (colID, id_2, sel_ln))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
res = run_sql("UPDATE collection_portalbox SET score=%s WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (res2[0][0], colID, id_1, sel_ln))
res = run_sql("UPDATE collection_portalbox SET score=%s WHERE id_collection=%s and id_portalbox=%s and ln='%s'" % (res1[0][0], colID, id_2, sel_ln))
return (1, "")
except Exception, e:
return (0, e)
def switch_score(colID, id_1, id_2, table):
"""Switch the scores of id_1 and id_2 in the table given by the argument.
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_%s WHERE id_collection=%s and id_%s=%s" % (table, colID, table, id_1))
res2 = run_sql("SELECT score FROM collection_%s WHERE id_collection=%s and id_%s=%s" % (table, colID, table, id_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
res = run_sql("UPDATE collection_%s SET score=%s WHERE id_collection=%s and id_%s=%s" % (table, res2[0][0], colID, table, id_1))
res = run_sql("UPDATE collection_%s SET score=%s WHERE id_collection=%s and id_%s=%s" % (table, res1[0][0], colID, table, id_2))
if verbose >= 9: print """ SELECT DISTINCT uq.id_user\n FROM user_query AS uq LEFT JOIN user AS u\n ON uq.id_user = u.id\n WHERE u.id IS NULL"""
result = run_sql("""SELECT DISTINCT uq.id_user
FROM user_query AS uq LEFT JOIN user AS u
ON uq.id_user = u.id
WHERE u.id IS NULL""")
if verbose >= 9: print result
# delete in user_query one by one
if verbose >= 9: print """ DELETE FROM user_query WHERE id_user = 'TRAVERSE LAST RESULT' \n"""
for (id_user, ) in result:
delcount['user_query'] += run_sql("""DELETE FROM user_query WHERE id_user = %s""" % (id_user, ))
# delete the actual queries
if verbose: print "- deleting queries not attached to any user"
# select queries that must be deleted
if verbose >= 9: print """ SELECT DISTINCT q.id\n FROM query AS q LEFT JOIN user_query AS uq\n ON uq.id_query = q.id\n WHERE uq.id_query IS NULL AND\n q.type <> 'p' """
result = run_sql("""SELECT DISTINCT q.id
FROM query AS q LEFT JOIN user_query AS uq
ON uq.id_query = q.id
WHERE uq.id_query IS NULL AND
q.type <> 'p'""")
if verbose >= 9: print result
# delete queries one by one
if verbose >= 9: print """ DELETE FROM query WHERE id = 'TRAVERSE LAST RESULT \n"""
for (id_user, ) in result:
delcount['query'] += run_sql("""DELETE FROM query WHERE id = %s""" % (id_user, ))
# 3 - DELETE BASKETS NOT OWNED BY ANY USER
if verbose: print "- deleting baskets not owned by any user"
# select basket ids
if verbose >= 9: print """ SELECT ub.id_basket\n FROM user_basket AS ub LEFT JOIN user AS u\n ON u.id = ub.id_user\n WHERE u.id IS NULL"""
result = run_sql("""SELECT ub.id_basket
FROM user_basket AS ub LEFT JOIN user AS u
ON u.id = ub.id_user
WHERE u.id IS NULL""")
if verbose >= 9: print result
# delete from user_basket and basket one by one
if verbose >= 9:
print """ DELETE FROM user_basket WHERE id_basket = 'TRAVERSE LAST RESULT' """
print """ DELETE FROM basket WHERE id = 'TRAVERSE LAST RESULT' \n"""
for (id_basket, ) in result:
delcount['user_basket'] += run_sql("""DELETE FROM user_basket WHERE id_basket = %s""" % (id_basket, ))
delcount['basket'] += run_sql("""DELETE FROM basket WHERE id = %s""" % (id_basket, ))
# 4 - DELETE ALERTS NOT OWNED BY ANY USER
if verbose: print '- deleting alerts not owned by any user'
# select user ids in uqb that reference non-existent users
if verbose >= 9: print """SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL"""
result = run_sql("""SELECT DISTINCT uqb.id_user FROM user_query_basket AS uqb LEFT JOIN user AS u ON uqb.id_user = u.id WHERE u.id IS NULL""")
if verbose >= 9: print result
# delete all these entries
for (id_user, ) in result:
if verbose >= 9: print """DELETE FROM user_query_basket WHERE id_user = 'TRAVERSE LAST RESULT """
delcount['user_query_basket'] += run_sql("""DELETE FROM user_query_basket WHERE id_user = %s """ % (id_user, ))
Defines the text that will be displayed on the 'lost password' page
Parameters:
- 'ln' *string* - The language to display the interface in
- 'supportemail' *string* - The email of the support team
"""
# load the right message language
_ = gettext_set_language(ln)
return _("If you have lost password for your CERN Document Server internal account, then please enter your email address below and the lost password will be emailed to you.") +\
"<br /><br />" +\
_("Note that if you have been using an external login system (such as CERN NICE), then we cannot do anything and you have to ask there.") +\
_("Alternatively, you can ask %s to change your login system from external to internal.") % ("""<a href="mailto:%(email)s">%(email)s</a>""" % { 'email' : supportemail }) +\
"<br><br>"
def tmpl_back_form(self, ln, message, act, link):
"""
A standard one-message-go-back-link page.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'message' *string* - The message to display
- 'act' *string* - The action to accomplish when going back
- 'ln' *string* - The language to display the interface in
- 'uid' *string* - The user id
- 'guest' *boolean* - If the user is guest
- 'cfg_cern_site' *boolean* - If the site is a CERN site
"""
# load the right message language
_ = gettext_set_language(ln)
out = """<P>%(account_offer)s</P>
<blockquote>
<dl>
""" % {
'account_offer' : _("The CDS Search offers you a possibility to personalize the interface, to set up your own personal library of documents, or to set up an automatic alert query that would run periodically and would notify you of search results by email."),
}
if not guest:
out += """
<dt>
<A href="./edit?ln=%(ln)s">%(your_settings)s</A>
<dd>%(change_account)s""" % {
'ln' : ln,
'your_settings' : _("Your Settings"),
'change_account' : _("Set or change your account Email address or password. Specify your preferences about the way the interface looks like.")
'search_explain' : _("View all the searches you performed during the last 30 days."),
'your_baskets' : _("Your Baskets"),
'basket_explain' : _("With baskets you can define specific collections of items, store interesting records you want to access later or share with others."),
}
if guest:
out += self.tmpl_warning_guest_user(ln = ln, type = "baskets")
'explain_alerts' : _("Subscribe to a search which will be run periodically by our service. The result can be sent to you via Email or stored in one of your baskets."),
}
if guest:
out += self.tmpl_warning_guest_user(type="alerts", ln = ln)
'explain_loans' : _("Check out book you have on load, submit borrowing requests, etc. Requires CERN ID."),
}
out += """
</dl>
</blockquote>"""
return out
def tmpl_warning_guest_user(self, ln, type):
"""
Displays a warning message about the specified type
Parameters:
- 'ln' *string* - The language to display the interface in
- 'type' *string* - The type of data that will get lost in case of guest account
"""
# load the right message language
_ = gettext_set_language(ln)
msg= _("""You are logged in as a guest user, so your %s will disappear at the end of the current session. If you wish you can
<a href="../youraccount.py/login?ln=%s">login or register here</a>.""") % (type, ln)
return """<table class="errorbox" summary="">
<thead>
<tr>
<th class="errorboxheader">%s</th>
</tr>
</thead>
</table>""" % msg
def tmpl_account_body(self, ln, user):
"""
Displays the body of the actions of the user
Parameters:
- 'ln' *string* - The language to display the interface in
- 'user' *string* - The user name
"""
# load the right message language
_ = gettext_set_language(ln)
return _("""You are logged in as %s. You may want to a) <A href="../youraccount.py/logout?ln=%s">logout</A>; b) edit your <A href="../youraccount.py/edit?ln=%s">account settings</a>.""") % (user, ln, ln) + "<BR><BR>"
def tmpl_account_template(self, title, body, ln):
"""
Displays a block of the your account page
Parameters:
- 'ln' *string* - The language to display the interface in
'password_contain' : _("The password phrase may contain punctuation, spaces, etc."),
'retype' : _("Retype Password"),
'register' : _("register"),
'explain_acc' : _("Please do not use valuable passwords such as your Unix, AFS or NICE passwords with this service. Your email address will stay strictly confidential and will not be disclosed to any third party. It will be used to identify you for personal services of %s. For example, you may set up an automatic alert search that will look for new preprints and will notify you daily of new arrivals by email.") % cdsname,
"""Send an email to the address given by sendTo about the new account newAccountEmail."""
fromaddr = "From: %s" % supportemail
toaddrs = "To: %s" % sendTo
to = toaddrs + "\n"
sub = "Subject: Your account created on '%s'\n\n" % cdsname
body = "You have created a new account on '%s':\n\n" % cdsname
body += " Username/Email: %s\n" % newAccountEmail
body += " Password: %s\n\n" % ("*" * len(password))
if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS >= 1:
body += "This account is awaiting approval by the site administrators and therefore cannot be used as of yet.\nYou will receive an email notification as soon as your account request has been processed.\n"
body += "\n---------------------------------"
body += "\n%s" % cdsname
body += "\nContact: %s" % supportemail
msg = to + sub + body
server = smtplib.SMTP('localhost')
server.set_debuglevel(1)
try:
server.sendmail(fromaddr, toaddrs, msg)
except smtplib.SMTPRecipientsRefused,e:
return 0
server.quit()
return 1
def get_email(uid):
"""Return email address of the user uid. Return string 'guest' in case
the user is not found."""
out = "guest"
res = run_sql("SELECT email FROM user WHERE id=%s", (uid,), 1)
if res and res[0][0]:
out = res[0][0]
return out
def create_userinfobox_body(uid, language="en"):
"""Create user info box body for user UID in language LANGUAGE."""
out = ""
return tmpl.tmpl_create_userinfobox(
ln = language,
weburl = weburl,
guest = isGuestUser(uid),
email = get_email(uid),
submitter = isUserSubmitter(uid),
referee = isUserReferee(uid),
admin = isUserAdmin(uid),
)
def list_registered_users():
"""List all registered users."""
return run_sql("SELECT id,email FROM user where email!=''")
def list_users_in_role(role):
"""List all users of a given role (see table accROLE)
@param role: role of user (string)
@return list of uids
"""
query = """SELECT uacc.id_user
FROM user_accROLE uacc JOIN accROLE acc
ON uacc.id_accROLE=acc.id
WHERE acc.name='%s'"""
res = run_sql(query% MySQLdb.escape_string(role))
if res:
return map(lambda x: int(x[0]), res)
return []
def list_users_in_roles(role_list):
"""List all users of given roles (see table accROLE)
@param role_list: list of roles [string]
@return list of uids
"""
if not(type(role_list) is list or type(role_list) is tuple):
role_list = [role_list]
params=''
query = """SELECT distinct(uacc.id_user)
FROM user_accROLE uacc JOIN accROLE acc
ON uacc.id_accROLE=acc.id
%s"""
if len(role_list) > 0:
params = 'WHERE '
for role in role_list[:-1]:
params += "acc.name='%s' OR " % MySQLdb.escape_string(role)
function authenticate($email,$rule,$doctype="*",$action="*")
{
global $ADMINEMAIL;
if (eregi($ADMINEMAIL,"$email"))
return true;
$res = mysql_query("select id from rules where name='superuser'");
$row = mysql_fetch_row($res);
$id_superuser = $row[0];
$res = mysql_query("select id_user from user_rule where id_rule='${id_superuser}' and (param1='$doctype' or param1='*') and (param2 LIKE '$action' or param2='*')");
while ($row = mysql_fetch_row($res))
{
$iduser = $row[0];
$emailuser = getEmail($iduser);
if (eregi("$email","$emailuser"))
return true;
}
$res = mysql_query("select id from rules where name='$rule'");
if (mysql_num_rows($res) == 0)
return false;
else
{
$row = mysql_fetch_row($res);
$idrule = $row[0];
$res = mysql_query("select id_user from user_rule where id_rule='$idrule' and (param1 LIKE '$doctype' or param1='*') and (param2 LIKE '$action' or param2='*')");
while ($row = mysql_fetch_row($res))
{
$iduser = $row[0];
$emailuser = getEmail($iduser);
if (eregi("$email","$emailuser"))
return true;
}
return false;
}
}
function getRuleID($rule)
{
////////////////////////////
// get the id of the rule //
////////////////////////////
$res = mysql_query("
SELECT id
FROM rules
WHERE name='$rule'");
if (mysql_num_rows($res) == 0)
{
// if it does not exist, attempt to create it
$res = mysql_query("
INSERT
INTO rules (name,description)
VALUES ('$rule','')");
$idrule = mysql_insert_id();
}
else
{
$row = mysql_fetch_row($res);
$idrule = $row[0];
}
return $idrule;
}
function mysql_perform_query($query, $link_identifier, $behaviour="die") {
## Function to call as an alternative to mysql_query. The function
## stops the execution if the query couldn't be executed and
## prints an error message (HTML formatted) (default behaviour). If
## behaviour is set to 'continue', then the function just goes on.
if($behaviour == "continue")
$result = mysql_query($query, $link_identifier);
else {
$result = mysql_query($query, $link_identifier)
or die ("<p>MySQL: could not execute your query<br>$query" .
"<br>Contact the <a href=\"mailto:search.support@cds.cern.ch\">" .
if (CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS >= 2 or (CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS <= 1 and webuser.checkemail(email))) and uid2 != -1 and (uid2 == uid or uid2 == 0) and password == password2:
if CFG_EXTERNAL_AUTHENTICATION.has_key(user_prefs['login_method']) or CFG_EXTERNAL_AUTHENTICATION.has_key(user_prefs['login_method']) and CFG_EXTERNAL_AUTHENTICATION[user_prefs['login_method']][0] != None:
<span class="guideheader">T</span>o add a document type to WebSubmit, you should go to the <a target=top href="<WEBURL>/admin/websubmit/index.php">main page</a>
and click on "New Doctype" in the left blue panel.<br><br>
<span class="guideheader">E</span>ven once created, a document type will not appear automatically on this page. To configure the list of catalogues and document
types displayed on this page, the administrator shall go to the <a target=top href="<WEBURL>/admin/websubmit/editCatalogues.php">edit catalogues</a>
page. (see the <a href=catalogues.<lang:star: *>.html>guide section</a>)<br>
<h3>The user can then click on the document type he is interested in.</h3>
<IMG src="<WEBURL>/img/sbm_admin_guide_menudoc.png" alt="Document type Page" class="guideimg" align="left">
<span class="guideheader">T</span>he text appearing under the header containing the name of the document
can be configured by going to the <a target=top href="<WEBURL>/admin/websubmit/">main page</a>, click on
the title of the document type then on the "Edit Document Types Details" button.<br><br>
<span class="guideheader">Y</span>ou can associate several categories to a document type which can be defined by going to the
<a target=top href="<WEBURL>/admin/websubmit/">main page</a>, click on the title of the document type
then on the "View Categories" button. The selected category will be saved in a file named "comboXXX"
(where XXX is the short name of the document type) in the submission directory.<br><br>
<span class="guideheader">T</span>o add an action button to this page, first implement this action by going to the
<a target=top href="<WEBURL>/admin/websubmit/">main page</a>, click on the title of the document type then
on the "Add a new submission" button. If the action is already implemented and the button still does not appear
on the submision page, then you should edit the details of this implementation: go to the
<a target=top href="<WEBURL>/admin/websubmit">main page</a>, click on the title of the document type then
on the icon in the "Edit Submission" column and in the line of the desired action. There you should set the
"Displayed" form field to "YES".<br><br>
<span class="guideheader">Y</span>ou can also change the order of the buttons, by going to the <a target=top href="<WEBURL>/admin/websubmit/">
main page</a>, click on the title of the document type then on the icon in the "Edit Submission" column and in the
line of the desired action. There you can set the "buttonorder" form field.<br><br>
<h3>The user now may choose a category, then click on the action button he wishes.<br>The submission starts, the first page of the web form appears.</h3>
<IMG src="<WEBURL>/img/sbm_admin_guide_form.png" alt="Document type Page" class="guideimg" align="left">
<span class="guideheader">T</span>his web form is composed of several pages, on each of these
pages form fields can be found. To modify the number of pages, add or withdraw form fields and modify
the texts before each form field, you shall go to the <a target=top href="<WEBURL>/admin/websubmit">main page</a>,
click on the title of the document type then on the icon in the "Edit Submission Pages" column and in the line of the
desired action. (see the <a href=actionimplement.<lang:star: *>.html>guide section</a>)<br><br>
<h3>On the last page of the submission, there should be a button like in the following image which will
trigger the end script</h3>
<IMG src="<WEBURL>/img/sbm_admin_guide_endaction.png" alt="Document type End Page" class="guideimg" align="left">
<span class="guideheader">T</span>his button is defined like any other form field. Its definition should include
a <i> onclick="finish();"</i> javascript attribute.<br><br>
<span class="guideheader">A</span>fter clicking this button, WebSubmit will apply the end script functions
to the gathered data. To modify the end script, you shall go to the <a target=top href="<WEBURL>/admin/websubmit/">
main page</a>, click on the title of the document type then on the icon in the "Edit Functions" column and in the line
of the desired action. (see the <a href="implementfunctions.<lang:star: *>.html">guide section</a>)<br>
This value depends on the web form configuration you did. It should contain the name of the form element used for storing the reference of the document.
If the authentication module (login) is active in webSubmit, this function compares the current login with the email of the original submitter. If it is the same (or if the current user has superuser rights), we go on. If it differs, an error message is issued.
email addresses of the people who will receive this email (comma separated list). this parameter may contain the <b><CATEG></b> string. In which case the variable computed from the [categformatDAM] parameter replaces this string.<br>
contains a regular expression used to compute the category of the document given the reference of the document.<br>
eg.: if [categformatAFP]="TEST-<CATEG>-.*" and the reference of the document is "TEST-CATEGORY1-2001-001", then the computed category equals "CATEGORY1"
email addresses of the people who will receive this email (comma separated list). this parameter may contain the <b><CATEG></b> string. In which case the variable computed from the [categformatDAM] parameter replaces this string.<br>
<span class="guideheader">T</span>his action allows you, once you have submitted a paper and its
associated files, to submit a revision of the document or to add a new format of the document.<BR>
<span class="guideheader">T</span>he prefered format for the files are PostScript or PDF.<BR>
<span class="guideheader">T</span>he old version will still appear online with a version number appended to its name.<BR>
<span class="guideheader">F</span>inally, an email is sent to the original author to notify the change.</SMALL>
</BLOCKQUOTE>
<h3>Delay:</h3>
<BLOCKQUOTE>
<span class="guideheader">T</span>he revised version should appear online right after the end of its submission. If you still do not find the revised version online, please <A HREF="mailto:support@documents.cern.ch">contact us</A></SMALL>
</BLOCKQUOTE>
<h3>See also:</h3>
<BLOCKQUOTE><SMALL>
<li><A HREF="modification.<lang:star: *>.html">modification of bibliographic information</A><BR>
<li><A HREF="file_transfer.<lang:star: *>.html">notes on file transfers</A><BR>
res = run_sql("select ln.id_bibdoc2,ln.type from bibdoc_bibdoc as ln,bibdoc where id=ln.id_bibdoc2 and ln.id_bibdoc1=%s and status!='deleted'",(self.id,))
"""Create an interface for the modification of a document, based on the fields that the user has
chosen to modify
"""
global sysno,rn
t=""
# variables declaration
fieldname = parameters['fieldnameMBI']
# Path of file containing fields to modify
if os.path.exists("%s/%s" % (curdir,fieldname)):
fp = open( "%s/%s" % (curdir,fieldname), "r" )
fieldstext = fp.read()
fp.close()
else:
raise functionError("cannot find fields to modify")
fieldstext = re.sub("\+","\n",fieldstext)
fields = fieldstext.split("\n")
#output some text
t=t+"<CENTER bgcolor=\"white\">The document <B>%s</B> has been found in the database.</CENTER><BR>Please modify the following fields:<BR>Then press the 'END' button at the bottom of the page<BR>\n" % rn
for field in fields:
subfield = ""
value = ""
marccode = ""
text = ""
# retrieve and display the modification text
t=t+"<FONT color=\"darkblue\">\n"
res = run_sql("SELECT modifytext FROM sbmFIELDDESC WHERE name=%s", (field,))
if len(res)>0:
t=t+"<small>%s</small> </FONT>\n" % res[0][0]
# retrieve the marc code associated with the field
res = run_sql("SELECT marccode FROM sbmFIELDDESC WHERE name=%s", (field,))
if len(res) > 0:
marccode = res[0][0]
# then retrieve the previous value of the field
if os.path.exists("%s/%s" % (curdir,"Create_Modify_Interface_DONE")):
# Page has been reloaded - get field value from text file on server, not from DB record
value = Create_Modify_Interface_getfieldval_fromfile(curdir,field)
else:
# First call to page - get field value from DB record
value = Create_Modify_Interface_getfieldval_fromDBrec(marccode, sysno)
# If field is a date value, transform date into format DD/MM/YYYY:
value = Create_Modify_Interface_transform_date(value)
res = run_sql("SELECT * FROM sbmFIELDDESC WHERE name=%s", (field,))
raise functionStop("<SCRIPT>document.forms[0].action=\"submit.py\";document.forms[0].curpage.value=1;document.forms[0].step.value=0;document.forms[0].submit();alert('The report %s cannot be found in our database.\\nPerhaps it has not been integrated yet?\\nAnyway, you can choose another report number if you wish.\\n Or retry this action in a few minutes.');</SCRIPT>" % rn)
elif len(searchresults) > 1:
raise functionStop("<SCRIPT>document.forms[0].action=\"submit.py\";document.forms[0].curpage.value=1;document.forms[0].step.value=0;document.forms[0].submit();alert('Multiple documents have been found with report number %s\\nYou can try with another report number if you wish.\\n Or retry this action in a few minutes.');</SCRIPT>" % rn)
if re.search(uid_email,email,re.IGNORECASE) == None and auth_code != 0:
raise functionStop("""
<SCRIPT>
document.forms[0].action="submit.py";
document.forms[0].curpage.value = 1;
document.forms[0].step.value = 0;
document.forms[0].submit();
alert('Only the submitter of this document has the right to do this action. \\nYour login (%s) is different from the one of the submitter (%s).');
</SCRIPT>""" % (uid_email,email))
elif re.search(uid_email,email,re.IGNORECASE) == None and auth_code == 0:
return ("""
<SCRIPT>
alert('Only the submitter of this document has the right to do this action. \\nYour login (%s) is different from the one of the submitter (%s).\\n\\nAnyway, as you have a special authorization for this type of documents,\\nyou are allowed to proceed! Watch out your actions!');
email_txt = "The document %s\nTitle: %s\nAuthor(s): %s\n\nhas been correctly received\n\n" % (fullrn,m_title,m_author)
# The user is either informed that the document has been added to the database, or sent for approval
if parameters['status'] == "APPROVAL":
email_txt = email_txt + "An email has been sent to the referee. You will be warned by email as soon as the referee takes his/her decision regarding your document.\n\n"
elif parameters['status'] == "ADDED":
email_txt = email_txt + "It will be soon added to our Document Server.\n\nOnce inserted, you will be able to check the bibliographic information and the quality of the electronic documents at this URL:\n<%s/search.py?recid=%s>\nIf you detect an error please let us know by sending an email to %s. \n\n" % (htdocsurl,sysno,supportemail)
email_txt = email_txt + "Thank you for using %s Submission Interface.\n" % cdsname
# send the mail
tostring = m_recipient.strip()
if cfg_websubmit_copy_mails_to_admin:
# Copy mail to admins:
if len(tostring) > 0:
tostring += ",%s" % (adminemail,)
else:
tostring = adminemail
body = forge_email(FROMADDR,m_recipient,"","%s: Document Received" % fullrn,email_txt)
t=t+Request_Print("A", "Your document has the following reference(s): <b>%s%s</b><br><br>" % (rn,additional_rn))
if status == "APPROVAL":
t=t+Request_Print("A", "An email has been sent to the referee. You will be warned by email as soon as the referee takes his/her decision regarding your document.<br><br>\n")
if status == "ADDED":
t=t+Request_Print("A", "It will soon appear on our server.<br><br>\n")
t=t+Request_Print("A", "Thank you for using %s!" % cdsname)
email_txt = "Dear Sir or Madam, \n%s %s has just been modified.\nModified fields: %s\n\n" % (type,rn,fields)
if accessurl != "" and sysno != "":
email_txt += "You can check the modified document here:\n"
email_txt += "<%s?id=%s>\n\n" % (accessurl,sysno)
email_txt += "Please note that the modifications will be taken into account in a couple of minutes.\n\nBest regards,\nThe %s Server support Team" % cdsname
# send the mail
tostring = sub.strip()
if len(addresses) > 0:
if len(tostring) > 0:
tostring += ",%s" % (addresses,)
else:
tostring = addresses
if cfg_websubmit_copy_mails_to_admin:
# Copy mail to admins:
if len(tostring) > 0:
tostring += ",%s" % (adminemail,)
else:
tostring = adminemail
body = forge_email(FROMADDR,sub,"","%s modified" % rn,email_txt)
message = "A revised version of document %s has been submitted.\n\nTitle: %s\nAuthor(s): %s\nURL: <%s?id=%s>%s" % (rn,title,author,accessurl,sysno,note)
# send the email
tostring = SuE.strip()
if len(addresses) > 0:
if len(tostring) > 0:
tostring += ",%s" % (addresses,)
else:
tostring = addresses
if cfg_websubmit_copy_mails_to_admin:
# Copy mail to admins:
if len(tostring) > 0:
tostring += ",%s" % (adminemail,)
else:
tostring = adminemail
body = forge_email(FROMADDR,SuE,"","%s revised" % rn,message)
# If we are in the last step of an action, we can update the "journal of submissions"
if last_step == 1:
if uid_email != "" and uid_email != "guest" and rn != "":
res = run_sql("SELECT * FROM sbmSUBMISSIONS WHERE doctype=%s and action=%s and id=%s and email=%s", (doctype,act,access,uid_email,))
if len(res) == 0:
run_sql("INSERT INTO sbmSUBMISSIONS values(%s,%s,%s,'finished',%s,%s,NOW(),NOW())", (uid_email,doctype,act,access,rn,))
else:
run_sql("UPDATE sbmSUBMISSIONS SET md=NOW(),reference=%s,status='finished' WHERE doctype=%s and action=%s and id=%s and email=%s", (rn,doctype,act,access,uid_email,))
res2 = run_sql("SELECT * FROM sbmIMPLEMENT LEFT JOIN sbmACTION on sbmACTION.sactname=sbmIMPLEMENT.actname WHERE docname=%s and displayed='Y' ORDER BY sbmIMPLEMENT.buttonorder", (docShortDesc,))
for arr2 in res2:
res = run_sql("SELECT * FROM sbmACTION WHERE sactname=%s", (arr2[1],))
for arr in res:
actionShortDesc.append(arr[1])
indir.append(arr[2])
actionbutton.append(arr[5])
statustext.append(arr[6])
t = websubmit_templates.tmpl_action_page(
ln = ln,
guest = (uid_email == "" or uid_email == "guest"),
res = run_sql("SELECT * FROM sbmIMPLEMENT WHERE docname=%s and level!='0' and level=%s and score>%s ORDER BY score", (doctype,currentlevel,action_score,))
if len(res) > 0:
actions = []
first_score = res[0][10]
for i in range(0,len(res)):
action = res[i]
if action[10] == first_score:
res2 = run_sql("SELECT dir FROM sbmACTION WHERE sactname=%s", (action[1],))
nextdir = res2[0][0]
curraction = {
'page' : action[11],
'action' : action[1],
'doctype' : doctype,
'nextdir' : nextdir,
'access' : access,
'indir' : indir,
'name' : action[12],
}
actions.append(curraction)
t = websubmit_templates.tmpl_next_action(
ln = ln,
actions = actions,
)
return t
def Test_Reload(uid_email,doctype,act,access):
res = run_sql("SELECT * FROM sbmSUBMISSIONS WHERE doctype=%s and action=%s and id=%s and email=%s and status='finished'", (doctype,act,access,uid_email,))
if len(res) > 0:
return 1
else:
return 0
class functionError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class functionStop(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
def errorMsg(title,req,c=cdsname,ln=cdslang):
# load the right message language
_ = gettext_set_language(ln)
return page(title = _("error"),
body = create_error_box(req, title=title,verbose=0, ln=ln),
description="%s - Internal Error" % c,
keywords="%s, CDSware, Internal Error" % c,
language=ln,
urlargs=req.args)
def warningMsg(title,req,c=cdsname,ln=cdslang):
# load the right message language
_ = gettext_set_language(ln)
return page(title = _("warning"),
body = title,
description="%s - Internal Error" % c,
keywords="%s, CDSware, Internal Error" % c,
language=ln,
urlargs=req.args)
def getCookie(name,uid):
# these are not real http cookies but are stored in the DB
res = run_sql("select value from sbmCOOKIES where uid=%s and name=%s", (uid,name,))
if len(res) > 0:
return res[0][0]
else:
return None
def setCookie(name,value,uid):
# these are not real http cookies but are stored in the DB
res = run_sql("select id from sbmCOOKIES where uid=%s and name=%s", (uid,name,))
if len(res) > 0:
run_sql("update sbmCOOKIES set value=%s where uid=%s and name=%s", (value,uid,name,))
else:
run_sql("insert into sbmCOOKIES(name,value,uid) values(%s,%s,%s)", (name,value,uid,))
return 1
def specialchars(text):
text = string.replace(text,"“","\042");
text = string.replace(text,"”","\042");
text = string.replace(text,"’","\047");
text = string.replace(text,"—","\055");
text = string.replace(text,"…","\056\056\056");
'surequit' : _("Are you sure you want to quit this submission?"),
'back' : _("back to main menu"),
'mainmenu' : mainmenu,
'images' : images,
'take_note' : _("(1) you should take note of this number at the beginning of the submission, it will allow you to get your information back in case your browser crashes before the end of the submission."),
'explain_summary' : _("(2) mandatory fields appear in red in the 'Summary' window."),
}
return out
def tmpl_submit_field(self, ln, field):
"""
Produces the HTML code for the specified field
Parameters:
- 'ln' *string* - The language to display the interface in
- 'field' *array* - the field to display in the page, with the following structure:
- 'javascript' *string* - if the field has some associated javascript code
- 'type' *string* - the type of field (T, F, I, H, D, S, R)
- 'name' *string* - the name of the field
- 'rows' *string* - the number of rows for textareas
- 'cols' *string* - the number of columns for textareas
- 'val' *string* - the default value of the field
- 'size' *string* - the size for text fields
- 'maxlength' *string* - the maximum length for text fields
- 'htmlcode' *string* - the complete HTML code for user-defined fields
global $calledBefore,$doctype,$action,$up,$x,$y,$previousScore,$currentStep,$previousStep,$function,$currentScore,$down,$nextStep,$nextScore,$deleteFunc,$previousFunction,$nextFunction;
if(!($calledBefore))
{
displayFunctions($doctype, $action);
} // END if
else
{
if(isset($up))
{
# Destroy memory space associated with x and y coordinates
# which were passed from the input image...
unset($x);
unset($y);
# Destroy $up...
unset($up);
# Make a query string to update the row to move, to its new
# score and step values...
$queryStringA = "UPDATE sbmFUNCTIONS SET score = "
. "'$previousScore'";
if($previousStep < $currentStep)
{ # If the function aboves step is lower, then we must also
# change this...
$queryStringA = $queryStringA . ", step = "
. "'$previousStep'";
} // END if
$queryStringA = $queryStringA . " WHERE doctype = '$doctype'"
. " AND function = '$function' AND score = "
. "'$currentScore' AND step = '$currentStep' and action='$action'";
# Now, make another query string to update the function that
# was above the function that we have just updated in the
# order. This function should be updated to have the values
# that the function that we just updated had!
$queryStringB = "UPDATE sbmFUNCTIONS SET score = "
. "'$currentScore'";
if($previousStep < $currentStep)
{ # If the function we are alterring to the one above has a
# lower step than the one that came after it in the list,
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
<?
require("commonPhpFunctions.php");
?>
#include "cdspage.wml" \
title="Edit the <I><protect><?print $fidesc;?></protect></I> Element on page <protect><?print $pageNumber;?></protect> of <protect><?print $subname;?></protect>" \
. "ERROR:</SPAN> Unable to conduct a query on the "
. "<EM>sbmFIELDDESC</EM> table of " . DOCS_DATABASE . ".<BR>Please"
. " inform system administrator.</P>\n");
} // END else
} // END function fabricateElementDetsFormEDS()
function displayPage()
{
global $doctype,$update,$changeType,$val,$rows,$cols,$cookie,$alephcode,$marccode,$type,$size,$maxlength,$fidesc,$name,$subname,$pageNumber,$nPgs,$caller,$modifytext;
# Conduct a test to see if this is the first call to this page, or a
# recursive call whereby the user is attempting to update the
# element descriptions details
if($update)
{
# In this case, this is a call to update the Element description.
# Therefore, update it, and redirect the "pageDetsEDS.php" page.
$updtStr = "UPDATE sbmFIELDDESC SET ";
# determine what kind of element we are
# actually updating here, as we only need to amend certain fields
## The CDSware is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## The CDSware is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDSware; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
<?
require("commonPhpFunctions.php");
?>
#include "cdspage.wml" \
title="List of parameters of function <I><protect><?print $functionName;?></protect></i> for the <I><protect><?print $doctype;?></protect></i> document type" \
# Run a query to get all of the submission types for the doctype to
# be cloned...
$cloneRes1 = mysql_query("SELECT * FROM sbmIMPLEMENT WHERE docname ="
. " '$doc2clone' ORDER BY actname");
if($cloneRes1)
{
# Fine, the query has worked...
# Get the current date...
$modDate = makeEDSmdDate();
# Okay, for each submission
while($sub2Clone = mysql_fetch_array($cloneRes1))
{
# Clone functions
$res = mysql_query("select * from sbmFUNCTIONS where doctype='$doc2clone' and action='".$sub2Clone["actname"]."'");
while ($row = mysql_fetch_array($res))
{
$res2 = mysql_query("select 1 from sbmFUNCTIONS where doctype='$sdocname' and action='".$row["action"]."' and function='".$row["function"]."' and score='".$row['score']."' and step='".$row['step']."'");
if (mysql_num_rows($res2) < 1)
mysql_query("insert into sbmFUNCTIONS(action, doctype, function, score, step) values ('".$row['action']."','$sdocname','".$row['function']."','".$row['score']."','".$row['step']."')");
}
#clone parameters
$res = mysql_query("select * from sbmPARAMETERS where doctype='$doc2clone'");
while ($row = mysql_fetch_array($res))
{
$res2 = mysql_query("select 1 from sbmPARAMETERS where doctype='$sdocname' and name='".$row['name']."'");
if (mysql_num_rows($res2) < 1)
mysql_query("insert into sbmPARAMETERS(doctype, name, value) values ('$sdocname','".$row['name']."','".$row['value']."')");
}
# Make a query to insert this submission data for the new
$msgTxt = "An error ocurred when a new submission"
. " was added to the $doctype document type. Thi"
. "s new submission is the ${subToAdd}$doctype "
. "submission and was cloned from the $toClone "
. "document type.\n\nIt was not possible to add "
. "the " . $element["fidesc"] . " element which "
. "should appear in position "
. $element["fieldnb"] . " on page "
. $element["pagenb"] . " of this new submission."
. "\n\nThe cloning of the rest of this submission"
. " was however continued, which means that there"
. " may be problems with elements being out of or"
. "der on this page of the submission.\n\nYou sho"
. "uld examine this and correct any problems imme"
. "diately.\n\nWebSubmit Administrator (";
$msgTxt .= makeDate();
$msgTxt .= ")";
# Now send the mail..
mail(ADMIN_EMAIL, "ERROR: Unable to Add "
. $element["fidesc"] . " Element "
. "to ${subToAdd}$doctype", $msgTxt,
"From: WebSubmit_Administrator");
} // END else
} // END while
# Now we also clone the functions used in this submission
$res = mysql_query("SELECT action,function,score,step FROM sbmFUNCTIONS WHERE action='$subToAdd' and doctype='$toClone'");
while ($row = mysql_fetch_array($res))
{
$res2 = mysql_query("SELECT 1 from sbmFUNCTIONS where action='".$row['action']."' and doctype='$doctype' and function='".$row['function']."' and score='".$row['score']."' and step='".$row['step']."'");
if (mysql_num_rows($res2) < 1)
mysql_query("INSERT INTO sbmFUNCTIONS (action,doctype,function,score,step) VALUES ('".$row['action']."','$doctype','".$row['function']."','".$row['score']."','".$row['step']."')");
}
#clone parameters
$res = mysql_query("select * from sbmPARAMETERS where doctype='$toClone'");
while ($row = mysql_fetch_array($res))
{
$res2 = mysql_query("select * from sbmPARAMETERS where doctype='$doctype' and name='".$row['name']."'");
if (mysql_num_rows($res2) < 1)
mysql_query("insert into sbmPARAMETERS(doctype, name, value) values ('$doctype','".$row['name']."','".$row['value']."')");
}
# Now that the submission has been cloned, it is
# possible to present the user with a message saying
# this, and redirect their browser to the
# "documentEDS.php" page so that they can look into the
# details of this new submission, and change anything
# that they might feel appropriate. A javascript alert
# should also be displayed however, advising them to
# check all of the values for their new submission and
# ensure that they are correct. We can then mail the
# administrator to let them know that a new submission
# has been created, and that it was cloned from another
. "Couldn't update the md in sbmIMPLEMENT!');</SCRIPT>\n");
} // END else
# Update the doctype modification date
updateEDSDOCTYPEmd($doctype, $modifiedDate);
} // END function deletePageElement()
function displayPage()
{
global $moveUp,$x,$y,$previousFieldnb,$subname,$pageNumber,$fidesc,$currFieldnb,$previousfiDesc,$doctype,$nPgs,$moveDown,$moveDigital,$elementPos,$deleteElement;
# Conduct a test to see what kind of call to this page this actually
# is, and then take the appropriate action based upon this
# outcome...
if(isset($moveUp))
{
# If the $up variable is set, it means that the user has pressed
# the up arrow button to move an element up in the order in which
t+= "<TD><a href=\"\" onClick=\"if (confirm('Are you sure you want to delete this referee?')){document.forms[0].todo.value='deleteuser';document.forms[0].id.value='%s';document.forms[0].role.value='%s';document.forms[0].submit();return false;}else{return false;}\">" % (referee[0],role)
res = run_sql("SELECT sbmSUBMISSIONS.* FROM sbmSUBMISSIONS,sbmACTION WHERE sactname=action and email=%s and id!='' "+docselect+" ORDER BY doctype,"+reqorder,(u_email,))
currentdoctype = ""
currentaction = ""
currentstatus = ""
submissions = []
for row in res:
if currentdoctype != row[1]:
currentdoctype = row[1]
currentaction = ""
currentstatus = ""
res2 = run_sql("SELECT ldocname FROM sbmDOCTYPE WHERE sdocname=%s",(currentdoctype,))
if res2:
ldocname = res2[0][0]
else:
ldocname = """***Unknown Document Type - (%s)""" % (currentdoctype,)
if currentaction != row[2]:
currentaction = row[2]
res2 = run_sql("SELECT lactname FROM sbmACTION WHERE sactname=%s",(currentaction,))