diff --git a/modules/bibformat/doc/hacking/bibformat-api.webdoc b/modules/bibformat/doc/hacking/bibformat-api.webdoc index e703eec65..dbc9f41ec 100644 --- a/modules/bibformat/doc/hacking/bibformat-api.webdoc +++ b/modules/bibformat/doc/hacking/bibformat-api.webdoc @@ -1,866 +1,885 @@ ## -*- mode: html; coding: utf-8; -*- ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. 
 ****************************************************************************
 ** IMPORTANT NOTE: Note that this documentation is an updated version of  **
 ** an earlier technical draft of BibFormat specifications. Please first   **
 ** refer to the BibFormat admin guide.                                    **
 ****************************************************************************
 
 Technical Overview of the new BibFormat
 =======================================
 
 Contents:
 1. Python API
 2. The philosophy behind BibFormat
 3. Differences between the old PHP version and the new Pythonic version
 4. Migrating from the previous PHP BibFormat version to the new Pythonic version
 5. Specifications of the new BibFormat configuration files.
 
 
 1. Python API
 
 The APIs of bibformat.py consists in these functions:
 
  def format_record(recID, of, ln=cdslang, verbose=0,
-                   search_pattern=None, xml_record=None, uid=None,
+                   search_pattern=None, xml_record=None, user_info=None,
                    on_the_fly=False):
      """
      Formats a record given its ID (or its XML representation)
      and an output format.
 
      Returns a formatted version of the record in the specified
      language, with pattern context, and specified output format.
      The function will define by itself which format template must be
      applied.
 
      Parameters that allow contextual formatting (like 'search_pattern'
-     and 'uid') are useful only when doing on-the-fly formatting,
-     or when caching with care (e.g. caching all formatted
+     and 'user_info') are useful only when doing on-the-fly
+     formatting, or when caching with care (e.g. caching all formatted
      versions of a record for each possible 'ln').
 
      The arguments are as follows:
 
                recID -  the ID of the record to format. If ID does not exist
                         the function returns empty string or an error
                         string, depending on level of verbosity.
                         If 'xml_record' parameter is specified, 'recID'
                         is ignored
 
                   of -  an output format code. If 'of' does not exist as code in
                         output format, the function returns empty
                         string or an error string, depending on level
                         of verbosity.  ;of' is case insensitive.
 
                   ln -  the language to use to format the record. If
                         'ln' is an unknown language, or translation
                         does not exist, default cdslang language
                         will be applied whenever possible.
                         Allows contextual formatting.
 
              verbose -  the level of verbosity in case of errors/warnings
                         0 - Silent mode
                         5 - Prints only errors
                         9 - Prints errors and warnings
 
       search_pattern -  the pattern used as search query when asked to
                         format this record (User request in web
                         interface). Allows contextual formatting.
 
           xml_record -  an XML string representation of the record to
                         format.  If it is specified, recID parameter is
                         ignored. The XML must be pasable by BibRecord.
 
-                 uid -  User ID of the user who will view the formatted
-                        record.  Useful to grant access to special
-                        functions on a page depending on user's
-                        priviledge.  Allows contextual formatting.
-                        Typically 'uid' is retrieved with webuser.getUid(req).
+           user_info - allows to grant access to some functionalities
+                       on a page depending on the user's
+                       priviledges. 'user_info' is the same structure
+                       as the one returned by webuser.collect_user_info(req),
+                       (that is a dictionary).
 
-          on_the_fly -  if False, try to return an already preformatted version
-                        of the record in the database.
+          on_the_fly - if False, try to return an already preformatted
+                       version of the record in the database.
 
      """
 
 
  Example:
    >> from invenio.bibformat import format_record
    >> format_record(5, "hb", "fr")
 
 
  def format_records(recIDs, of, ln=cdslang, verbose=0, search_pattern=None,
-                    xml_records=None, uid=None, record_prefix=None,
+                    xml_records=None, user_info=None, record_prefix=None,
                     record_separator=None, record_suffix=None,
                     prologue="", epilogue="", req=None, on_the_fly=False):
      """
      Returns a list of formatted records given by a list of record IDs or a
      list of records as xml.
      Adds a prefix before each record, a suffix after each record,
      plus a separator between records.
 
      Also add optional prologue and epilogue to the complete formatted list.
 
      You can either specify a list of record IDs to format, or a list of
      xml records, but not both (if both are specified recIDs is ignored).
 
      'record_separator' is a function that returns a string as separator between
      records. The function must take an integer as unique parameter,
      which is the index in recIDs (or xml_records) of the record that has
      just been formatted. For example separator(i) must return the separator
      between recID[i] and recID[i+1]. Alternatively separator can be a single
      string, which will be used to separate all formatted records.
      The same applies to 'record_prefix' and 'record_suffix'.
 
      'req' is an optional parameter on which the result of the function
      are printed lively (prints records after records) if it is given.
      Note that you should set 'req' content-type by yourself, and send
      http header before calling this function as it will not do it.
 
      This function takes the same parameters as 'format_record' except for:
 
                recIDs -  a list of record IDs to format
 
           xml_records -  a list of xml string representions of the records to
                          format. If this list is specified, 'recIDs' is ignored.
 
         record_prefix - a string or a function the takes the index of the record
                         in 'recIDs' or 'xml_records' for which the function must
                         return a string.
                         Printed before each formatted record.
 
      record_separator - either a string or a function that returns string to
                         separate formatted records. The function takes the index
                         of the record in 'recIDs' or 'xml_records' that is being
                         formatted.
 
         record_prefix - a string or a function the takes the index of the record
                         in 'recIDs' or 'xml_records' for which the function must
                         return a string.
                         Printed after each formatted record
 
                   req - an optional request object on which formatted records
                         can be printed (for "live" output )
 
              prologue - a string printed before all formatted records string
 
              epilogue - a string printed after all formatted records string
 
            on_the_fly - if False, try to return an already preformatted version
                         of the records in the database
      """
 
 
  def get_output_format_content_type(of):
      """
      Returns the content type (eg. 'text/html' or 'application/ms-excel') \
      of the given output format.
 
      The function takes this mandatory parameter:
 
      of - the code of output format for which we want to get the content type
      """
 
 
  def record_get_xml(recID, format='xm', decompress=zlib.decompress):
      """
      Returns an XML string of the record given by recID.
 
      The function builds the XML directly from the database,
      without using the standard formatting process.
 
      'format' allows to define the flavour of XML:
         - 'xm' for standard XML
         - 'marcxml' for MARC XML
         - 'oai_dc' for OAI Dublin Core
         - 'xd' for XML Dublin Core
 
      If record does not exist, returns empty string.
 
      The function takes the following parameters:
 
           recID - the id of the record to retrieve
 
          format - the XML flavor in which we want to get the record
 
      decompress _ a function used to decompress the record from the database
     """
 
 The API of the BibFormat Object ('bfo') given as a parameter to
 format function of format elements consist in the following
 functions. This API is to be used only inside format elements.
 
  def control_field(self, tag, escape='0'):
     """
     Returns the value of control field given by tag in record.
 
     If the value does not exist, returns empty string
     The returned value is always a string.
 
     'escape' parameter allows to escape special characters
     of the field. The value of escape can be:
           0 - no escaping
           1 - escape all HTML characters
           2 - escape all HTML characters by default. If field starts with ,
               escape only unsafe characters, but leave basics HTML tags.
               This is particularly useful if you want to store HTML text in your
               metadata but still want to escape some tags to prevent
               XSS vulnerabilities. Note that this method is slower than
               basic escaping of mode 1.
 
     The arguments are:
 
          tag    -  the marc code of a field
          escape -  1 if returned value should be escaped. Else 0.
                    (see above for other modes)
     """
 
  def field(self, tag, escape='0'):
     """
     Returns the value of the field corresponding to tag in the
     current record.
 
     If the value does not exist, returns empty string
     The returned value is always a string.
 
     'escape' parameter allows to escape special characters
     of the field. The value of escape can be:
           0 - no escaping
           1 - escape all HTML characters
           2 - escape all HTML characters by default. If field starts with ,
               escape only unsafe characters, but leaves basic HTML tags.
               This is particularly useful if you want to store HTML text in your
               metadata but still want to escape some tags to prevent
               XSS vulnerabilities. Note that this method is slower than
               basic escaping of mode 1.
 
     The arguments are:
 
          tag  -  the marc code of a field
          escape -  1 if returned value should be escaped. Else 0.
                    (see above for other modes)
     """
 
 
  def fields(self, tag, escape='0', repeatable_subfields_p=False):
     """
     Returns the list of values corresonding to "tag".
 
     If tag has an undefined subcode (such as 999C5),
     the function returns a list of dictionaries, whoose keys
     are the subcodes and the values are the values of tag.subcode.
     If the tag has a subcode, simply returns list of values
     corresponding to tag.
     Eg. for given MARC:
         999C5 $a value_1a $b value_1b
         999C5 $b value_2b
         999C5 $b value_3b $b value_3b_bis
 
         >> bfo.fields('999C5b')
         >> ['value_1b', 'value_2b', 'value_3b', 'value_3b_bis']
         >> bfo.fields('999C5')
         >> [{'a':'value_1a', 'b':'value_1b'},
             {'b':'value_2b'},
             {'b':'value_3b'}]
 
     By default the function returns only one value for each
     subfield (that is it considers that repeatable subfields are
     not allowed). It is why in the above example 'value3b_bis' is
     not shown for bfo.fields('999C5').  (Note that it is not
     defined which of value_3b or value_3b_bis is returned).  This
     is to simplify the use of the function, as most of the time
     subfields are not repeatable (in that way we get a string
     instead of a list).  You can allow repeatable subfields by
     setting 'repeatable_subfields_p' parameter to True. In
     this mode, the above example would return:
         >> bfo.fields('999C5b', repeatable_subfields_p=True)
         >> ['value_1b', 'value_2b', 'value_3b']
         >> bfo.fields('999C5', repeatable_subfields_p=True)
         >> [{'a':['value_1a'], 'b':['value_1b']},
             {'b':['value_2b']},
             {'b':['value_3b', 'value3b_bis']}]
     NOTICE THAT THE RETURNED STRUCTURE IS DIFFERENT.  Also note
     that whatever the value of 'repeatable_subfields_p' is,
     bfo.fields('999C5b') always show all fields, even repeatable
     ones. This is because the parameter has no impact on the
     returned structure (it is always a list).
 
     'escape' parameter allows to escape special characters
     of the field. The value of escape can be:
           0 - no escaping
           1 - escape all HTML characters
           2 - escape all HTML characters by default. If field starts with ,
               escape only unsafe characters, but leaves basic HTML tags.
               This is particularly useful if you want to store HTML text in your
               metadata but still want to escape some tags to prevent
               XSS vulnerabilities. Note that this method is slower than
               basic escaping of mode 1.
 
     The arguments are:
 
           tag  -  the marc code of a field
           escape -  1 if returned value should be escaped. Else 0.
                    (see above for other modes)
     """
 
  def kb(self, kb, string, default=""):
     """
     Returns the value of the "string" in the knowledge base "kb".
 
     If kb does not exist or string does not exist in kb,
     returns 'default' string or empty string if not specified
 
     The arguments are as follows:
 
           kb  -  the knowledge base name in which we want to find the mapping.
                  If it does not exist the function returns the original
                  'string' parameter value. The name is case insensitive (Uses
                  the SQL 'LIKE' syntax to retrieve value).
 
       string  -  the value for which we want to find a translation-
                  If it does not exist the function returns 'default' string.
                  The string is case insensitive (Uses the SQL 'LIKE' syntax
                  to retrieve value).
 
      default  -  a default value returned if 'string' not found in 'kb'.
 
     """
 
  def get_record(self):
     """
     Returns the record encapsulated in bfo as a BibRecord structure.
     You can get full access to the record through bibrecord.py functions.
     """
 
   Example (from inside BibFormat element):
   >> bfo.field("520.a")
   >> 'We present a quantitative appraisal of the physics potential
       for neutrino experiments.'
   >>
   >> bfo.control_field("001")
   >> '12'
   >>
   >> bfo.fields("700.a")
   >>['Alekhin, S I', 'Anselmino, M', 'Ball, R D', 'Boglione, M']
   >>
   >> bfo.kb("DBCOLLID2COLL", "ARTICLE")
   >> 'Published Article'
   >>
   >> bfo.kb("DBCOLLID2COLL", "not in kb", "My Value")
   >> 'My Value'
 
 Moreover you can have access to the language requested for the
 formatting, the search pattern used by the user in the web
 interface and the userID by directly getting the attribute from 'bfo':
 
     bfo.lang
     """
     Returns the language that was asked to be used for the
     formatting. Always returns a string.
     """
 
     bfo.search_pattern
     """
     Returns the search pattern specified by the user when
     the record had to be formatted. Always returns a string.
     """
 
+    bfo.user_info
+    """
+    Returns a dictionary with information about current user.
+    The returned dictionary has the following structure:
+        user_info = {
+            'remote_ip' : '',
+            'remote_host' : '',
+            'referer' : '',
+            'uri' : '',
+            'agent' : '',
+            'apache_user' : '',
+            'apache_group' : [],
+            'uid' : -1,
+            'nickname' : '',
+            'email' : '',
+            'group' : [],
+            'guest' : '1'
+        }
+    """
+
     bfo.uid
     """
-    Returns the user ID of the user who shall view the formatted
-    record.
+    ! DEPRECATED: use bfo.user_info['uid'] instead
     """
 
     bfo.recID
     """
     Returns the id of the record
     """
 
     bfo.req
     """
-    Returns the mod_python request object
+    ! DEPRECATED: use bfo.user_info instead
     """
 
     bfo.format
     """
     Returns the format in which the record is being formatted
     """
 
   Example (from inside BibFormat element):
   >> bfo.lang
   >> 'en'
   >>
   >> bfo.search_pattern
   >> 'mangano and neutrino and factory'
 
 
 2. The philosophy behind BibFormat
 
 BibFormat is in charge of formatting the bibliographic records that
 are displayed to your users. As you potentially have a huge amount of
 bibliographic records, you cannot specify manually for each of them
 how it should be formatted. This is why you can define rules that will
 allow BibFormat to understand which kind of formatting to apply to a given
 record. You define this set of rules in what is called an "output
 format".
 
 You can have different output formats, each with its own characteristics.
 For example you certainly want that when multiple bibliographic records are
 displayed at the same time (as it happens in search results), only
 short versions are shown to the user, while a detailed record is
 preferable when a single record is displayed. You might also want to
 let your users decide which kind of output they want. For example you
 might need to display HTML for regular web browsing, but would also
 give a BibTeX version of the bibliographic reference for direct
 inclusion in a LaTeX document.
 See section 5.1 to learn how to create or modify output formats.
 
 While output formats define what kind of formatting must be applied,
 they do not define HOW the formatting is done. This is the role of the
 "format templates", which define the layout and look of a
 bibliographic reference. These format templates are rather easy to
 write if you know a little bit of HTML (see section 5.2 "Format
 templates specifications"). You will certainly have to create
 different format templates, for different kinds of records. For
 example you might want records that contain pictures to display them,
 maybe with captions, while records that do not have pictures limited
 to printing a title and an abstract.
 
 In summary, you have different output formats (like 'brief HTML',
 'detailed HTML' or 'BibTeX') that call different format templates
 according to some criteria.
 
 There is still one kind of configuration file that we have not talked
 about: the "format elements". These are the "bricks" that you use in
 format templates, to get the values of a record. You will learn to use
 them in your format template in section 5.2 "Format templates
 specifications", but you will mostly not need to modify them or create
 new ones. However if you do need to edit one, read section 5.3 "Format
 elements specifications" (And if you know Python it will be easy, as
 they are written in Python).
 
 Finally BibFormat can make use of mapping tables called "knowledge
 bases". Their primary use is to act like a translation table, to
 normalize records before displaying them. For example, you can say
 that records that have value "Phys Rev D" or "Physical Review D" for
 field "published in" must display "Phys Rev : D." to users. See
 section 5.4 to learn how to edit knowledge bases.
 
 In summary, there are three layers.  Output formats:
 
 +-----------------------------------------------------+
 |                    Output Format                    | (Layer 1)
 |                    eg: HTML_Brief.bfo               |
 +-----------------------------------------------------+
 
 call one of several `format templates':
 
 +-------------------------+ +-------------------------+
 |     Format Template     | |     Format Template     | (Layer 2)
 |     eg: preprint.bft    | |     eg: default.bft     |
 +-------------------------+ +-------------------------+
 
 that use one or several format elements:
 
 +--------------+ +----------------+ +-----------------+
 |Format Element| |Format Element  | | Format Element  | (Layer 3)
 |eg: authors.py| |eg: abstract.py | | eg: title.py    |
 +--------------+ +----------------+ +-----------------+
 
 
 3. Differences between the old PHP version and the new Pythonic version
 
 The most noticeable differences are:
 
  a) "Behaviours" have been renamed "Output formats".
  b) "Formats" have been renamed "Format templates". They are now
      written in HTML.
  c) "User defined functions" have been dropped.
  d) "Extraction rules" have been dropped.
  e) "Link rules" have been dropped.
  f) "File formats" have been dropped.
  g) "Format elements" have been introduced. They are written in Python,
      and can simulate c), d) and e).
  h)  Formats can be managed through web interface or through
      human-readable config files.
  i)  Introduction of tools like validator and dependencies checker.
  j)  Better support for multi-language formatting.
 
 Some of the advantages are:
 
  + Management of formats is much clearer and easier (less concepts,
    more tools).
  + Writing formats is easier to learn : less concepts
    to learn, redesigned work-flow, use of existing well known and
    well documented languages.
  + Editing formats is easier: You can use your preferred HTML editor such as
    Emacs, Dreamweaver or Frontpage to modify templates, or any text
    editor for output formats and format elements. You can also use the
    simplified web administration interface.
  + Faster and more powerful templating system.
  + Separation of business logic (output formats, format elements)
    and presentation layer (format templates). This makes the management
    of formats simpler.
 
 The disadvantages are:
 
  - No backward compatibility with old formats.
  - Stricter separation of business logic and presentation layer:
    no more use of statements such as if(), forall() inside templates,
    and this requires more work to put logic inside format elements.
 
 
 4. Migrating from the previous PHP BibFormat version to the new Pythonic version
 
 Old BibFormat formats are no longer compatible with the new BibFormat
 files. If you have not modified the "Formats" or modified only a
 little bit the "Behaviours", then the transition will be painless and
 automatic. Otherwise you will have to manually rewrite some of the
 formats. This should however not be a big problem. Firstly because the
 CDS Invenio installation will provide both versions of BibFormat for
 some time. Secondly because both BibFormat versions can run side by
 side, so that you can migrate your formats while your server still
 works with the old formats.  Thirdly because we provide a migration
 kit that can help you go through this process. Finally because the
 migration is not so difficult, and because it will be much easier for
 you to customize how BibFormat formats your bibliographic data.
 
 Concerning the migration kit it can:
  a) Effortlessly migrate your behaviours, unless they include complex
     logic, which usually they don't.
  b) Help you migrate formats to format templates and format elements.
  c) Effortlessly migrate your knowledge bases.
 
 Point b) is the most difficult to achieve: previous formats did mix
 business logic and code for the presentation, and could use PHP
 functions. The new BibFormat separates business logic and
 presentation, and does not support PHP. The transition kit will try to
 move business logic to the format elements, and the presentation to
 the format templates. These files will be created for you, includes
 the original code and, if possible, a proposal of Python
 translation. We recommend that you do not to use the transition kit to
 translate formats, especially if you have not modified default
 formats, or only modified default formats in some limited places. You
 will get cleaner code if you write format elements and format
 templates yourself.
 
 
 5. Specifications of the new BibFormat configuration files.
 
    BibFormat uses human readable configuration files. However (apart
    from format elements) these files can be edited and managed through
    a web interface.
 
 5.1 Output formats specifications
 
 Output formats specify rules that define which format template
 to use to format a record.
 While the syntax of output formats is basic, we recommend that you use
 the web interface do edit them, to be sure that you make no error.
 
 The syntax of output format is the following one. First you
 define which field code you put as the conditon for the rule.
 You suffix it with a column. Then on next lines, define the values of
 the condition, followed by --- and then the filename of the template
 to use:
 
   tag 980.a:
   PICTURE --- PICTURE_HTML_BRIEF.bft
   PREPRINT --- PREPRINT_HTML_BRIEF.bft
   PUBLICATION --- PUBLICATION_HTML_BRIEF.bft
 
 This means that if value of field 980.a is equal to PICTURE, then we
 will use format template PICTURE_HTML_BRIEF.bft. Note that you must
 use the filename of the template, not the name. Also note that spaces
 at the end or beginning are not considered. On the following lines,
 you can either put other conditions on tag 980.a, or add another tag on
 which you want to put conditions.
 
 At the end you can add a default condition:
 
    default: PREPRINT_HTML_BRIEF.bft
 
 which means that if no condition is matched, a format suitable for
 Preprints will be used to format the current record.
 
 The output format file could then look like this:
 
   tag 980.a:
   PICTURE --- PICTURE_HTML_BRIEF.bft
   PREPRINT --- PREPRINT_HTML_BRIEF.bft
   PUBLICATION --- PUBLICATION_HTML_BRIEF.bft
 
   tag 8560.f:
   .*@cern.ch --- SPECIAL_MEMBER_FORMATTING.bft
 
   default: PREPRINT_HTML_BRIEF.bft
 
 You can add as many rules as you want. Keep in mind that they are read
 in the order they are defined, and that only first rule that
 matches will be used.
 Notice the condition on tag 8560.f: it uses a regular expression to
 match any email address that ends with @cern.ch (the regular
 expression must be understandable by Python)
 
 Some other considerations on the management of output formats:
 - Format outputs must be placed inside directory
   /etc/bibformat/outputs/ of your CDS Invenio installation.
 - Note that as long as you have not provided a name to an output
   THROUGH the web interface, it will not be available as a choice
   for your users in some parts of CDS Invenio.
 - You should remove output formats THROUGH the web interface.
 - The format extension of output format is .bfo
 
 
 5.2 Format templates specifications
 
 Format templates are written in HTML-like syntax. You can use the
 standard HTML and CSS markup languague to do the formatting. The best
 thing to do is to create a page in your favourite editor, and once you
 are glad with it, add the dynamic part of the page, that is print the
 fields of the records. Let's say you have defined this page:
 
   <h1>Some title</h1>
   <p><i>Abstract: </i>Some abstract</p>
 
 Then you want that instead of "Some title" and "Some abstract", the
 value of the current record that is being displayed is used. To do so,
 you must use a format element brick. Either you know the name of the
 brick by heart, or you look for it in the elements documentation (see
 section 5.3). For example you would find there that you can print the
 title of the record by writting the HTML tag <BFE_TITLE /> in your
 format template, with parameter 'default' for a default value.
 
   <h1><BFE_TITLE default="No Title"/></h1>
   <p><BFE_ABSTRACT limit="1" prefix="<i>Abstract: </i>"
   default="No abstract"/></p>
 
 Notice that <BFE_ABSTRACT /> has a parameter "limit" that <BFE_title/>
  had not ("limit" allows to limit the number of sentences of the
 abstract, according to the documentation). Note that while format
 elements might have different parameters, they always can take the the
 three following ones: "prefix" and "suffix", whose values are printed
 only if the element is not empty, and "default", which is printed only
 if element is an empty string. We have used "prefix" for the abstract,
 so that the label "<i>Abstract: </i>" is only printed if the record
 has an abstract.
 
 You should also provide these tags in all of your templates:
  -<name>a name for this template in the admin web interface</name>
  -<description>a description to be used in admin web interface for
   this template</description>
 
 Another feature of the templates is the support for multi-languages
 outputs. You can include <lang> tags, which contain tags labeled with
 the names of the languages supported in CDS Invenio. For example, one
 might write:
 
   <lang><en>A record:</en><fr>Une notice:</fr></lang>
   <h1><BFE_TITLE default="No Title"/></h1>
   <p><BFE_ABSTRACT limit="1" prefix="<i>Abstract: </i>"
   default="No abstract"/></p>
 
 When doing this you should at least make sure that the default
 language of your server installation is available in each <lang>
 tag. It is the one that is used if the requested language to display
 the record is not available. Note that we could also provide a
 translation in a similar way for the "No Title" default value inside
 <BFE_Title /> tag.
 
 Some other considerations on the use of elements inside templates:
  -Format elements names are not case sensitive
  -Format elements names always start with <BFE_
  -Format elements parameters can contain '<' characters,
   and quotes different from the kind that delimit parameters (you can
   for example have <BFE_Title default='<a href="#">No Title</a>'/> )
  -Format templates must be placed inside the directory
   /etc/bibformat/templates/ of your CDS Invenio installation
  -The format extension of a template is .bft
 
 Trick: you can use the <BFE_FIELD tag="245__a" /> to print the value
 of any field 245 $a in your templates.  This practice is however not
 recommended because it would necessitate to revise all format
 templates if you change meaning of the MARC code schema.
 
 5.3 Format elements specifications
 
 Format elements are the bricks used in format templates to provide the
 dynamic contents inside format templates.
 
 For the most basic format elements, you do not even need to write
 them: as long as you define `tag names' for MARC tags in the BibIndex
 Admin's Manage logical fields interface (database table tag),
 BibFormat knows which field must be printed when <BFE_tag_name/> is
 used inside a template.
 
 However for more complex processing, you will need to write a format
 element. A format element is written in Python. Therefore its file
 extension is ".py". The name you choose for the file is the one that
 will be used inside format template to call the element, so choose it
 carefully such that it is not too long, but self explanatory (you can
 prefix the filename with BFE or not, but the element will always be called
 with prefix <BFE_ inside templates).  Then you just need to drop the
 file in the /lib/python/invenio/bibformat_elements/ directory
 of your CDS Invenio installation. Inside your file you have to define a
 function named "format", which takes at least a "bfo" parameter (bfo
 for BibFormat Object). The function must return a string:
 
   def format(bfo):
       out = ""
 
       return out
 
 You can have as many parameters as you want, as long as you make sure
 that parameter bfo is here. Let's see how to define an element that
 will print a brief title. It will take a parameter 'limit' that will
 limit the number of characters printed. We can provide some
 documentation for the elemen in the docstring of the
 function.
 
  def format(bfo, limit="10"):
       """
       Prints a short title
 
       @param limit a limit for the number of printed characters
       """
 
       out = ""
 
       return out
 
 Note that we put a default value of 10 in the 'limit' parameter.  To
 get some value of a field, we must request the 'bfo' object. For
 example we can get the value of field 245.a (field "title"):
 
  def format(bfo, limit="10"):
       """
       Prints a short title
 
       @param limit a limit for the number of printed characters
       """
 
       title = bfo.field('245.a')
 
       limit = int(limit)
       if limit > len(title):
           limit = len(title)
 
       return title[:limit]
 
 As format elements are written in Python, we have decided not to give
 permission to edit elements through the web interface. Firstly for
 security reasons. Secondly because Python requires correct indentation,
 which is difficult to achieve through a web interface.
 
 You can have access to the documentation of your element through a web
 interface. This is very useful when you are writing a format template,
 to see which elements are available, what they do, which parameters they
 take, what are the default values of parameters, etc. The
 documentation is automatically extracted from format elements.
 Here follows an sample documentation generated for the element
 <BFE_TITLE />:
 
 +--------------------------------------------------------------------------------------------+
 |  TITLE                                                                                     |
 |  -----                                                                                     |
 |  <BFE_TITLE separator="..." prefix="..." suffix="..." default="..." />                  |
 |                                                                                            |
 |      Prints the title of a record.                                                         |
 |                                                                                            |
 |      Parameters:                                                                           |
 |            separator - separator between the different titles.                             |
 |            prefix - A prefix printed only if the record has a value for this element.      |
 |            suffix - A suffix printed only if the record has a value for this element.      |
 |            default - A default value printed if the record has no value for this element.  |
 |                                                                                            |
 |       See also:                                                                            |
 |            Format templates that use this element                                          |
 |            The Python code of this element                                                 |
 +--------------------------------------------------------------------------------------------+
 
 The more you provide documentation in the docstring of your elements,
 the easier it will be to write format template afterwards.
 
 Some remarks concerning format elements:
  -parameters are always string values
  -if no value is given as parameter in format the template, then the
   value of parameter is "" (emtpy string)
  -the docstring should contain a description, followed by
   "@param parameter some description for parameter" for each
   parameter (to give description for each parameter
   in element documentation), and @see an_element.py, another_element.py
   (to link to other elements in the documentation). Similar to JavaDoc.
  -the following names cannot be used as parameters:
   "default", "prefix", "suffix" and escape. They can however always be
   used in the format template for any element.
 
 Another important remark concerns the 'escaping' of output of format
 elements. In most cases, format elements output is to be used for
 HTML/XML. Therefore special characters such as < or & have to be
 'escaped', replaced by '<' and '&'. This is why all outputs
 produced by format elements are automatically escaped by BibFormat,
 unless specified otherwise.  This means that you do not have to care
 about meta-data that would break your HTML displaying or XML export
 such as a physics formula like 'a < b'. Please also note that value
 given in 'prefix', 'suffix' and 'default' parameters are not escaped,
 such that you can safely use HTML tags for these.
 
 There are always cases where the default 'escaping' behaviour of
 BibFormat is not desired. For example when you explicitely output HTML
 text, like links: you do not want to see them escaped.  The first way
 to avoid this is to modify the call to your format element in the
 format template, by setting the default 'escape' parameter to 0:
 
  
 
 This is however inconvenient as you have to possibly need to modify a
 lot of templates. The other way of doing is to add another function to
 your format element, named 'escape':
 
  def escape_values(bfo):
      """
      Called by BibFormat in order to check if output of this element
      should be escaped.
      """
      return 0
 
 In that way all calls to your format element will produce unescaped
 output.  You will have to take care of escaping values "manually" in
 your format element code, in order to avoid non valid outputs or XSS
 vulnerabilities. There are methods to ease the escaping in your code
 described in section 1.
 Please also note that if you use this method, your element can still
 be escaped if a call to your element from a format template
 explicitely specifies to escape value using parameter 'escape'.
 
 
 5.4 Knowledge bases specifications
 
 Knowledge bases cannot be managed through configuration files.
 You can very easily add new bases and mappings using the given web GUI.
 
  -- End of file --
diff --git a/modules/bibformat/lib/bibformat.py b/modules/bibformat/lib/bibformat.py index bef441260..a62f7ac8a 100644 --- a/modules/bibformat/lib/bibformat.py +++ b/modules/bibformat/lib/bibformat.py @@ -1,436 +1,449 @@ # -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Format records using specified format. -API functions: format_record, format_records, create_excel, get_output_format_content_type +API functions: format_record, format_records, create_excel, + get_output_format_content_type -Used to wrap the BibFormat engine and associated functions. This is also where -special formatting of multiple records (that the engine does not handle, as it works -on a single record basis) should be put, with name create_*. +Used to wrap the BibFormat engine and associated functions. This is +also where special formatting of multiple records (that the engine +does not handle, as it works on a single record basis) should be put, +with name create_*. SEE: bibformat_utils.py FIXME: currently copies record_exists() code from search engine. Refactor later. """ __revision__ = "$Id$" import zlib from invenio import bibformat_dblayer from invenio import bibformat_engine from invenio import bibformat_utils from invenio.config import cdslang, weburl, CFG_PATH_PHP from invenio.bibformat_config import CFG_BIBFORMAT_USE_OLD_BIBFORMAT try: import invenio.template websearch_templates = invenio.template.load('websearch') except: pass import getopt import sys # Functions to format a single record ## -def format_record(recID, of, ln=cdslang, verbose=0, search_pattern=[], xml_record=None, uid=None, on_the_fly=False): +def format_record(recID, of, ln=cdslang, verbose=0, search_pattern=[], + xml_record=None, user_info=None, on_the_fly=False): """ Formats a record given output format. - Returns a formatted version of the record in - the specified language, search pattern, and with the specified output format. + Returns a formatted version of the record in the specified + language, search pattern, and with the specified output format. The function will define which format template must be applied. - The record to be formatted can be specified with its ID (with 'recID' parameter) or given - as XML representation(with 'xml_record' parameter). If both are specified 'recID' is ignored. + The record to be formatted can be specified with its ID (with + 'recID' parameter) or given as XML representation(with + 'xml_record' parameter). If both are specified 'recID' is ignored. - 'uid' allows to grant access to some functionalities on a page depending - on the user's priviledges. Typically use webuser.getUid(req). This uid has sense - only in the case of on-the-fly formatting. + 'user_info' allows to grant access to some functionalities on a + page depending on the user's priviledges. The 'user_info' object + makes sense only in the case of on-the-fly formatting. 'user_info' + is the same object as the one returned by + 'webuser.collect_user_info(req)' @param recID the ID of record to format @param of an output format code (or short identifier for the output format) @param ln the language to use to format the record @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @param search_pattern list of strings representing the user request in web interface @param xml_record an xml string represention of the record to format - @param uid the user id of the person who will view the formatted page (if applicable) + @param user_info the information of the user who will view the formatted page (if applicable) @param on_the_fly if False, try to return an already preformatted version of the record in the database @return formatted record """ out = "" if verbose == 9: out += """\n Formatting record %i with output format %s. """ % (recID, of) ############### FIXME: REMOVE WHEN MIGRATION IS DONE ############### if CFG_BIBFORMAT_USE_OLD_BIBFORMAT and CFG_PATH_PHP: return bibformat_engine.call_old_bibformat(recID, format=of, on_the_fly=on_the_fly) ############################# END ################################## if not on_the_fly and \ (ln==cdslang or CFG_BIBFORMAT_USE_OLD_BIBFORMAT): # Try to fetch preformatted record # Only possible for records formatted in cdslang language (other are never stored) res = bibformat_dblayer.get_preformatted_record(recID, of) if res is not None: # record 'recID' is formatted in 'of', so return it if verbose == 9: last_updated = bibformat_dblayer.get_preformatted_record_date(recID, of) out += """\n
Found preformatted output for record %i (cache updated on %s).
""" % (recID, last_updated) out += res return out else: if verbose == 9: out+= """\n
No preformatted output found for record %s. """% recID # Live formatting of records in all other cases if verbose == 9: out+= """\n
Formatting record %i on-the-fly. """ % recID try: out += bibformat_engine.format_record(recID=recID, of=of, ln=ln, verbose=verbose, search_pattern=search_pattern, xml_record=xml_record, - uid=uid) + user_info=user_info) return out except Exception, e: #Failsafe execution mode if verbose == 9: out+= """\n
An error occured while formatting record %i. (%s) """ % (recID, str(e)) if of.lower() == 'hd': if verbose == 9: out+= """\n
Formatting record %i with websearch_templates.tmpl_print_record_detailed.
""" % recID return out + websearch_templates.tmpl_print_record_detailed( ln = ln, recID = recID, weburl = weburl, ) if verbose == 9: out+= """\n
Formatting record %i with websearch_templates.tmpl_print_record_brief.
""" % recID return out + websearch_templates.tmpl_print_record_brief(ln = ln, recID = recID, weburl = weburl, ) def record_get_xml(recID, format='xm', decompress=zlib.decompress): """ Returns an XML string of the record given by recID. The function builds the XML directly from the database, without using the standard formatting process. 'format' allows to define the flavour of XML: - 'xm' for standard XML - 'marcxml' for MARC XML - 'oai_dc' for OAI Dublin Core - 'xd' for XML Dublin Core If record does not exist, returns empty string. @param recID the id of the record to retrieve @return the xml string of the record """ return bibformat_utils.record_get_xml(recID=recID, format=format, decompress=decompress) # Helper functions to do complex formatting of multiple records # # You should not modify format_records when adding a complex # formatting of multiple records, but add a create_* method # that relies on format_records to do the formatting. ## -def format_records(recIDs, of, ln=cdslang, verbose=0, search_pattern=None, xml_records=None, uid=None, - record_prefix=None, record_separator=None, record_suffix=None, - prologue="", epilogue="", req=None, on_the_fly=False): +def format_records(recIDs, of, ln=cdslang, verbose=0, search_pattern=None, + xml_records=None, user_info=None, record_prefix=None, + record_separator=None, record_suffix=None, prologue="", + epilogue="", req=None, on_the_fly=False): """ - Returns a list of formatted records given by a list of record IDs or a list of records as xml. - Adds a prefix before each record, a suffix after each record, plus a separator between records. - - Also add optional prologue and epilogue to the complete formatted list. - - You can either specify a list of record IDs to format, or a list of xml records, - but not both (if both are specified recIDs is ignored). - - 'record_separator' is a function that returns a string as separator between records. - The function must take an integer as unique parameter, which is the index - in recIDs (or xml_records) of the record that has just been formatted. For example - separator(i) must return the separator between recID[i] and recID[i+1]. - Alternatively separator can be a single string, which will be used to separate - all formatted records. - The same applies to 'record_prefix' and 'record_suffix'. + Returns a list of formatted records given by a list of record IDs + or a list of records as xml. Adds a prefix before each record, a + suffix after each record, plus a separator between records. + + Also add optional prologue and epilogue to the complete formatted + list. + + You can either specify a list of record IDs to format, or a list + of xml records, but not both (if both are specified recIDs is + ignored). + + 'record_separator' is a function that returns a string as + separator between records. The function must take an integer as + unique parameter, which is the index in recIDs (or xml_records) of + the record that has just been formatted. For example separator(i) + must return the separator between recID[i] and recID[i+1]. + Alternatively separator can be a single string, which will be used + to separate all formatted records. The same applies to + 'record_prefix' and 'record_suffix'. 'req' is an optional parameter on which the result of the function are printed lively (prints records after records) if it is given. - Note that you should set 'req' content-type by yourself, and send http header before calling - this function as it will not do it. + Note that you should set 'req' content-type by yourself, and send + http header before calling this function as it will not do it. This function takes the same parameters as 'format_record' except for: @param recIDs a list of record IDs @param xml_records a list of xml string representions of the records to format @param header a string printed before all formatted records @param separator either a string or a function that returns string to separate formatted records @param req an optional request object where to print records @param on_the_fly if False, try to return an already preformatted version of the record in the database """ if req is not None: req.write(prologue) formatted_records = '' #Fill one of the lists with Nones if xml_records is not None: recIDs = map(lambda x:None, xml_records) else: xml_records = map(lambda x:None, recIDs) total_rec = len(recIDs) last_iteration = False for i in range(total_rec): if i == total_rec - 1: last_iteration = True #Print prefix if record_prefix is not None: if isinstance(record_prefix, str): formatted_records += record_prefix if req is not None: req.write(record_prefix) else: string_prefix = record_prefix(i) formatted_records += string_prefix if req is not None: req.write(string_prefix) #Print formatted record - formatted_record = format_record(recIDs[i], of, ln, verbose, search_pattern, xml_records[i], uid, on_the_fly) + formatted_record = format_record(recIDs[i], of, ln, verbose, \ + search_pattern, xml_records[i],\ + user_info, on_the_fly) formatted_records += formatted_record if req is not None: req.write(formatted_record) #Print suffix if record_suffix is not None: if isinstance(record_suffix, str): formatted_records += record_suffix if req is not None: req.write(record_suffix) else: string_suffix = record_suffix(i) formatted_records += string_suffix if req is not None: req.write(string_suffix) #Print separator if needed if record_separator is not None and not last_iteration: if isinstance(record_separator, str): formatted_records += record_separator if req is not None: req.write(record_separator) else: string_separator = record_separator(i) formatted_records += string_separator if req is not None: req.write(string_separator) if req is not None: req.write(epilogue) return prologue + formatted_records + epilogue def create_excel(recIDs, req=None, ln=cdslang): """ Returns an Excel readable format containing the given recIDs. If 'req' is given, also prints the output in 'req' while individual records are being formatted. This method shows how to create a custom formatting of multiple records. The excel format is a basic HTML table that most spreadsheets applications can parse. @param recIDs a list of record IDs @return a string in Excel format """ # Prepare the column headers to display in the Excel file column_headers_list = ['Title', 'Authors', 'Addresses', 'Affiliation', 'Date', 'Publisher', 'Place', 'Abstract', 'Keywords', 'Notes'] # Prepare Content column_headers = ''.join(column_headers_list) + '' column_headers = '\n'+ '' footer = '
' + column_headers + '
' #Apply content_type and print column headers if req is not None: req.content_type = get_output_format_content_type('excel') req.headers_out["Content-Disposition"] = "inline; filename=%s" % 'results.xls' req.send_http_header() #Format the records excel_formatted_records = format_records(recIDs, 'excel', ln=cdslang, record_separator='\n', prologue = '', epilogue = footer, req=req) return excel_formatted_records # Utility functions ## def get_output_format_content_type(of): """ Returns the content type (eg. 'text/html' or 'application/ms-excel') \ of the given output format. @param of the code of output format for which we want to get the content type """ content_type = bibformat_dblayer.get_output_format_content_type(of) if content_type == '': content_type = 'text/html' return content_type def usage(exitcode=1, msg=""): """Prints usage info.""" if msg: sys.stderr.write("Error: %s.\n" % msg) print """BibFormat: outputs the result of the formatting of a record. Usage: bibformat required [options] Examples: $ bibformat -i 10 -o HB $ bibformat -i 10,11,13 -o HB $ bibformat -i 10:13 $ bibformat -i 10 -o HB -v 9 Required: -i, --id=ID[ID2,ID3:ID5] ID (or range of IDs) of the record(s) to be formatted. Options: -o, --output=CODE short code of the output format used for formatting (default HB). -l, --lang=LN language used for formatting. -y, --onthefly on-the-fly formatting, avoiding caches created by BibReformat. General options: -h, --help print this help and exit -v, --verbose=LEVEL verbose level (from 0 to 9, default 0) -V --version print the script version """ sys.exit(exitcode) def main(): """main entry point for biformat via command line""" options = {} # will hold command-line options options["verbose"] = 0 options["onthefly"] = False options["lang"] = cdslang options["output"] = "HB" options["recID"] = None try: opts, args = getopt.getopt(sys.argv[1:], "hVv:yl:i:o:", ["help", "version", "verbose=", "onthefly", "lang=", "id=", "output="]) except getopt.GetoptError, err: usage(1, err) pass try: for opt in opts: if opt[0] in ["-h", "--help"]: usage(0) elif opt[0] in ["-V", "--version"]: print __revision__ sys.exit(0) elif opt[0] in ["-v", "--verbose"]: options["verbose"] = int(opt[1]) elif opt[0] in ["-y", "--onthefly"]: options["onthefly"] = True elif opt[0] in ["-l", "--lang"]: options["lang"] = opt[1] elif opt[0] in ["-i", "--id"]: recIDs = [] for recID in opt[1].split(','): if ":" in recID: start = int(recID.split(':')[0]) end = int(recID.split(':')[1]) recIDs.extend(range(start, end)) else: recIDs.append(int(recID)) options["recID"] = recIDs elif opt[0] in ["-o", "--output"]: options["output"] = opt[1] if options["recID"] == None: usage(1, "-i argument is needed") except StandardError, e: usage(e) print format_records(recIDs=options["recID"], of=options["output"], ln=options["lang"], verbose=options["verbose"], on_the_fly=options["onthefly"]) return if __name__ == "__main__": main() diff --git a/modules/bibformat/lib/bibformat_engine.py b/modules/bibformat/lib/bibformat_engine.py index 50a56dba0..ef5fbf90f 100644 --- a/modules/bibformat/lib/bibformat_engine.py +++ b/modules/bibformat/lib/bibformat_engine.py @@ -1,1973 +1,1991 @@ # -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Formats a single XML Marc record using specified format. There is no API for the engine. Instead use bibformat.py. SEE: bibformat.py, bibformat_utils.py """ __revision__ = "$Id$" import re import sys import os import inspect import traceback import zlib import cgi from invenio.config import \ CFG_PATH_PHP, \ bindir, \ cdslang from invenio.errorlib import \ register_errors, \ get_msgs_for_code_list from invenio.bibrecord import \ create_record, \ record_get_field_instances, \ record_get_field_value, \ record_get_field_values from invenio.bibformat_xslt_engine import format from invenio.dbquery import run_sql from invenio.messages import \ language_list_long, \ wash_language from invenio import bibformat_dblayer from invenio.bibformat_config import \ CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION, \ CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION, \ CFG_BIBFORMAT_TEMPLATES_PATH, \ CFG_BIBFORMAT_ELEMENTS_PATH, \ CFG_BIBFORMAT_OUTPUTS_PATH, \ CFG_BIBFORMAT_ELEMENTS_IMPORT_PATH from invenio.bibformat_utils import \ record_get_xml, \ parse_tag from invenio.htmlutils import HTMLWasher +from invenio.webuser import collect_user_info if CFG_PATH_PHP: #Remove when call_old_bibformat is removed from xml.dom import minidom import tempfile # Cache for data we have already read and parsed format_templates_cache = {} format_elements_cache = {} format_outputs_cache = {} kb_mappings_cache = {} cdslangs = language_list_long() html_field = '' # String indicating that field should be # treated as HTML (and therefore no escaping of # HTML tags should occur. # Appears in some field values. washer = HTMLWasher() # Used to remove dangerous tags from HTML # sources # Regular expression for finding ... tag in format templates pattern_lang = re.compile(r''' #closing start tag (?P.*?) #anything but the next group (greedy) () #end tag ''', re.IGNORECASE | re.DOTALL | re.VERBOSE) # Builds regular expression for finding each known language in tags ln_pattern_text = r"<(" for lang in cdslangs: ln_pattern_text += lang[0] +r"|" ln_pattern_text = ln_pattern_text.rstrip(r"|") ln_pattern_text += r")>(.*?)" ln_pattern = re.compile(ln_pattern_text, re.IGNORECASE | re.DOTALL) # Regular expression for finding tag in format templates pattern_format_template_name = re.compile(r''' #closing start tag (?P.*?) #name value. any char that is not end tag ()(\n)? #end tag ''', re.IGNORECASE | re.DOTALL | re.VERBOSE) # Regular expression for finding tag in format templates pattern_format_template_desc = re.compile(r''' #closing start tag (?P.*?) #description value. any char that is not end tag (\n)? #end tag ''', re.IGNORECASE | re.DOTALL | re.VERBOSE) # Regular expression for finding tags in format templates pattern_tag = re.compile(r''' [^/\s]+) #any char but a space or slash \s* #any number of spaces (?P(\s* #params here (?P([^=\s])*)\s* #param name: any chars that is not a white space or equality. Followed by space(s) =\s* #equality: = followed by any number of spaces (?P[\'"]) #one of the separators (?P.*?) #param value: any chars that is not a separator like previous one (?P=sep) #same separator as starting one )*) #many params \s* #any number of spaces (/)?> #end of the tag ''', re.IGNORECASE | re.DOTALL | re.VERBOSE) # Regular expression for finding params inside tags in format templates pattern_function_params = re.compile(''' (?P([^=\s])*)\s* # Param name: any chars that is not a white space or equality. Followed by space(s) =\s* # Equality: = followed by any number of spaces (?P[\'"]) # One of the separators (?P.*?) # Param value: any chars that is not a separator like previous one (?P=sep) # Same separator as starting one ''', re.VERBOSE | re.DOTALL ) # Regular expression for finding format elements "params" attributes # (defined by @param) pattern_format_element_params = re.compile(''' @param\s* # Begins with @param keyword followed by space(s) (?P[^\s=]*)\s* # A single keyword, and then space(s) #(=\s*(?P[\'"]) # Equality, space(s) and then one of the separators #(?P.*?) # Default value: any chars that is not a separator like previous one #(?P=sep) # Same separator as starting one #)?\s* # Default value for param is optional. Followed by space(s) (?P.*) # Any text that is not end of line (thanks to MULTILINE parameter) ''', re.VERBOSE | re.MULTILINE) # Regular expression for finding format elements "see also" attribute # (defined by @see) pattern_format_element_seealso = re.compile('''@see\s*(?P.*)''', re.VERBOSE | re.MULTILINE) #Regular expression for finding 2 expressions in quotes, separated by #comma (as in template("1st","2nd") ) #Used when parsing output formats ## pattern_parse_tuple_in_quotes = re.compile(''' ## (?P[\'"]) ## (?P.*) ## (?P=sep1) ## \s*,\s* ## (?P[\'"]) ## (?P.*) ## (?P=sep2) ## ''', re.VERBOSE | re.MULTILINE) def call_old_bibformat(recID, format="HD", on_the_fly=False, verbose=0): """ FIXME: REMOVE FUNCTION WHEN MIGRATION IS DONE Calls BibFormat for the record RECID in the desired output format FORMAT. @param on_the_fly if False, try to return an already preformatted version of the record in the database Note: this functions always try to return HTML, so when bibformat returns XML with embedded HTML format inside the tag FMT $g, as is suitable for prestoring output formats, we perform un-XML-izing here in order to return HTML body only. """ out = "" res = [] if not on_the_fly: # look for formatted notice existence: query = "SELECT value, last_updated FROM bibfmt WHERE "\ "id_bibrec='%s' AND format='%s'" % (recID, format) res = run_sql(query, None, 1) if res: # record 'recID' is formatted in 'format', so print it if verbose == 9: last_updated = res[0][1] out += """\n
Found preformatted output for record %i (cache updated on %s). """ % (recID, last_updated) decompress = zlib.decompress return "%s" % decompress(res[0][0]) else: # record 'recID' is not formatted in 'format', # so try to call BibFormat on the fly or use default format: if verbose == 9: out += """\n
Formatting record %i on-the-fly with old BibFormat.
""" % recID # Retrieve MARCXML # Build it on-the-fly only if 'call_old_bibformat' was called # with format=xm and on_the_fly=True xm_record = record_get_xml(recID, 'xm', on_the_fly=(on_the_fly and format == 'xm')) ## import platform ## # Some problem have been found using either popen or os.system command. ## # Here is a temporary workaround until the issue is solved. ## if platform.python_compiler().find('Red Hat') > -1: ## # use os.system ## (result_code, result_path) = tempfile.mkstemp() ## command = "( %s/bibformat otype=%s ) > %s" % (bindir, format, result_path) ## (xm_code, xm_path) = tempfile.mkstemp() ## xm_file = open(xm_path, "w") ## xm_file.write(xm_record) ## xm_file.close() ## command = command + " <" + xm_path ## os.system(command) ## result_file = open(result_path,"r") ## bibformat_output = result_file.read() ## result_file.close() ## os.remove(result_path) ## os.remove(xm_path) ## else: ## # use popen pipe_input, pipe_output, pipe_error = os.popen3(["%s/bibformat" % bindir, "otype=%s" % format], 'rw') pipe_input.write(xm_record) pipe_input.flush() pipe_input.close() bibformat_output = pipe_output.read() pipe_output.close() pipe_error.close() if bibformat_output.startswith(""): dom = minidom.parseString(bibformat_output) for e in dom.getElementsByTagName('subfield'): if e.getAttribute('code') == 'g': for t in e.childNodes: out += t.data.encode('utf-8') else: out += bibformat_output return out def format_record(recID, of, ln=cdslang, verbose=0, - search_pattern=[], xml_record=None, uid=None): + search_pattern=[], xml_record=None, user_info=None): """ - Formats a record given output format. Main entry function of bibformat engine. + Formats a record given output format. Main entry function of + bibformat engine. - Returns a formatted version of the record in - the specified language, search pattern, and with the specified output format. + Returns a formatted version of the record in the specified + language, search pattern, and with the specified output format. The function will define which format template must be applied. - You can either specify an record ID to format, or give its xml representation. - if 'xml_record' is not None, then use it instead of recID. + You can either specify an record ID to format, or give its xml + representation. if 'xml_record' is not None, then use it instead + of recID. - 'uid' allows to grant access to some functionalities on a page depending - on the user's priviledges. + 'user_info' allows to grant access to some functionalities on a + page depending on the user's priviledges. 'user_info' is the same + object as the one returned by 'webuser.collect_user_info(req)' @param recID the ID of record to format @param of an output format code (or short identifier for the output format) @param ln the language to use to format the record @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, stop if error in format elements 9: errors and warnings, stop if error (debug mode )) @param search_pattern list of strings representing the user request in web interface @param xml_record an xml string representing the record to format - @param uid the user id of the person who will view the formatted page + @param user_info the information of the user who will view the formatted page @return formatted record """ out = "" errors_ = [] # Temporary workflow (during migration of formats): # Call new BibFormat # But if format not found for new BibFormat, then call old BibFormat #Create a BibFormat Object to pass that contain record and context - bfo = BibFormatObject(recID, ln, search_pattern, xml_record, uid, of) + bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of) #Find out which format template to use based on record and output format. template = decide_format_template(bfo, of) if verbose == 9 and template is not None: out += """\n
Using %s template for record %i. """ % (template, recID) ############### FIXME: REMOVE WHEN MIGRATION IS DONE ############### path = "%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, template) if template is None or not os.access(path, os.R_OK): # template not found in new BibFormat. Call old one if verbose == 9: if template is None: out += """\n
No template found for output format %s and record %i. (Check invenio.err log file for more details) """ % (of, recID) else: out += """\n
Template %s could not be read. """ % (template) if CFG_PATH_PHP: if verbose == 9: out += """\n
Using old BibFormat for record %s. """ % recID return out + call_old_bibformat(recID, format=of, on_the_fly=True, verbose=verbose) ############################# END ################################## error = get_msgs_for_code_list([("ERR_BIBFORMAT_NO_TEMPLATE_FOUND", of)], stream='error', ln=cdslang) errors_.append(error) if verbose == 0: register_errors(error, 'error') elif verbose > 5: return out + error[0][1] return out # Format with template (out_, errors) = format_with_format_template(template, bfo, verbose) errors_.extend(errors) out += out_ return out def decide_format_template(bfo, of): """ Returns the format template name that should be used for formatting given output format and BibFormatObject. Look at of rules, and take the first matching one. If no rule matches, returns None To match we ignore lettercase and spaces before and after value of rule and value of record @param bfo a BibFormatObject @param of the code of the output format to use """ output_format = get_output_format(of) for rule in output_format['rules']: value = bfo.field(rule['field']).strip()#Remove spaces pattern = rule['value'].strip() #Remove spaces match_obj = re.match(pattern, value, re.IGNORECASE) if match_obj is not None and \ match_obj.start() == 0 and match_obj.end() == len(value): return rule['template'] template = output_format['default'] if template != '': return template else: return None def format_with_format_template(format_template_filename, bfo, verbose=0, format_template_code=None): """ Format a record given a format template. Also returns errors Returns a formatted version of the record represented by bfo, in the language specified in bfo, and with the specified format template. If format_template_code is provided, the template will not be loaded from format_template_filename (but format_template_filename will still be used to determine if bft or xsl transformation applies). This allows to preview format code without having to save file on disk. @param format_template_filename the dilename of a format template @param bfo the object containing parameters for the current formatting @param format_template_code if not empty, use code as template instead of reading format_template_filename (used for previews) @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return tuple (formatted text, errors) """ errors_ = [] if format_template_code is not None: format_content = str(format_template_code) else: format_content = get_format_template(format_template_filename)['code'] if format_template_filename is None or \ format_template_filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION): # .bft localized_format = filter_languages(format_content, bfo.lang) (evaluated_format, errors) = eval_format_template_elements(localized_format, bfo, verbose) errors_ = errors else: #.xsl # Fetch MARCXML. On-the-fly xm if we are now formatting in xm xml_record = record_get_xml(bfo.recID, 'xm', on_the_fly=(bfo.format != 'xm')) # Transform MARCXML using stylesheet evaluated_format = format(xml_record, template_source=format_content) return (evaluated_format, errors_) def eval_format_template_elements(format_template, bfo, verbose=0): """ Evalutes the format elements of the given template and replace each element with its value. Also returns errors. Prepare the format template content so that we can directly replace the marc code by their value. This implies: 1) Look for special tags 2) replace special tags by their evaluation @param format_template the format template code @param bfo the object containing parameters for the current formatting @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return tuple (result, errors) """ errors_ = [] # First define insert_element_code(match), used in re.sub() function def insert_element_code(match): """ Analyses 'match', interpret the corresponding code, and return the result of the evaluation. Called by substitution in 'eval_format_template_elements(...)' @param match a match object corresponding to the special tag that must be interpreted """ function_name = match.group("function_name") try: format_element = get_format_element(function_name, verbose) except Exception, e: if verbose >= 5: return '' + \ cgi.escape(str(e)).replace('\n', '
') + \ '' if format_element is None: error = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_RESOLVE_ELEMENT_NAME", function_name)], stream='error', ln=cdslang) errors_.append(error) if verbose >= 5: return '' + \ error[0][1]+'' else: params = {} # Look for function parameters given in format template code all_params = match.group('params') if all_params is not None: function_params_iterator = pattern_function_params.finditer(all_params) for param_match in function_params_iterator: name = param_match.group('param') value = param_match.group('value') params[name] = value # Evaluate element with params and return (Do not return errors) (result, errors) = eval_format_element(format_element, bfo, params, verbose) errors_.append(errors) return result # Substitute special tags in the format by our own text. # Special tags have the form format = pattern_tag.sub(insert_element_code, format_template) return (format, errors_) def eval_format_element(format_element, bfo, parameters={}, verbose=0): """ Returns the result of the evaluation of the given format element name, with given BibFormatObject and parameters. Also returns the errors of the evaluation. @param format_element a format element structure as returned by get_format_element @param bfo a BibFormatObject used for formatting @param parameters a dict of parameters to be used for formatting. Key is parameter and value is value of parameter @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return tuple (result, errors) """ errors = [] #Load special values given as parameters prefix = parameters.get('prefix', "") suffix = parameters.get('suffix', "") default_value = parameters.get('default', "") escape = parameters.get('escape', "") output_text = '' # 3 possible cases: # a) format element file is found: we execute it # b) format element file is not found, but exist in tag table (e.g. bfe_isbn) # c) format element is totally unknown. Do nothing or report error if format_element is not None and format_element['type'] == "python": # a) We found an element with the tag name, of type "python" # Prepare a dict 'params' to pass as parameter to 'format' # function of element params = {} # Look for parameters defined in format element # Fill them with specified default values and values # given as parameters for param in format_element['attrs']['params']: name = param['name'] default = param['default'] params[name] = parameters.get(name, default) # Add BibFormatObject params['bfo'] = bfo # Execute function with given parameters and return result. function = format_element['code'] try: output_text = apply(function, (), params) except Exception, e: name = format_element['attrs']['name'] error = ("ERR_BIBFORMAT_EVALUATING_ELEMENT", name, str(params)) errors.append(error) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: tb = sys.exc_info()[2] error_string = get_msgs_for_code_list(error, stream='error', ln=cdslang) stack = traceback.format_exception(Exception, e, tb, limit=None) output_text = ''+ \ str(error_string[0][1]) + "".join(stack) +' ' # None can be returned when evaluating function if output_text is None: output_text = "" else: output_text = str(output_text) # Escaping: # (1) By default, everything is escaped in mode 1 # (2) If evaluated element has 'escape_values()' function, use # its returned value as escape mode, and override (1) # (3) If template has a defined parameter (in allowed values), # use it, and override (1) and (2) # (1) escape_mode = 1 # (2) escape_function = format_element['escape_function'] if escape_function is not None: try: escape_mode = apply(escape_function, (), {'bfo': bfo}) except Exception, e: error = ("ERR_BIBFORMAT_EVALUATING_ELEMENT_ESCAPE", name) errors.append(error) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: tb = sys.exc_info()[2] error_string = get_msgs_for_code_list(error, stream='error', ln=cdslang) output_text += ''+ \ str(error_string[0][1]) +' ' # (3) if escape in ['0', '1', '2', '3', '4']: escape_mode = int(escape) #If escape is equal to 1, then escape all # HTML reserved chars. if escape_mode > 0: output_text = escape_field(output_text, mode=escape_mode) # Add prefix and suffix if they have been given as parameters and if # the evaluation of element is not empty if output_text.strip() != "": output_text = prefix + output_text + suffix # Add the default value if output_text is empty if output_text == "": output_text = default_value return (output_text, errors) elif format_element is not None and format_element['type'] == "field": # b) We have not found an element in files that has the tag # name. Then look for it in the table "tag" # # # # Load special values given as parameters separator = parameters.get('separator ', "") nbMax = parameters.get('nbMax', "") escape = parameters.get('escape', "1") # By default, escape here # Get the fields tags that have to be printed tags = format_element['attrs']['tags'] output_text = [] # Get values corresponding to tags for tag in tags: p_tag = parse_tag(tag) values = record_get_field_values(bfo.get_record(), p_tag[0], p_tag[1], p_tag[2], p_tag[3]) if len(values)>0 and isinstance(values[0], dict): #flatten dict to its values only values_list = map(lambda x: x.values(), values) #output_text.extend(values) for values in values_list: output_text.extend(values) else: output_text.extend(values) if nbMax != "": try: nbMax = int(nbMax) output_text = output_text[:nbMax] except: name = format_element['attrs']['name'] error = ("ERR_BIBFORMAT_NBMAX_NOT_INT", name) errors.append(error) if verbose < 5: register_errors(error, 'error') elif verbose >= 5: error_string = get_msgs_for_code_list(error, stream='error', ln=cdslang) output_text = output_text.append(error_string[0][1]) # Add prefix and suffix if they have been given as parameters and if # the evaluation of element is not empty. # If evaluation is empty string, return default value if it exists. # Else return empty string if ("".join(output_text)).strip() != "": # If escape is equal to 1, then escape all # HTML reserved chars. if escape == '1': output_text = cgi.escape(separator.join(output_text)) else: output_text = separator.join(output_text) output_text = prefix + output_text + suffix else: #Return default value output_text = default_value return (output_text, errors) else: # c) Element is unknown error = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_RESOLVE_ELEMENT_NAME", format_element)], stream='error', ln=cdslang) errors.append(error) if verbose < 5: register_errors(error, 'error') return ("", errors) elif verbose >= 5: if verbose >= 9: sys.exit(error[0][1]) return ('' + \ error[0][1]+'', errors) def filter_languages(format_template, ln='en'): """ Filters the language tags that do not correspond to the specified language. @param format_template the format template code @param ln the language that is NOT filtered out from the template @return the format template with unnecessary languages filtered out """ # First define search_lang_tag(match) and clean_language_tag(match), used # in re.sub() function def search_lang_tag(match): """ Searches for the ... tag and remove inner localized tags such as , , that are not current_lang. If current_lang cannot be found inside ... , try to use 'cdslang' @param match a match object corresponding to the special tag that must be interpreted """ current_lang = ln def clean_language_tag(match): """ Return tag text content if tag language of match is output language. Called by substitution in 'filter_languages(...)' @param match a match object corresponding to the special tag that must be interpreted """ if match.group(1) == current_lang: return match.group(2) else: return "" # End of clean_language_tag lang_tag_content = match.group("langs") # Try to find tag with current lang. If it does not exists, # then current_lang becomes cdslang until the end of this # replace pattern_current_lang = re.compile(r"<("+current_lang+ \ r")\s*>(.*?)()", re.IGNORECASE | re.DOTALL) if re.search(pattern_current_lang, lang_tag_content) is None: current_lang = cdslang cleaned_lang_tag = ln_pattern.sub(clean_language_tag, lang_tag_content) return cleaned_lang_tag # End of search_lang_tag filtered_format_template = pattern_lang.sub(search_lang_tag, format_template) return filtered_format_template def get_format_template(filename, with_attributes=False): """ Returns the structured content of the given formate template. if 'with_attributes' is true, returns the name and description. Else 'attrs' is not returned as key in dictionary (it might, if it has already been loaded previously) {'code':"Some template code" 'attrs': {'name': "a name", 'description': "a description"} } @param filename the filename of an format template @param with_attributes if True, fetch the attributes (names and description) for format' @return strucured content of format template """ # Get from cache whenever possible global format_templates_cache if not filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION) and \ not filename.endswith(".xsl"): return None if format_templates_cache.has_key(filename): # If we must return with attributes and template exist in # cache with attributes then return cache. # Else reload with attributes if with_attributes and \ format_templates_cache[filename].has_key('attrs'): return format_templates_cache[filename] format_template = {'code':""} try: path = "%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, filename) format_file = open(path) format_content = format_file.read() format_file.close() # Load format template code # Remove name and description if filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION): code_and_description = pattern_format_template_name.sub("", format_content) code = pattern_format_template_desc.sub("", code_and_description) else: code = format_content format_template['code'] = code except Exception, e: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", filename, str(e))], stream='error', ln=cdslang) register_errors(errors, 'error') # Save attributes if necessary if with_attributes: format_template['attrs'] = get_format_template_attrs(filename) # Cache and return format_templates_cache[filename] = format_template return format_template def get_format_templates(with_attributes=False): """ Returns the list of all format templates, as dictionary with filenames as keys if 'with_attributes' is true, returns the name and description. Else 'attrs' is not returned as key in each dictionary (it might, if it has already been loaded previously) [{'code':"Some template code" 'attrs': {'name': "a name", 'description': "a description"} }, ... } @param with_attributes if True, fetch the attributes (names and description) for formats """ format_templates = {} files = os.listdir(CFG_BIBFORMAT_TEMPLATES_PATH) for filename in files: if filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION) or \ filename.endswith(".xsl"): format_templates[filename] = get_format_template(filename, with_attributes) return format_templates def get_format_template_attrs(filename): """ Returns the attributes of the format template with given filename The attributes are {'name', 'description'} Caution: the function does not check that path exists or that the format element is valid. @param the path to a format element """ attrs = {} attrs['name'] = "" attrs['description'] = "" try: template_file = open("%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, filename)) code = template_file.read() template_file.close() match = None if filename.endswith(".xsl"): # .xsl attrs['name'] = filename[:-4] else: # .bft match = pattern_format_template_name.search(code) if match is not None: attrs['name'] = match.group('name') else: attrs['name'] = filename match = pattern_format_template_desc.search(code) if match is not None: attrs['description'] = match.group('desc').rstrip('.') except Exception, e: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", filename, str(e))], stream='error', ln=cdslang) register_errors(errors, 'error') attrs['name'] = filename return attrs def get_format_element(element_name, verbose=0, with_built_in_params=False): """ Returns the format element structured content. Return None if element cannot be loaded (file not found, not readable or invalid) The returned structure is {'attrs': {some attributes in dict. See get_format_element_attrs_from_*} 'code': the_function_code, 'type':"field" or "python" depending if element is defined in file or table, 'escape_function': the function to call to know if element output must be escaped} @param element_name the name of the format element to load @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @param with_built_in_params if True, load the parameters built in all elements @return a dictionary with format element attributes """ # Get from cache whenever possible global format_elements_cache errors = [] # Resolve filename and prepare 'name' as key for the cache filename = resolve_format_element_filename(element_name) if filename is not None: name = filename.upper() else: name = element_name.upper() if format_elements_cache.has_key(name): element = format_elements_cache[name] if not with_built_in_params or \ (with_built_in_params and \ element['attrs'].has_key('builtin_params')): return element if filename is None: # Element is maybe in tag table if bibformat_dblayer.tag_exists_for_name(element_name): format_element = {'attrs': get_format_element_attrs_from_table( \ element_name, with_built_in_params), 'code':None, 'escape_function':None, 'type':"field"} # Cache and returns format_elements_cache[name] = format_element return format_element else: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_FORMAT_ELEMENT_NOT_FOUND", element_name)], stream='error', ln=cdslang) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: sys.stderr.write(errors[0][1]) return None else: format_element = {} module_name = filename if module_name.endswith(".py"): module_name = module_name[:-3] # Load element try: module = __import__(CFG_BIBFORMAT_ELEMENTS_IMPORT_PATH + \ "." + module_name) # Load last module in import path # For eg. load bfe_name in # invenio.bibformat_elements.bfe_name # Used to keep flexibility regarding where elements # directory is (for eg. test cases) components = CFG_BIBFORMAT_ELEMENTS_IMPORT_PATH.split(".") for comp in components[1:]: module = getattr(module, comp) except Exception, e: # We catch all exceptions here, as we just want to print # traceback in all cases tb = sys.exc_info()[2] stack = traceback.format_exception(Exception, e, tb, limit=None) errors = get_msgs_for_code_list([("ERR_BIBFORMAT_IN_FORMAT_ELEMENT", element_name,"\n" + "\n".join(stack[-2:-1]))], stream='error', ln=cdslang) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: sys.stderr.write(errors[0][1]) if errors: if verbose >= 7: raise Exception, errors[0][1] return None # Load function 'format()' inside element try: function_format = module.__dict__[module_name].format format_element['code'] = function_format except AttributeError, e: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_FORMAT_ELEMENT_FORMAT_FUNCTION", element_name)], stream='warning', ln=cdslang) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: sys.stderr.write(errors[0][1]) if errors: if verbose >= 7: raise Exception, errors[0][1] return None # Load function 'escape_values()' inside element function_escape = getattr(module.__dict__[module_name], 'escape_values', None) format_element['escape_function'] = function_escape # Prepare, cache and return format_element['attrs'] = get_format_element_attrs_from_function( \ function_format, element_name, with_built_in_params) format_element['type'] = "python" format_elements_cache[name] = format_element return format_element def get_format_elements(with_built_in_params=False): """ Returns the list of format elements attributes as dictionary structure Elements declared in files have priority over element declared in 'tag' table The returned object has this format: {element_name1: {'attrs': {'description':..., 'seealso':... 'params':[{'name':..., 'default':..., 'description':...}, ...] 'builtin_params':[{'name':..., 'default':..., 'description':...}, ...] }, 'code': code_of_the_element }, element_name2: {...}, ...} Returns only elements that could be loaded (not error in code) @return a dict of format elements with name as key, and a dict as attributes @param with_built_in_params if True, load the parameters built in all elements """ format_elements = {} mappings = bibformat_dblayer.get_all_name_tag_mappings() for name in mappings: format_elements[name.upper().replace(" ", "_").strip()] = get_format_element(name, with_built_in_params=with_built_in_params) files = os.listdir(CFG_BIBFORMAT_ELEMENTS_PATH) for filename in files: filename_test = filename.upper().replace(" ", "_") if filename_test.endswith(".PY") and filename.upper() != "__INIT__.PY": if filename_test.startswith("BFE_"): filename_test = filename_test[4:] element_name = filename_test[:-3] element = get_format_element(element_name, with_built_in_params=with_built_in_params) if element is not None: format_elements[element_name] = element return format_elements def get_format_element_attrs_from_function(function, element_name, with_built_in_params=False): """ Returns the attributes of the function given as parameter. It looks for standard parameters of the function, default values and comments in the docstring. The attributes are {'description', 'seealso':['element.py', ...], 'params':{name:{'name', 'default', 'description'}, ...], name2:{}} The attributes are {'name' : "name of element" #basically the name of 'name' parameter 'description': "a string description of the element", 'seealso' : ["element_1.py", "element_2.py", ...] #a list of related elements 'params': [{'name':"param_name", #a list of parameters for this element (except 'bfo') 'default':"default value", 'description': "a description"}, ...], 'builtin_params': {name: {'name':"param_name",#the parameters builtin for all elem of this kind 'default':"default value", 'description': "a description"}, ...}, } @param function the formatting function of a format element @param element_name the name of the element @param with_built_in_params if True, load the parameters built in all elements """ attrs = {} attrs['description'] = "" attrs['name'] = element_name.replace(" ", "_").upper() attrs['seealso'] = [] docstring = function.__doc__ if isinstance(docstring, str): # Look for function description in docstring #match = pattern_format_element_desc.search(docstring) description = docstring.split("@param")[0] description = description.split("@see")[0] attrs['description'] = description.strip().rstrip('.') # Look for @see in docstring match = pattern_format_element_seealso.search(docstring) if match is not None: elements = match.group('see').rstrip('.').split(",") for element in elements: attrs['seealso'].append(element.strip()) params = {} # Look for parameters in function definition (args, varargs, varkw, defaults) = inspect.getargspec(function) # Prepare args and defaults_list such that we can have a mapping # from args to defaults args.reverse() if defaults is not None: defaults_list = list(defaults) defaults_list.reverse() else: defaults_list = [] for arg, default in map(None, args, defaults_list): if arg == "bfo": #Don't keep this as parameter. It is hidden to users, and #exists in all elements of this kind continue param = {} param['name'] = arg if default is None: #In case no check is made inside element, we prefer to #print "" (nothing) than None in output param['default'] = "" else: param['default'] = default param['description'] = "(no description provided)" params[arg] = param if isinstance(docstring, str): # Look for @param descriptions in docstring. # Add description to existing parameters in params dict params_iterator = pattern_format_element_params.finditer(docstring) for match in params_iterator: name = match.group('name') if params.has_key(name): params[name]['description'] = match.group('desc').rstrip('.') attrs['params'] = params.values() # Load built-in parameters if necessary if with_built_in_params: builtin_params = [] # Add 'prefix' parameter param_prefix = {} param_prefix['name'] = "prefix" param_prefix['default'] = "" param_prefix['description'] = """A prefix printed only if the record has a value for this element""" builtin_params.append(param_prefix) # Add 'suffix' parameter param_suffix = {} param_suffix['name'] = "suffix" param_suffix['default'] = "" param_suffix['description'] = """A suffix printed only if the record has a value for this element""" builtin_params.append(param_suffix) # Add 'default' parameter param_default = {} param_default['name'] = "default" param_default['default'] = "" param_default['description'] = """A default value printed if the record has no value for this element""" builtin_params.append(param_default) # Add 'escape' parameter param_escape = {} param_escape['name'] = "escape" param_escape['default'] = "" param_escape['description'] = """If set to 1, replaces special characters '&', '<' and '>' of this element by SGML entities""" builtin_params.append(param_escape) attrs['builtin_params'] = builtin_params return attrs def get_format_element_attrs_from_table(element_name, with_built_in_params=False): """ Returns the attributes of the format element with given name in 'tag' table. Returns None if element_name does not exist in tag table. The attributes are {'name' : "name of element" #basically the name of 'element_name' parameter 'description': "a string description of the element", 'seealso' : [] #a list of related elements. Always empty in this case 'params': [], #a list of parameters for this element. Always empty in this case 'builtin_params': [{'name':"param_name", #the parameters builtin for all elem of this kind 'default':"default value", 'description': "a description"}, ...], 'tags':["950.1", 203.a] #the list of tags printed by this element } @param element_name an element name in database @param element_name the name of the element @param with_built_in_params if True, load the parameters built in all elements """ attrs = {} tags = bibformat_dblayer.get_tags_from_name(element_name) field_label = "field" if len(tags)>1: field_label = "fields" attrs['description'] = "Prints %s %s of the record" % (field_label, ", ".join(tags)) attrs['name'] = element_name.replace(" ", "_").upper() attrs['seealso'] = [] attrs['params'] = [] attrs['tags'] = tags # Load built-in parameters if necessary if with_built_in_params: builtin_params = [] # Add 'prefix' parameter param_prefix = {} param_prefix['name'] = "prefix" param_prefix['default'] = "" param_prefix['description'] = """A prefix printed only if the record has a value for this element""" builtin_params.append(param_prefix) # Add 'suffix' parameter param_suffix = {} param_suffix['name'] = "suffix" param_suffix['default'] = "" param_suffix['description'] = """A suffix printed only if the record has a value for this element""" builtin_params.append(param_suffix) # Add 'separator' parameter param_separator = {} param_separator['name'] = "separator" param_separator['default'] = " " param_separator['description'] = """A separator between elements of the field""" builtin_params.append(param_separator) # Add 'nbMax' parameter param_nbMax = {} param_nbMax['name'] = "nbMax" param_nbMax['default'] = "" param_nbMax['description'] = """The maximum number of values to print for this element. No limit if not specified""" builtin_params.append(param_nbMax) # Add 'default' parameter param_default = {} param_default['name'] = "default" param_default['default'] = "" param_default['description'] = """A default value printed if the record has no value for this element""" builtin_params.append(param_default) # Add 'escape' parameter param_escape = {} param_escape['name'] = "escape" param_escape['default'] = "" param_escape['description'] = """If set to 1, replaces special characters '&', '<' and '>' of this element by SGML entities""" builtin_params.append(param_escape) attrs['builtin_params'] = builtin_params return attrs def get_output_format(code, with_attributes=False, verbose=0): """ Returns the structured content of the given output format If 'with_attributes' is true, also returns the names and description of the output formats, else 'attrs' is not returned in dict (it might, if it has already been loaded previously). if output format corresponding to 'code' is not found return an empty structure. See get_output_format_attrs() to learn more on the attributes {'rules': [ {'field': "980__a", 'value': "PREPRINT", 'template': "filename_a.bft", }, {...} ], 'attrs': {'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}} 'description': "a description" 'code': "fnm1", 'content_type': "application/ms-excel", 'visibility': 1 } 'default':"filename_b.bft" } @param code the code of an output_format @param with_attributes if True, fetch the attributes (names and description) for format @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return strucured content of output format """ output_format = {'rules':[], 'default':""} filename = resolve_output_format_filename(code, verbose) if filename is None: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_OUTPUT_FORMAT_CODE_UNKNOWN", code)], stream='error', ln=cdslang) register_errors(errors, 'error') if with_attributes: #Create empty attrs if asked for attributes output_format['attrs'] = get_output_format_attrs(code, verbose) return output_format # Get from cache whenever possible global format_outputs_cache if format_outputs_cache.has_key(filename): # If was must return with attributes but cache has not # attributes, then load attributes if with_attributes and not \ format_outputs_cache[filename].has_key('attrs'): format_outputs_cache[filename]['attrs'] = get_output_format_attrs(code, verbose) return format_outputs_cache[filename] try: if with_attributes: output_format['attrs'] = get_output_format_attrs(code, verbose) path = "%s%s%s" % (CFG_BIBFORMAT_OUTPUTS_PATH, os.sep, filename ) format_file = open(path) current_tag = '' for line in format_file: line = line.strip() if line == "": # Ignore blank lines continue if line.endswith(":"): # Retrieve tag # Remove : spaces and eol at the end of line clean_line = line.rstrip(": \n\r") # The tag starts at second position current_tag = "".join(clean_line.split()[1:]).strip() elif line.find('---') != -1: words = line.split('---') template = words[-1].strip() condition = ''.join(words[:-1]) value = "" output_format['rules'].append({'field': current_tag, 'value': condition, 'template': template, }) elif line.find(':') != -1: # Default case default = line.split(':')[1].strip() output_format['default'] = default except Exception, e: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", filename, str(e))], stream='error', ln=cdslang) register_errors(errors, 'error') # Cache and return format_outputs_cache[filename] = output_format return output_format def get_output_format_attrs(code, verbose=0): """ Returns the attributes of an output format. The attributes contain 'code', which is the short identifier of the output format (to be given as parameter in format_record function to specify the output format), 'description', a description of the output format, 'visibility' the visibility of the format in the output format list on public pages and 'names', the localized names of the output format. If 'content_type' is specified then the search_engine will send a file with this content type and with result of formatting as content to the user. The 'names' dict always contais 'generic', 'ln' (for long name) and 'sn' (for short names) keys. 'generic' is the default name for output format. 'ln' and 'sn' contain long and short localized names of the output format. Only the languages for which a localization exist are used. {'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}} 'description': "a description" 'code': "fnm1", 'content_type': "application/ms-excel", 'visibility': 1 } @param code the short identifier of the format @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return strucured content of output format attributes """ if code.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION): code = code[:-(len(CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION) + 1)] attrs = {'names':{'generic':"", 'ln':{}, 'sn':{}}, 'description':'', 'code':code.upper(), 'content_type':"", 'visibility':1} filename = resolve_output_format_filename(code, verbose) if filename is None: return attrs attrs['names'] = bibformat_dblayer.get_output_format_names(code) attrs['description'] = bibformat_dblayer.get_output_format_description(code) attrs['content_type'] = bibformat_dblayer.get_output_format_content_type(code) attrs['visibility'] = bibformat_dblayer.get_output_format_visibility(code) return attrs def get_output_formats(with_attributes=False): """ Returns the list of all output format, as a dictionary with their filename as key If 'with_attributes' is true, also returns the names and description of the output formats, else 'attrs' is not returned in dicts (it might, if it has already been loaded previously). See get_output_format_attrs() to learn more on the attributes {'filename_1.bfo': {'rules': [ {'field': "980__a", 'value': "PREPRINT", 'template': "filename_a.bft", }, {...} ], 'attrs': {'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}} 'description': "a description" 'code': "fnm1" } 'default':"filename_b.bft" }, 'filename_2.bfo': {...}, ... } @return the list of output formats """ output_formats = {} files = os.listdir(CFG_BIBFORMAT_OUTPUTS_PATH) for filename in files: if filename.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION): code = "".join(filename.split(".")[:-1]) output_formats[filename] = get_output_format(code, with_attributes) return output_formats def get_kb_mapping(kb, string, default=""): """ Returns the value of the string' in the knowledge base 'kb'. If kb does not exist or string does not exist in kb, returns 'default' string value. @param kb a knowledge base name @param string a key in a knowledge base @param default a default value if 'string' is not in 'kb' @return the value corresponding to the given string in given kb """ global kb_mappings_cache if kb_mappings_cache.has_key(kb): kb_cache = kb_mappings_cache[kb] if kb_cache.has_key(string): value = kb_mappings_cache[kb][string] if value is None: return default else: return value else: # Precreate for caching this kb kb_mappings_cache[kb] = {} value = bibformat_dblayer.get_kb_mapping_value(kb, string) kb_mappings_cache[kb][str(string)] = value if value is None: return default else: return value def resolve_format_element_filename(string): """ Returns the filename of element corresponding to string This is necessary since format templates code call elements by ignoring case, for eg. is the same as . It is also recommended that format elements filenames are prefixed with bfe_ . We need to look for these too. The name of the element has to start with "BFE_". @param name a name for a format element @return the corresponding filename, with right case """ if not string.endswith(".py"): name = string.replace(" ", "_").upper() +".PY" else: name = string.replace(" ", "_").upper() files = os.listdir(CFG_BIBFORMAT_ELEMENTS_PATH) for filename in files: test_filename = filename.replace(" ", "_").upper() if test_filename == name or \ test_filename == "BFE_" + name or \ "BFE_" + test_filename == name: return filename # No element with that name found # Do not log error, as it might be a normal execution case: # element can be in database return None def resolve_output_format_filename(code, verbose=0): """ Returns the filename of output corresponding to code This is necessary since output formats names are not case sensitive but most file systems are. @param code the code for an output format @param verbose the level of verbosity from 0 to 9 (O: silent, 5: errors, 7: errors and warnings, 9: errors and warnings, stop if error (debug mode )) @return the corresponding filename, with right case, or None if not found """ #Remove non alphanumeric chars (except .) code = re.sub(r"[^.0-9a-zA-Z]", "", code) if not code.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION): code = re.sub(r"\W", "", code) code += "."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION files = os.listdir(CFG_BIBFORMAT_OUTPUTS_PATH) for filename in files: if filename.upper() == code.upper(): return filename # No output format with that name found errors = get_msgs_for_code_list([("ERR_BIBFORMAT_CANNOT_RESOLVE_OUTPUT_NAME", code)], stream='error', ln=cdslang) if verbose == 0: register_errors(errors, 'error') elif verbose >= 5: sys.stderr.write(errors[0][1]) if verbose >= 9: sys.exit(errors[0][1]) return None def get_fresh_format_template_filename(name): """ Returns a new filename and name for template with given name. Used when writing a new template to a file, so that the name has no space, is unique in template directory Returns (unique_filename, modified_name) @param a name for a format template @return the corresponding filename, and modified name if necessary """ #name = re.sub(r"\W", "", name) #Remove non alphanumeric chars name = name.replace(" ", "_") filename = name # Remove non alphanumeric chars (except .) filename = re.sub(r"[^.0-9a-zA-Z]", "", filename) path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename \ + "." + CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION index = 1 while os.path.exists(path): index += 1 filename = name + str(index) path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename \ + "." + CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION if index > 1: returned_name = (name + str(index)).replace("_", " ") else: returned_name = name.replace("_", " ") return (filename + "." + CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION, returned_name) #filename.replace("_", " ")) def get_fresh_output_format_filename(code): """ Returns a new filename for output format with given code. Used when writing a new output format to a file, so that the code has no space, is unique in output format directory. The filename also need to be at most 6 chars long, as the convention is that filename == output format code (+ .extension) We return an uppercase code Returns (unique_filename, modified_code) @param code the code of an output format @return the corresponding filename, and modified code if necessary """ #code = re.sub(r"\W", "", code) #Remove non alphanumeric chars code = code.upper().replace(" ", "_") # Remove non alphanumeric chars (except .) code = re.sub(r"[^.0-9a-zA-Z]", "", code) if len(code) > 6: code = code[:6] filename = code path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename \ + "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION index = 2 while os.path.exists(path): filename = code + str(index) if len(filename) > 6: filename = code[:-(len(str(index)))]+str(index) index += 1 path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename \ + "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION # We should not try more than 99999... Well I don't see how we # could get there.. Sanity check. if index >= 99999: errors = get_msgs_for_code_list([("ERR_BIBFORMAT_NB_OUTPUTS_LIMIT_REACHED", code)], stream='error', ln=cdslang) register_errors(errors, 'error') sys.exit("Output format cannot be named as %s"%code) return (filename + "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION, filename) def clear_caches(): """ Clear the caches (Output Format, Format Templates and Format Elements) """ global format_templates_cache, format_elements_cache , \ format_outputs_cache, kb_mappings_cache format_templates_cache = {} format_elements_cache = {} format_outputs_cache = {} kb_mappings_cache = {} class BibFormatObject: """ An object that encapsulates a record and associated methods, and that is given as parameter to all format elements 'format' function. The object is made specifically for a given formatting, i.e. it includes for example the language for the formatting. The object provides basic accessors to the record. For full access, one can get the record with get_record() and then use BibRecord methods on the returned object. """ # The record record = None # The language in which the formatting has to be done lang = cdslang # A list of string describing the context in which the record has # to be formatted. # It represents the words of the user request in web interface search search_pattern = [] # The id of the record recID = 0 - # The user id of the person who will view the formatted page (if applicable) - # This allows for example to print a "edit record" link for people - # who have right to edit a record. - uid = None + uid = None # DEPRECATED: use bfo.user_info['uid'] instead + + # The information about the user, as returned by + # 'webuser.collect_user_info(req)' + user_info = None # The format in which the record is being formatted format = '' - # The mod_python request object - req = None + req = None # DEPRECATED: use bfo.user_info instead def __init__(self, recID, ln=cdslang, search_pattern=[], - xml_record=None, uid=None, format='', req=None): + xml_record=None, user_info=None, format=''): """ Creates a new bibformat object, with given record. You can either specify an record ID to format, or give its xml representation. if 'xml_record' is not None, use 'xml_record' instead of recID for the record. - 'uid' allows to grant access to some functionalities on a page depending - on the user's priviledges. + 'user_info' allows to grant access to some functionalities on + a page depending on the user's priviledges. It is a dictionary + in the following form: + user_info = { + 'remote_ip' : '', + 'remote_host' : '', + 'referer' : '', + 'uri' : '', + 'agent' : '', + 'apache_user' : '', + 'apache_group' : [], + 'uid' : -1, + 'nickname' : '', + 'email' : '', + 'group' : [], + 'guest' : '1' + } @param recID the id of a record @param ln the language in which the record has to be formatted @param search_pattern list of string representing the request used by the user in web interface @param xml_record a xml string of the record to format - @param uid the user id of the person who will view the formatted page + @param user_info the information of the user who will view the formatted page @param format the format used for formatting this record """ if xml_record is not None: # If record is given as parameter self.record = create_record(xml_record)[0] - # raise repr(create_record(xml_record.decode('utf-8').encode('utf-8'))) - recID = record_get_field_value(self.record,"001") - + recID = record_get_field_value(self.record, "001") self.lang = wash_language(ln) self.search_pattern = search_pattern self.recID = recID - self.uid = uid self.format = format - self.req = req + self.user_info = user_info + if self.user_info is None: + self.user_info = collect_user_info(None) def get_record(self): """ Returns the record of this BibFormatObject instance @return the record structure as returned by BibRecord """ # Create record if necessary if self.record is None: # on-the-fly creation if current output is xm record = create_record(record_get_xml(self.recID, 'xm', on_the_fly=(self.format.lower() == 'xm'))) self.record = record[0] return self.record def control_field(self, tag, escape=0): """ Returns the value of control field given by tag in record @param tag the marc code of a field @param escape 1 if returned value should be escaped. Else 0. @return value of field tag in record """ if self.get_record() is None: #Case where BibRecord could not parse object return '' p_tag = parse_tag(tag) field_value = record_get_field_value(self.get_record(), p_tag[0], p_tag[1], p_tag[2], p_tag[3]) if escape == 0: return field_value else: return escape_field(field_value, escape) def field(self, tag, escape=0): """ Returns the value of the field corresponding to tag in the current record. If the value does not exist, return empty string 'escape' parameter allows to escape special characters of the field. The value of escape can be: 0 - no escaping 1 - escape all HTML characters 2 - escape all HTML characters by default. If field starts with , escape only unsafe characters, but leave basic HTML tags. @param tag the marc code of a field @param escape 1 if returned value should be escaped. Else 0. (see above for other modes) @return value of field tag in record """ list_of_fields = self.fields(tag) if len(list_of_fields) > 0: # Escaping below if escape == 0: return list_of_fields[0] else: return escape_field(list_of_fields[0], escape) else: return "" def fields(self, tag, escape=0, repeatable_subfields_p=False): """ Returns the list of values corresonding to "tag". If tag has an undefined subcode (such as 999C5), the function returns a list of dictionaries, whoose keys are the subcodes and the values are the values of tag.subcode. If the tag has a subcode, simply returns list of values corresponding to tag. Eg. for given MARC: 999C5 $a value_1a $b value_1b 999C5 $b value_2b 999C5 $b value_3b $b value_3b_bis >> bfo.fields('999C5b') >> ['value_1b', 'value_2b', 'value_3b', 'value_3b_bis'] >> bfo.fields('999C5') >> [{'a':'value_1a', 'b':'value_1b'}, {'b':'value_2b'}, {'b':'value_3b'}] By default the function returns only one value for each subfield (that is it considers that repeatable subfields are not allowed). It is why in the above example 'value3b_bis' is not shown for bfo.fields('999C5'). (Note that it is not defined which of value_3b or value_3b_bis is returned). This is to simplify the use of the function, as most of the time subfields are not repeatable (in that way we get a string instead of a list). You can allow repeatable subfields by setting 'repeatable_subfields_p' parameter to True. In this mode, the above example would return: >> bfo.fields('999C5b', repeatable_subfields_p=True) >> ['value_1b', 'value_2b', 'value_3b'] >> bfo.fields('999C5', repeatable_subfields_p=True) >> [{'a':['value_1a'], 'b':['value_1b']}, {'b':['value_2b']}, {'b':['value_3b', 'value3b_bis']}] NOTICE THAT THE RETURNED STRUCTURE IS DIFFERENT. Also note that whatever the value of 'repeatable_subfields_p' is, bfo.fields('999C5b') always show all fields, even repeatable ones. This is because the parameter has no impact on the returned structure (it is always a list). 'escape' parameter allows to escape special characters of the fields. The value of escape can be: 0 - no escaping 1 - escape all HTML characters 2 - escape all dangerous HTML tags. 3 - Mix of mode 1 and 2. If value of field starts with , then use mode 2. Else use mode 1. 4 - Remove all HTML tags @param tag the marc code of a field @param escape 1 if returned values should be escaped. Else 0. @repeatable_subfields_p if True, returns the list of subfields in the dictionary @return values of field tag in record """ if self.get_record() is None: # Case where BibRecord could not parse object return [] p_tag = parse_tag(tag) if p_tag[3] != "": # Subcode has been defined. Simply returns list of values values = record_get_field_values(self.get_record(), p_tag[0], p_tag[1], p_tag[2], p_tag[3]) if escape == 0: return values else: return [escape_field(value, escape) for value in values] else: # Subcode is undefined. Returns list of dicts. # However it might be the case of a control field. instances = record_get_field_instances(self.get_record(), p_tag[0], p_tag[1], p_tag[2]) if repeatable_subfields_p: list_of_instances = [] for instance in instances: instance_dict = {} for subfield in instance[0]: if not instance_dict.has_key(subfield[0]): instance_dict[subfield[0]] = [] if escape == 0: instance_dict[subfield[0]].append(subfield[1]) else: instance_dict[subfield[0]].append(escape_field(subfield[1], escape)) list_of_instances.append(instance_dict) return list_of_instances else: if escape == 0: return [dict(instance[0]) for instance in instances] else: return [dict([ (subfield[0], escape_field(subfield[1], escape)) \ for subfield in instance[0] ]) \ for instance in instances] def kb(self, kb, string, default=""): """ Returns the value of the "string" in the knowledge base "kb". If kb does not exist or string does not exist in kb, returns 'default' string or empty string if not specified. @param kb a knowledge base name @param string the string we want to translate @param default a default value returned if 'string' not found in 'kb' """ if string is None: return default val = get_kb_mapping(kb, string, default) if val is None: return default else: return val def escape_field(value, mode=0): """ Utility function used to escape the value of a field in given mode. - mode 0: no escaping - mode 1: escaping all HTML/XML characters (escaped chars are shown as escaped) - mode 2: escaping dangerous HTML tags to avoid XSS, but keep basic one (such as
) Escaped characters are removed. - mode 3: mix of mode 1 and mode 2. If field_value starts with , then use mode 2. Else use mode 1. - mode 4: escaping all HTML/XML tags (escaped tags are removed) - """ if mode == 1: return cgi.escape(value) elif mode == 2: return washer.wash(value, allowed_attribute_whitelist=['href', 'name', 'class'] ) elif mode == 3: if value.lstrip(' \n').startswith(html_field): return washer.wash(value, allowed_attribute_whitelist=['href', 'name', 'class'] ) else: return cgi.escape(value) elif mode == 4: return washer.wash(value, allowed_attribute_whitelist=[], allowed_tag_whitelist=[] ) else: return value def bf_profile(): """ Runs a benchmark """ for i in range(1, 51): format_record(i, "HD", ln=cdslang, verbose=9, search_pattern=[]) return if __name__ == "__main__": import profile import pstats #bf_profile() profile.run('bf_profile()', "bibformat_profile") p = pstats.Stats("bibformat_profile") p.strip_dirs().sort_stats("cumulative").print_stats() diff --git a/modules/bibformat/lib/bibformat_regression_tests.py b/modules/bibformat/lib/bibformat_regression_tests.py index 338a9ebd8..3820d463f 100644 --- a/modules/bibformat/lib/bibformat_regression_tests.py +++ b/modules/bibformat/lib/bibformat_regression_tests.py @@ -1,410 +1,410 @@ # -*- coding: utf-8 -*- ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """WebSearch module regression tests.""" __revision__ = "$Id$" import unittest from invenio.config import weburl, cdslang from invenio.testutils import make_test_suite, \ warn_user_about_tests_and_run, \ test_web_page_content from invenio.bibformat import format_record class BibFormatAPITest(unittest.TestCase): """Check BibFormat API""" def test_basic_formatting(self): """bibformat - Checking BibFormat API""" result = format_record(recID=73, of='hx', ln=cdslang, verbose=0, search_pattern=[], xml_record=None, - uid=None, + user_info=None, on_the_fly=True) pageurl = weburl + '/record/73?of=hx' result = test_web_page_content(pageurl, expected_text=result) class BibFormatBibTeXTest(unittest.TestCase): """Check output produced by BibFormat for BibTeX output for various records""" def setUp(self): """Prepare some ideal outputs""" self.record_74_hx = '''
 @article{Wang:74,
       author       = "Wang, B and Lin, C Y and Abdalla, E",
       title        = "Quasinormal modes of Reissner-Nordstrom Anti-de Sitter
                       Black Holes",
       journal      = "Phys. Lett., B",
       number       = "hep-th/0003295",
       volume       = "481",
       pages        = "79-88",
       year         = "2000",
 }
''' def test_bibtex_output(self): """bibformat - BibTeX output""" pageurl = weburl + '/record/74?of=hx' result = test_web_page_content(pageurl, expected_text=self.record_74_hx) self.assertEqual([], result) class BibFormatDetailedHTMLTest(unittest.TestCase): """Check output produced by BibFormat for detailed HTML ouput for various records""" def setUp(self): """Prepare some ideal outputs""" # Record 7 (Article) self.record_74_hd_header = '''
Published Article / Particle Physics - Theory hep-th/0003295
''' self.record_74_hd_title = '''
Quasinormal modes of Reissner-Nordstrom Anti-de Sitter Black Holes
''' self.record_74_hd_authors = '''Wang, B (Fudan University) ; Lin, C Y ; Abdalla, E
'''% \ {'weburl' : weburl, 'lang': cdslang} self.record_74_hd_abstract = '''Abstract: Complex frequencies associated with quasinormal modes for large Reissner-Nordstr$\ddot{o}$m Anti-de Sitter black holes have been computed. These frequencies have close relation to the black hole charge and do not linearly scale withthe black hole temperature as in Schwarzschild Anti-de Sitter case. In terms of AdS/CFT correspondence, we found that the bigger the black hole charge is, the quicker for the approach to thermal equilibrium in the CFT. The propertiesof quasinormal modes for $l>0$ have also been studied.
''' self.record_74_hd_pubinfo = '''Published in: Phys. Lett., B :481 2000 79-88''' self.record_74_hd_fulltext = '''0003295.pdf">Cited by: try citation search for hep-th/0003295'''% \ {'weburl' : weburl, 'lang': cdslang} self.record_74_hd_references = '''
  • [17] A. Chamblin, R. Emparan, C. V. Johnson and R. C. Myers, Phys. Rev., D60: 104026 (1999) 5070 90 110 130 150 r+ 130 230 330 50 70 90 110 130 150 r+
    • ''' # Record 7 (Picture) self.record_7_hd_header = '''
    Pictures / Life at CERN CERN-GE-9806033
    ''' self.record_7_hd_title = '''
    Tim Berners-Lee
    ''' self.record_7_hd_date = '''
    28 Jun 1998
    ''' self.record_7_hd_abstract = '''

    Caption
    Conference "Internet, Web, What's next?" on 26 June 1998 at CERN : Tim Berners-Lee, inventor of the World-Wide Web and Director of the W3C, explains how the Web came to be and give his views on the future.

    Légende
    Conference "Internet, Web, What's next?" le 26 juin 1998 au CERN: Tim Berners-Lee, inventeur du World-Wide Web et directeur du W3C, explique comment le Web est ne, et donne ses opinions sur l'avenir.

    ''' self.record_7_hd_resource = '''
    © CERN Geneva''' % weburl self.record_7_hd_resource_link = '%s/record/7/files/9806033.jpeg' % weburl def test_detailed_html_output(self): """bibformat - Detailed HTML output""" # Test record 74 (Article) pageurl = weburl + '/record/74?of=hd' result = test_web_page_content(pageurl, expected_text=[self.record_74_hd_header, self.record_74_hd_title, self.record_74_hd_authors, self.record_74_hd_abstract, self.record_74_hd_pubinfo, self.record_74_hd_fulltext, #self.record_74_hd_citations, #self.record_74_hd_references ]) self.assertEqual([], result) # Test record 7 (Picture) pageurl = weburl + '/record/7?of=hd' result = test_web_page_content(pageurl, expected_text=[self.record_7_hd_header, self.record_7_hd_title, self.record_7_hd_date, self.record_7_hd_abstract, self.record_7_hd_resource, self.record_7_hd_resource_link]) self.assertEqual([], result) def test_detailed_html_edit_record(self): """bibformat - Detailed HTML output edit record link presence""" pageurl = weburl + '/record/74?of=hd' result = test_web_page_content(pageurl, username='admin', expected_text="Edit This Record") self.assertEqual([], result) def test_detailed_html_no_error_message(self): """bibformat - Detailed HTML output without error message""" # No error message should be displayed in the web interface, whatever happens pageurl = weburl + '/record/74?of=hd' result = test_web_page_content(pageurl, username='admin', expected_text=["Exception", "Could not"]) self.assertNotEqual([], result) pageurl = weburl + '/record/7?of=hd' result = test_web_page_content(pageurl, username='admin', expected_text=["Exception", "Could not"]) self.assertNotEqual([], result) class BibFormatNLMTest(unittest.TestCase): """Check output produced by BibFormat for NLM output for various records""" def setUp(self): """Prepare some ideal outputs""" self.record_70_xn = '''
    J. High Energy Phys. J. High Energy Phys. 1126-6708 AdS/CFT For Non-Boundary Manifolds McInnes B National University of Singapore 2000 05 In its Euclidean formulation, the AdS/CFT correspondence begins as a study of Yang-Mills conformal field theories on the sphere, S^4. It has been successfully extended, however, to S^1 X S^3 and to the torus T^4. It is natural tohope that it can be made to work for any manifold on which it is possible to define a stable Yang-Mills conformal field theory. We consider a possible classification of such manifolds, and show how to deal with the most obviousobjection : the existence of manifolds which cannot be represented as boundaries. We confirm Witten's suggestion that this can be done with the help of a brane in the bulk. research-article
    ''' % {'weburl': weburl} def test_nlm_output(self): """bibformat - NLM output""" pageurl = weburl + '/record/70?of=xn' result = test_web_page_content(pageurl, expected_text=self.record_70_xn) self.assertEqual([], result) class BibFormatBriefHTMLTest(unittest.TestCase): """Check output produced by BibFormat for brief HTML ouput for various records""" def setUp(self): """Prepare some ideal outputs""" self.record_76_hb = '''Ιθάκη / Καβάφης, Κ Π
    Σα βγεις στον πηγαιμό για την Ιθάκη,
    να εύχεσαι νάναι μακρύς ο δρόμος,
    γεμάτος περιπέτειες, γεμάτος γνώσεις [...]     ''' % (weburl, cdslang) def test_brief_html_output(self): """bibformat - Brief HTML output""" pageurl = weburl + '/record/76?of=HB' result = test_web_page_content(pageurl, expected_text=self.record_76_hb) self.assertEqual([], result) class BibFormatMARCXMLTest(unittest.TestCase): """Check output produced by BibFormat for MARCXML ouput for various records""" def setUp(self): """Prepare some ideal outputs""" self.record_9_xm = ''' 9 eng PRE-25553 RL-82-024 Ellis, J University of Oxford Grand unification with large supersymmetry breaking Mar 1982 18 p SzGeCERN General Theoretical Physics Ibanez, L E Ross, G G 1982 11 Oxford Univ. Univ. Auton. Madrid Rutherford Lab. 1990-01-28 50 2002-01-04 BATCH h 1982n PREPRINT ''' def test_marcxml_output(self): """bibformat - MARCXML output""" pageurl = weburl + '/record/9?of=xm' result = test_web_page_content(pageurl, expected_text=self.record_9_xm) self.assertEqual([], result) class BibFormatMARCTest(unittest.TestCase): """Check output produced by BibFormat for MARC ouput for various records""" def setUp(self): """Prepare some ideal outputs""" self.record_29_hm = '''000000029 001__ 29 000000029 041__ $$aeng 000000029 080__ $$a517.11 000000029 100__ $$aKleene, Stephen Cole$$uUniversity of Wisconsin 000000029 245__ $$aIntroduction to metamathematics 000000029 260__ $$aAmsterdam$$bNorth-Holland$$c1952 (repr.1964.) 000000029 300__ $$a560 p 000000029 490__ $$aBibl. Matematica$$v1 000000029 909C0 $$y1952 000000029 909C0 $$b21 000000029 909C1 $$c1990-01-27$$l00$$m2002-04-12$$oBATCH 000000029 909CS $$sm$$w198606 000000029 980__ $$aBOOK''' def test_marc_output(self): """bibformat - MARC output""" pageurl = weburl + '/record/29?of=hm' result = test_web_page_content(pageurl, expected_text=self.record_29_hm) self.assertEqual([], result) class BibFormatTitleFormattingTest(unittest.TestCase): """Check title formatting produced by BibFormat.""" def test_subtitle_in_html_brief(self): """bibformat - title subtitle in HTML brief formats""" self.assertEqual([], test_web_page_content(weburl + '/search?p=statistics+computer', expected_text="Statistics: a computer approach")) def test_subtitle_in_html_detailed(self): """bibformat - title subtitle in HTML detailed formats""" self.assertEqual([], test_web_page_content(weburl + '/search?p=statistics+computer&of=HD', expected_text="Statistics: a computer approach")) def test_title_edition_in_html_brief(self): """bibformat - title edition in HTML brief formats""" self.assertEqual([], test_web_page_content(weburl + '/search?p=2nd', expected_text="Introductory statistics: a decision map; 2nd ed")) def test_title_edition_in_html_detailed(self): """bibformat - title edition in HTML detailed formats""" self.assertEqual([], test_web_page_content(weburl + '/search?p=2nd&of=HD', expected_text="Introductory statistics: a decision map; 2nd ed")) test_suite = make_test_suite(BibFormatBibTeXTest, BibFormatDetailedHTMLTest, BibFormatBriefHTMLTest, BibFormatNLMTest, BibFormatMARCTest, BibFormatMARCXMLTest, BibFormatAPITest, BibFormatTitleFormattingTest) if __name__ == "__main__": warn_user_about_tests_and_run(test_suite) diff --git a/modules/bibformat/lib/bibformatadminlib.py b/modules/bibformat/lib/bibformatadminlib.py index 24ffe9618..1e9b4195b 100644 --- a/modules/bibformat/lib/bibformatadminlib.py +++ b/modules/bibformat/lib/bibformatadminlib.py @@ -1,1618 +1,1621 @@ # -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Handle requests from the web interface to configure BibFormat. """ __revision__ = "$Id$" import os import re import stat import time import cgi from invenio.config import cdslang, weburl, etcdir from invenio.bibformat_config import \ CFG_BIBFORMAT_TEMPLATES_PATH, \ CFG_BIBFORMAT_OUTPUTS_PATH, \ CFG_BIBFORMAT_ELEMENTS_PATH, \ CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION from invenio.urlutils import wash_url_argument from invenio.errorlib import get_msgs_for_code_list from invenio.messages import gettext_set_language, wash_language, language_list_long from invenio.search_engine import perform_request_search, encode_for_xml from invenio import bibformat_dblayer from invenio import bibformat_engine import invenio.template bibformat_templates = invenio.template.load('bibformat') def getnavtrail(previous = '', ln=cdslang): """Get the navtrail""" previous = wash_url_argument(previous, 'str') ln = wash_language(ln) _ = gettext_set_language(ln) navtrail = '''%s > %s ''' % \ (weburl, _("Admin Area"), weburl, ln, _("BibFormat Admin")) navtrail = navtrail + previous return navtrail def perform_request_index(ln=cdslang, warnings=None, is_admin=False): """ Returns the main BibFormat admin page. @param ln language @param warnings a list of messages to display at top of the page, that prevents writability in etc @param is_admin indicate if user is authorized to use BibFormat @return the main admin page """ if warnings is not None and len(warnings) > 0: warnings = get_msgs_for_code_list(warnings, 'warning', ln) warnings = [x[1] for x in warnings] # Get only message, not code return bibformat_templates.tmpl_admin_index(ln, warnings, is_admin) def perform_request_format_templates_management(ln=cdslang, checking=0): """ Returns the main management console for format templates @param ln language @param checking the level of checking (0: basic, 1:extensive (time consuming) ) @return the main page for format templates management """ # Reload in case a format was changed bibformat_engine.clear_caches() # Get formats lists of attributes formats = bibformat_engine.get_format_templates(with_attributes=True) formats_attrs = [] for filename in formats: attrs = formats[filename]['attrs'] attrs['filename'] = filename if filename.endswith('.xsl'): attrs['name'] += ' (XSL)' attrs['editable'] = can_write_format_template(filename) path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename try: attrs['last_mod_date'] = time.ctime(os.stat(path)[stat.ST_MTIME]) except OSError: # File does not exist. Happens with temporary files # created by editors. continue status = check_format_template(filename, checking) if len(status) > 1 or (len(status)==1 and status[0][0] != 'ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE'): status = ''' Not OK ''' % {'weburl':weburl, 'ln':ln, 'bft':filename} else: status = 'OK' attrs['status'] = status formats_attrs.append(attrs) def sort_by_attr(seq): """ Sort 'seq' by attribute name. @param seq a list of dictionaries, containing each one key named 'name' """ intermed = [ (x['name'].lower(), i, x) for i, x in enumerate(seq)] intermed.sort() return [x[-1] for x in intermed] sorted_format_templates = sort_by_attr(formats_attrs) return bibformat_templates.tmpl_admin_format_templates_management(ln, sorted_format_templates) def perform_request_format_template_show(bft, ln=cdslang, code=None, ln_for_preview=cdslang, pattern_for_preview="", content_type_for_preview="text/html"): """ Returns the editor for format templates. @param ln language @param bft the template to edit @param code, the code being edited @param ln_for_preview the language for the preview (for bfo) @param pattern_for_preview the search pattern to be used for the preview (for bfo) @return the main page for formats management """ format_template = bibformat_engine.get_format_template(filename=bft, with_attributes=True) # Either use code being edited, or the original code inside template if code is None: code = cgi.escape(format_template['code']) # Build a default pattern if it is empty if pattern_for_preview == "": recIDs = perform_request_search() if len(recIDs) > 0: recID = recIDs[0] pattern_for_preview = "recid:%s" % recID editable = can_write_format_template(bft) # Look for all existing content_types content_types = bibformat_dblayer.get_existing_content_types() # Add some standard content types if not already there standard_content_types = ['text/xml', 'application/rss+xml', 'text/plain', 'text/html'] content_types.extend([content_type for content_type in standard_content_types if content_type not in content_types]) return bibformat_templates.tmpl_admin_format_template_show(ln, format_template['attrs']['name'], format_template['attrs']['description'], code, bft, ln_for_preview=ln_for_preview, pattern_for_preview=pattern_for_preview, editable=editable, content_type_for_preview=content_type_for_preview, content_types=content_types) def perform_request_format_template_show_dependencies(bft, ln=cdslang): """ Show the dependencies (on elements) of the given format. @param ln language @param bft the filename of the template to show """ format_template = bibformat_engine.get_format_template(filename=bft, with_attributes=True) name = format_template['attrs']['name'] output_formats = get_outputs_that_use_template(bft) format_elements = get_elements_used_by_template(bft) tags = [] for output_format in output_formats: for tag in output_format['tags']: tags.append(tag) for format_element in format_elements: for tag in format_element['tags']: tags.append(tag) tags.sort() return bibformat_templates.tmpl_admin_format_template_show_dependencies(ln, name, bft, output_formats, format_elements, tags) def perform_request_format_template_show_attributes(bft, ln=cdslang, new=False): """ Page for template name and descrition attributes edition. If format template is new, offer the possibility to make a duplicate of an existing format template. @param ln language @param bft the template to edit @param new if True, the template has just been added (is new) @return the main page for format templates attributes edition """ all_templates = [] if new: all_templates_attrs = bibformat_engine.get_format_templates(with_attributes=True) if all_templates_attrs.has_key(bft): # Sanity check. Should always be true at this stage del all_templates_attrs[bft] # Remove in order not to make a duplicate of self.. # Sort according to name, inspired from Python Cookbook def sort_by_name(seq, keys): """ Sort the sequence 'seq' by 'keys' """ intermed = [(x['attrs']['name'], keys[i], i, x) for i, x in enumerate(seq)] intermed.sort() return [(x[1], x[0]) for x in intermed] all_templates = sort_by_name(all_templates_attrs.values(), all_templates_attrs.keys()) #keys = all_templates_attrs.keys() #keys.sort() #all_templates = map(lambda x: (x, all_templates_attrs.get(x)['attrs']['name']), keys) format_template = bibformat_engine.get_format_template(filename=bft, with_attributes=True) name = format_template['attrs']['name'] description = format_template['attrs']['description'] editable = can_write_format_template(bft) return bibformat_templates.tmpl_admin_format_template_show_attributes(ln, name, description, bft, editable, all_templates, new) def perform_request_format_template_show_short_doc(ln=cdslang, search_doc_pattern=""): """ Returns the format elements documentation to be included inside format templated editor. Keep only elements that have 'search_doc_pattern' text inside description, if pattern not empty @param ln language @param search_doc_pattern a search pattern that specified which elements to display @return a brief version of the format element documentation """ # Get format elements lists of attributes elements = bibformat_engine.get_format_elements(with_built_in_params=True) keys = elements.keys() keys.sort() elements = map(elements.get, keys) def filter_elem(element): """Keep element if is string representation contains all keywords of search_doc_pattern, and if its name does not start with a number (to remove 'garbage' from elements in tags table)""" if element['type'] != 'python' and \ element['attrs']['name'][0] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']: return False text = str(element).upper() # Basic text representation if search_doc_pattern != "": for word in search_doc_pattern.split(): if word.upper() != "AND" and text.find(word.upper()) == -1: return False return True elements = filter(filter_elem, elements) return bibformat_templates.tmpl_admin_format_template_show_short_doc(ln, elements) def perform_request_format_elements_documentation(ln=cdslang): """ Returns the main management console for format elements. Includes list of format elements and associated administration tools. @param ln language @return the main page for format elements management """ # Get format elements lists of attributes elements = bibformat_engine.get_format_elements(with_built_in_params=True) keys = elements.keys() keys.sort() elements = map(elements.get, keys) # Remove all elements found in table and that begin with a number (to remove 'garbage') filtered_elements = [element for element in elements \ if element is not None and \ element['type'] == 'python' and \ element['attrs']['name'][0] not in \ ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']] return bibformat_templates.tmpl_admin_format_elements_documentation(ln, filtered_elements) def perform_request_format_element_show_dependencies(bfe, ln=cdslang): """ Show the dependencies of the given format. @param ln language @param bfe the filename of the format element to show """ format_templates = get_templates_that_use_element(bfe) tags = get_tags_used_by_element(bfe) return bibformat_templates.tmpl_admin_format_element_show_dependencies(ln, bfe, format_templates, tags) -def perform_request_format_element_test(bfe, ln=cdslang, param_values=None, uid=None, req=None): +def perform_request_format_element_test(bfe, ln=cdslang, param_values=None, user_info=None): """ Show the dependencies of the given format. 'param_values' is the list of values to pass to 'format' function of the element as parameters, in the order ... If params is None, this means that they have not be defined by user yet. @param ln language @param bfe the name of the format element to show @param params the list of parameters to pass to element format function - @param uid the user id for this request - @param req the mod_python request object + @param user_info the user_info of this request """ _ = gettext_set_language(ln) format_element = bibformat_engine.get_format_element(bfe, with_built_in_params=True) # Load parameter names and description ## param_names = [] param_descriptions = [] # First value is a search pattern to choose the record param_names.append(_("Test with record:")) # Caution: keep in sync with same text below param_descriptions.append(_("Enter a search query here.")) # Parameters defined in this element for param in format_element['attrs']['params']: param_names.append(param['name']) param_descriptions.append(param['description']) # Parameters common to all elements of a kind for param in format_element['attrs']['builtin_params']: param_names.append(param['name']) param_descriptions.append(param['description']) # Load parameters values ## if param_values is None: #First time the page is loaded param_values = [] # Propose an existing record id by default recIDs = perform_request_search() if len(recIDs) > 0: recID = recIDs[0] param_values.append("recid:%s" % recID) # Default values defined in this element for param in format_element['attrs']['params']: param_values.append(param['default']) #Parameters common to all elements of a kind for param in format_element['attrs']['builtin_params']: param_values.append(param['default']) # Execute element with parameters ## params = dict(zip(param_names, param_values)) # Find a record corresponding to search pattern search_pattern = params[_("Test with record:")] # Caution keep in sync with same text above and below recIDs = perform_request_search(p=search_pattern) del params[_("Test with record:")] # Caution keep in sync with same text above if len(recIDs) > 0: - bfo = bibformat_engine.BibFormatObject(recIDs[0], ln, search_pattern, None, uid, req) + bfo = bibformat_engine.BibFormatObject(recID = recIDs[0], + ln = ln, + search_pattern = search_pattern.split(' '), + xml_record = None, + user_info = user_info) (result, errors) = bibformat_engine.eval_format_element(format_element, bfo, params) else: result = get_msgs_for_code_list([("ERR_BIBFORMAT_NO_RECORD_FOUND_FOR_PATTERN", search_pattern)], stream='error', ln=cdslang)[0][1] return bibformat_templates.tmpl_admin_format_element_test(ln, bfe, format_element['attrs']['description'], param_names, param_values, param_descriptions, result) def perform_request_output_formats_management(ln=cdslang, sortby="code"): """ Returns the main management console for output formats. Includes list of output formats and associated administration tools. @param ln language @param sortby the sorting crieteria (can be 'code' or 'name') @return the main page for output formats management """ # Reload in case a format was changed bibformat_engine.clear_caches() # Get output formats lists of attributes output_formats_list = bibformat_engine.get_output_formats(with_attributes=True) output_formats = {} for filename in output_formats_list: output_format = output_formats_list[filename] code = output_format['attrs']['code'] path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename output_format['editable'] = can_write_output_format(code) try: output_format['last_mod_date'] = time.ctime(os.stat(path)[stat.ST_MTIME]) except OSError: # File does not exist. Happens with temporary files # created by editors. continue # Validate the output format status = check_output_format(code) # If there is an error but the error is just 'format is not writable', do not display as error if len(status) > 1 or (len(status)==1 and status[0][0] != 'ERR_BIBFORMAT_CANNOT_WRITE_OUTPUT_FILE'): status = ''' Not OK ''' % {'weburl':weburl, 'ln':ln, 'bfo':code} else: status = 'OK' output_format['status'] = status output_formats[filename] = output_format # Sort according to code or name, inspired from Python Cookbook def get_attr(dic, attr): """ Returns the value given by 'attr' in the dictionary 'dic', representing an output format attributes. If attr is equal to 'code', returns the code attribute of the dictionary. Else returns the generic name @param dic a dictionary of the attribute of an output format, as returned by bibformat_engine.get_output_format @param the attribute we want to fetch. Either 'code' or any other string """ if attr == "code": return dic['attrs']['code'] else: return dic['attrs']['names']['generic'] def sort_by_attr(seq, attr): """ Sort dictionaries given in 'seq' according to parameter 'attr' """ intermed = [ (get_attr(x, attr), i, x) for i, x in enumerate(seq)] intermed.sort() return [x[-1] for x in intermed] if sortby != "code" and sortby != "name": sortby = "code" sorted_output_formats = sort_by_attr(output_formats.values(), sortby) return bibformat_templates.tmpl_admin_output_formats_management(ln, sorted_output_formats) def perform_request_output_format_show(bfo, ln=cdslang, r_fld=[], r_val=[], r_tpl=[], default="", r_upd="", args={}): """ Returns the editing tools for a given output format. The page either shows the output format from file, or from user's POST session, as we want to let him edit the rules without saving. Policy is: r_fld, r_val, rules_tpl are list of attributes of the rules. If they are empty, load from file. Else use POST. The i th value of each list is one of the attributes of rule i. Rule i is the i th rule in order of evaluation. All list have the same number of item. r_upd contains an action that has to be performed on rules. It can composed of a number (i, the rule we want to modify) and an operator : "save" to save the rules, "add" or "del". syntax: operator [number] For eg: r_upd = _("Save Changes") saves all rules (no int should be specified). For eg: r_upd = _("Add New Rule") adds a rule (no int should be specified). For eg: r_upd = _("Remove Rule") + " 5" deletes rule at position 5. The number is used only for operation delete. An action can also be in **args. We must look there for string starting with '(+|-) [number]' to increase (+) or decrease (-) a rule given by its index (number). For example "+ 5" increase priority of rule 5 (put it at fourth position). The string in **args can be followed by some garbage that looks like .x or .y, as this is returned as the coordinate of the click on the . We HAVE to use args and reason on its keys, because for of type image, iexplorer does not return the value of the tag, but only the name. Action is executed only if we are working from user's POST session (means we must have loaded the output format first, which is totally normal and expected behaviour) IMPORTANT: we display rules evaluation index starting at 1 in interface, but we start internally at 0 @param ln language @param bfo the filename of the output format to show @param r_fld the list of 'field' attribute for each rule @param r_val the list of 'value' attribute for each rule @param r_tpl the list of 'template' attribute for each rule @param default the default format template used by this output format @param r_upd the rule that we want to increase/decrease in order of evaluation """ output_format = bibformat_engine.get_output_format(bfo, with_attributes=True) format_templates = bibformat_engine.get_format_templates(with_attributes=True) name = output_format['attrs']['names']['generic'] rules = [] debug = "" if len(r_fld) == 0 and r_upd=="": # Retrieve rules from file rules = output_format['rules'] default = output_format['default'] else: # Retrieve rules from given lists # Transform a single rule (not considered as a list with length # 1 by the templating system) into a list if not isinstance(r_fld, list): r_fld = [r_fld] r_val = [r_val] r_tpl = [r_tpl] for i in range(len(r_fld)): rule = {'field': r_fld[i], 'value': r_val[i], 'template': r_tpl[i]} rules.append(rule) # Execute action _ = gettext_set_language(ln) if r_upd.startswith(_("Remove Rule")): # Remove rule index = int(r_upd.split(" ")[-1]) -1 del rules[index] elif r_upd.startswith(_("Save Changes")): # Save update_output_format_rules(bfo, rules, default) elif r_upd.startswith(_("Add New Rule")): # Add new rule rule = {'field': "", 'value': "", 'template': ""} rules.append(rule) else: # Get the action in 'args' # The action must be constructed from string of the kind: # + 5 or - 4 or + 5.x or -4.y for button_val in args.keys():#for all elements of form not handled yet action = button_val.split(" ") if action[0] == '-' or action[0] == '+': index = int(action[1].split(".")[0]) -1 if action[0] == '-': # Decrease priority rule = rules[index] del rules[index] rules.insert(index + 1, rule) # debug = 'Decrease rule '+ str(index) break elif action[0] == '+': # Increase priority rule = rules[index] del rules[index] rules.insert(index - 1, rule) # debug = 'Increase rule ' + str(index) break editable = can_write_output_format(bfo) return bibformat_templates.tmpl_admin_output_format_show(ln, bfo, name, rules, default, format_templates, editable) def perform_request_output_format_show_dependencies(bfo, ln=cdslang): """ Show the dependencies of the given format. @param ln language @param bfo the filename of the output format to show """ output_format = bibformat_engine.get_output_format(code=bfo, with_attributes=True) name = output_format['attrs']['names']['generic'] format_templates = get_templates_used_by_output(bfo) return bibformat_templates.tmpl_admin_output_format_show_dependencies(ln, name, bfo, format_templates) def perform_request_output_format_show_attributes(bfo, ln=cdslang): """ Page for output format names and description attributes edition. @param ln language @param bfo filename of output format to edit @return the main page for output format attributes edition """ output_format = bibformat_engine.get_output_format(code=bfo, with_attributes=True) name = output_format['attrs']['names']['generic'] description = output_format['attrs']['description'] content_type = output_format['attrs']['content_type'] visible = output_format['attrs']['visibility'] # Get translated names. Limit to long names now. # Translation are given in order of languages in language_list_long() names_trans = [] for lang in language_list_long(): name_trans = output_format['attrs']['names']['ln'].get(lang[0], "") names_trans.append({'lang':lang[1], 'trans':name_trans}) editable = can_write_output_format(bfo) return bibformat_templates.tmpl_admin_output_format_show_attributes(ln, name, description, content_type, bfo, names_trans, editable, visible) def perform_request_knowledge_bases_management(ln=cdslang): """ Returns the main page for knowledge bases management. @param ln language @return the main page for knowledge bases management """ kbs = bibformat_dblayer.get_kbs() return bibformat_templates.tmpl_admin_kbs_management(ln, kbs) def perform_request_knowledge_base_show(kb_id, ln=cdslang, sortby="to"): """ Show the content of a knowledge base @param ln language @param kb a knowledge base id @param sortby the sorting criteria ('from' or 'to') @return the content of the given knowledge base """ name = bibformat_dblayer.get_kb_name(kb_id) mappings = bibformat_dblayer.get_kb_mappings(name, sortby) return bibformat_templates.tmpl_admin_kb_show(ln, kb_id, name, mappings, sortby) def perform_request_knowledge_base_show_attributes(kb_id, ln=cdslang, sortby="to"): """ Show the attributes of a knowledge base @param ln language @param kb a knowledge base id @param sortby the sorting criteria ('from' or 'to') @return the content of the given knowledge base """ name = bibformat_dblayer.get_kb_name(kb_id) description = bibformat_dblayer.get_kb_description(name) return bibformat_templates.tmpl_admin_kb_show_attributes(ln, kb_id, name, description, sortby) def perform_request_knowledge_base_show_dependencies(kb_id, ln=cdslang, sortby="to"): """ Show the dependencies of a kb @param ln language @param kb a knowledge base id @param sortby the sorting criteria ('from' or 'to') @return the dependencies of the given knowledge base """ name = bibformat_dblayer.get_kb_name(kb_id) format_elements = get_elements_that_use_kb(name) return bibformat_templates.tmpl_admin_kb_show_dependencies(ln, kb_id, name, sortby, format_elements) def add_format_template(): """ Adds a new format template (mainly create file with unique name) @return the filename of the created format """ (filename, name) = bibformat_engine.get_fresh_format_template_filename("Untitled") out = '%(name)s' % {'name':name} path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename format = open(path, 'w') format.write(out) format.close return filename def delete_format_template(filename): """ Delete a format template given by its filename If format template is not writable, do not remove @param filename the format template filename """ if not can_write_format_template(filename): return path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename os.remove(path) bibformat_engine.clear_caches() def update_format_template_code(filename, code=""): """ Saves code inside template given by filename """ format_template = bibformat_engine.get_format_template_attrs(filename) name = format_template['name'] description = format_template['description'] out = ''' %(name)s %(description)s %(code)s ''' % {'name':name, 'description':description, 'code':code} path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename format = open(path, 'w') format.write(out) format.close bibformat_engine.clear_caches() def update_format_template_attributes(filename, name="", description="", duplicate=None): """ Saves name and description inside template given by filename. the filename must change according to name, and every output format having reference to filename must be updated. If name already exist, use fresh filename (we never overwrite other templates) amd remove old one. if duplicate is different from None and is not empty string, then it means that we must copy the code of the template whoose filename is given in 'duplicate' for the code of our template. @param duplicate the filename of a template that we want to copy @return the filename of the modified format """ if filename.endswith('.'+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION): format_template = bibformat_engine.get_format_template(filename, with_attributes=True) if duplicate is not None and duplicate != "": format_template_to_copy = bibformat_engine.get_format_template(duplicate) code = format_template_to_copy['code'] else: code = format_template['code'] if format_template['attrs']['name'] != name: # Name has changed, so update filename old_filename = filename old_path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + old_filename # Remove old one os.remove(old_path) (filename, name) = bibformat_engine.get_fresh_format_template_filename(name) # Change output formats that calls this template output_formats = bibformat_engine.get_output_formats() for output_format_filename in output_formats: if can_read_output_format(output_format_filename) and can_write_output_format(output_format_filename): output_path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + output_format_filename format = open(output_path, 'r') output_text = format.read() format.close output_pattern = re.compile("---(\s)*" + old_filename, re.IGNORECASE) mod_output_text = output_pattern.sub("--- " + filename, output_text) if output_text != mod_output_text: format = open(output_path, 'w') format.write(mod_output_text) format.close description = cgi.escape(description) name = cgi.escape(name) # Write updated format template out = '''%(name)s%(description)s%(code)s''' % {'name':name, 'description':description, 'code':code} path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + filename format = open(path, 'w') format.write(out) format.close bibformat_engine.clear_caches() return filename def add_output_format(): """ Adds a new output format (mainly create file with unique name) @return the code of the created format """ (filename, code) = bibformat_engine.get_fresh_output_format_filename("UNTLD") # Add entry in database bibformat_dblayer.add_output_format(code) bibformat_dblayer.set_output_format_name(code, "Untitled", lang="generic") bibformat_dblayer.set_output_format_content_type(code, "text/html") # Add file out = "" path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename format = open(path, 'w') format.write(out) format.close return code def delete_output_format(code): """ Delete a format template given by its code if file is not writable, don't remove @param code the 6 letters code of the output format to remove """ if not can_write_output_format(code): return # Remove entry from database bibformat_dblayer.remove_output_format(code) # Remove file filename = bibformat_engine.resolve_output_format_filename(code) path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename os.remove(path) bibformat_engine.clear_caches() def update_output_format_rules(code, rules=[], default=""): """ Saves rules inside output format given by code """ # Generate output format syntax # Try to group rules by field previous_field = "" out = "" for rule in rules: field = rule["field"] value = rule["value"] template = rule["template"] if previous_field != field: out += "tag %s:\n" % field out +="%(value)s --- %(template)s\n" % {'value':value, 'template':template} previous_field = field out += "default: %s" % default filename = bibformat_engine.resolve_output_format_filename(code) path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename format = open(path, 'w') format.write(out) format.close bibformat_engine.clear_caches() def update_output_format_attributes(code, name="", description="", new_code="", content_type="", names_trans=[], visibility=1): """ Saves name and description inside output format given by filename. If new_code already exist, use fresh code (we never overwrite other output). @param description the new description @param name the new name @param code the new short code (== new bfo) of the output format @param code the code of the output format to update @param names_trans the translations in the same order as the languages from get_languages() @param content_type the new content_type of the output format @param visibility the visibility of the output format in the output formats list (public pages) @return the filename of the modified format """ bibformat_dblayer.set_output_format_description(code, description) bibformat_dblayer.set_output_format_content_type(code, content_type) bibformat_dblayer.set_output_format_visibility(code, visibility) bibformat_dblayer.set_output_format_name(code, name, lang="generic") i = 0 for lang in language_list_long(): if names_trans[i] != "": bibformat_dblayer.set_output_format_name(code, names_trans[i], lang[0]) i += 1 new_code = new_code.upper() if code != new_code: # If code has changed, we must update filename with a new unique code old_filename = bibformat_engine.resolve_output_format_filename(code) old_path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + old_filename (new_filename, new_code) = bibformat_engine.get_fresh_output_format_filename(new_code) new_path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + new_filename os.rename(old_path, new_path) bibformat_dblayer.change_output_format_code(code, new_code) bibformat_engine.clear_caches() return new_code def add_kb_mapping(kb_name, key, value=""): """ Adds a new mapping to given kb @param kb_name the name of the kb where to insert the new value @param key the key of the mapping @param value the value of the mapping """ bibformat_dblayer.add_kb_mapping(kb_name, key, value) def remove_kb_mapping(kb_name, key): """ Delete an existing kb mapping in kb @param kb_name the name of the kb where to insert the new value @param key the key of the mapping """ bibformat_dblayer.remove_kb_mapping(kb_name, key) def update_kb_mapping(kb_name, old_key, key, value): """ Update an existing kb mapping with key old_key with a new key and value @param kb_name the name of the kb where to insert the new value @param the key of the mapping in the kb @param key the new key of the mapping @param value the new value of the mapping """ bibformat_dblayer.update_kb_mapping(kb_name, old_key, key, value) def kb_exists(kb_name): """Returns True if a kb with the given name exists""" return bibformat_dblayer.kb_exists(kb_name) def get_kb_name(kb_id): """ Returns the name of the kb given by id """ return bibformat_dblayer.get_kb_name(kb_id) def update_kb_attributes(kb_name, new_name, new_description): """ Updates given kb_name with a new name and new description @param kb_name the name of the kb to update @param new_name the new name for the kb @param new_description the new description for the kb """ bibformat_dblayer.update_kb(kb_name, new_name, new_description) def add_kb(kb_name="Untitled"): """ Adds a new kb in database, and returns its id The name of the kb will be 'Untitled#' such that it is unique. @param kb_name the name of the kb @return the id of the newly created kb """ name = kb_name i = 1 while bibformat_dblayer.kb_exists(name): name = kb_name + " " + str(i) i += 1 kb_id = bibformat_dblayer.add_kb(name, "") return kb_id def delete_kb(kb_name): """ Deletes given kb from database """ bibformat_dblayer.delete_kb(kb_name) def can_read_format_template(filename): """ Returns 0 if we have read permission on given format template, else returns other integer """ path = "%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, filename) return os.access(path, os.R_OK) def can_read_output_format(bfo): """ Returns 0 if we have read permission on given output format, else returns other integer """ filename = bibformat_engine.resolve_output_format_filename(bfo) path = "%s%s%s" % (CFG_BIBFORMAT_OUTPUTS_PATH, os.sep, filename) return os.access(path, os.R_OK) def can_read_format_element(name): """ Returns 0 if we have read permission on given format element, else returns other integer """ filename = bibformat_engine.resolve_format_element_filename(name) path = "%s%s%s" % (CFG_BIBFORMAT_ELEMENTS_PATH, os.sep, filename) return os.access(path, os.R_OK) def can_write_format_template(bft): """ Returns 0 if we have write permission on given format template, else returns other integer """ if not can_read_format_template(bft): return False path = "%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, bft) return os.access(path, os.W_OK) def can_write_output_format(bfo): """ Returns 0 if we have write permission on given output format, else returns other integer """ if not can_read_output_format(bfo): return False filename = bibformat_engine.resolve_output_format_filename(bfo) path = "%s%s%s" % (CFG_BIBFORMAT_OUTPUTS_PATH, os.sep, filename) return os.access(path, os.W_OK) def can_write_etc_bibformat_dir(): """ Returns true if we can write in etc/bibformat dir. """ path = "%s%sbibformat" % (etcdir, os.sep) return os.access(path, os.W_OK) def get_outputs_that_use_template(filename): """ Returns a list of output formats that call the given format template. The returned output formats also give their dependencies on tags. We don't return the complete output formats but some reference to them (filename + names) [ {'filename':"filename_1.bfo" 'names': {'en':"a name", 'fr': "un nom", 'generic':"a name"} 'tags': ['710__a', '920__'] }, ... ] Returns output formats references sorted by (generic) name @param filename a format template filename """ output_formats_list = {} tags = [] output_formats = bibformat_engine.get_output_formats(with_attributes=True) for output_format in output_formats: name = output_formats[output_format]['attrs']['names']['generic'] # First look at default template, and add it if necessary if output_formats[output_format]['default'] == filename: output_formats_list[name] = {'filename':output_format, 'names':output_formats[output_format]['attrs']['names'], 'tags':[]} # Second look at each rule found = False for rule in output_formats[output_format]['rules']: if rule['template'] == filename: found = True tags.append(rule['field']) #Also build dependencies on tags # Finally add dependency on template from rule (overwrite default dependency, # which is weaker in term of tag) if found: output_formats_list[name] = {'filename':output_format, 'names':output_formats[output_format]['attrs']['names'], 'tags':tags} keys = output_formats_list.keys() keys.sort() return map(output_formats_list.get, keys) def get_elements_used_by_template(filename): """ Returns a list of format elements that are called by the given format template. The returned elements also give their dependencies on tags. Dependencies on tag might be approximative. See get_tags_used_by_element() doc string. We must handle usage of bfe_field in a special way if we want to retrieve used tag: used tag is given in "tag" parameter, not inside element code. The list is returned sorted by name [ {'filename':"filename_1.py" 'name':"filename_1" 'tags': ['710__a', '920__'] }, ... ] Returns elements sorted by name @param filename a format template filename """ format_elements = {} format_template = bibformat_engine.get_format_template(filename=filename, with_attributes=True) code = format_template['code'] format_elements_iter = bibformat_engine.pattern_tag.finditer(code) for result in format_elements_iter: function_name = result.group("function_name").lower() if function_name is not None and not format_elements.has_key(function_name) \ and not function_name == "field": filename = bibformat_engine.resolve_format_element_filename("BFE_"+function_name) if filename is not None: tags = get_tags_used_by_element(filename) format_elements[function_name] = {'name':function_name.lower(), 'filename':filename, 'tags':tags} elif function_name == "field": # Handle bfe_field element in a special way if not format_elements.has_key(function_name): #Indicate usage of bfe_field if not already done filename = bibformat_engine.resolve_format_element_filename("BFE_"+function_name) format_elements[function_name] = {'name':function_name.lower(), 'filename':filename, 'tags':[]} # Retrieve value of parameter "tag" all_params = result.group('params') function_params_iterator = bibformat_engine.pattern_function_params.finditer(all_params) for param_match in function_params_iterator: name = param_match.group('param') if name == "tag": value = param_match.group('value') if not value in format_elements[function_name]['tags']: format_elements[function_name]['tags'].append(value) break keys = format_elements.keys() keys.sort() return map(format_elements.get, keys) # Format Elements Dependencies ## def get_tags_used_by_element(filename): """ Returns a list of tags used by given format element APPROXIMATIVE RESULTS: the tag are retrieved in field(), fields() and control_field() function. If they are used computed, or saved in a variable somewhere else, they are not retrieved @TODO: There is room for improvements. For example catch call to BibRecord functions. Returns tags sorted by value @param filename a format element filename """ tags = {} format_element = bibformat_engine.get_format_element(filename) if format_element is None: return [] elif format_element['type']=="field": tags = format_element['attrs']['tags'] return tags filename = bibformat_engine.resolve_format_element_filename(filename) path = CFG_BIBFORMAT_ELEMENTS_PATH + os.sep + filename format = open(path, 'r') code = format.read() format.close tags_pattern = re.compile(''' (field|fields|control_field)\s* #Function call \(\s* #Opening parenthesis [\'"]+ #Single or double quote (?P.+?) #Tag [\'"]+\s* #Single or double quote (,[^\)]+)* #Additional function param \) #Closing parenthesis ''', re.VERBOSE | re.MULTILINE) tags_iter = tags_pattern.finditer(code) for result in tags_iter: tags[result.group("tag")] = result.group("tag") return tags.values() def get_templates_that_use_element(name): """ Returns a list of format templates that call the given format element. The returned format templates also give their dependencies on tags. [ {'filename':"filename_1.bft" 'name': "a name" 'tags': ['710__a', '920__'] }, ... ] Returns templates sorted by name @param name a format element name """ format_templates = {} tags = [] files = os.listdir(CFG_BIBFORMAT_TEMPLATES_PATH) #Retrieve all templates for possible_template in files: if possible_template.endswith(CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION): format_elements = get_elements_used_by_template(possible_template) #Look for elements used in template format_elements = map(lambda x: x['name'].lower(), format_elements) try: #Look for element format_elements.index(name.lower()) #If not found, get out of "try" statement format_template = bibformat_engine.get_format_template(filename=possible_template, with_attributes=True) template_name = format_template['attrs']['name'] format_templates[template_name] = {'name':template_name, 'filename':possible_template} except: pass keys = format_templates.keys() keys.sort() return map(format_templates.get, keys) # Output Formats Dependencies ## def get_templates_used_by_output(code): """ Returns a list of templates used inside an output format give by its code The returned format templates also give their dependencies on elements and tags [ {'filename':"filename_1.bft" 'name': "a name" 'elements': [{'filename':"filename_1.py", 'name':"filename_1", 'tags': ['710__a', '920__'] }, ...] }, ... ] Returns templates sorted by name """ format_templates = {} output_format = bibformat_engine.get_output_format(code, with_attributes=True) filenames = map(lambda x: x['template'], output_format['rules']) if output_format['default'] != "": filenames.append(output_format['default']) for filename in filenames: template = bibformat_engine.get_format_template(filename, with_attributes=True) name = template['attrs']['name'] elements = get_elements_used_by_template(filename) format_templates[name] = {'name':name, 'filename':filename, 'elements':elements} keys = format_templates.keys() keys.sort() return map(format_templates.get, keys) # Knowledge Bases Dependencies ## def get_elements_that_use_kb(name): """ Returns a list of elements that call given kb [ {'filename':"filename_1.py" 'name': "a name" }, ... ] Returns elements sorted by name """ format_elements = {} files = os.listdir(CFG_BIBFORMAT_ELEMENTS_PATH) #Retrieve all elements in files for filename in files: if filename.endswith(".py"): path = CFG_BIBFORMAT_ELEMENTS_PATH + os.sep + filename format = open(path, 'r') code = format.read() format.close # Search for use of kb inside code kb_pattern = re.compile(''' (bfo.kb)\s* #Function call \(\s* #Opening parenthesis [\'"]+ #Single or double quote (?P%s) #kb [\'"]+\s* #Single or double quote , #comma ''' % name, re.VERBOSE | re.MULTILINE | re.IGNORECASE) result = kb_pattern.search(code) if result is not None: name = ("".join(filename.split(".")[:-1])).lower() if name.startswith("bfe_"): name = name[4:] format_elements[name] = {'filename':filename, 'name': name} keys = format_elements.keys() keys.sort() return map(format_elements.get, keys) # Validation tools ## def perform_request_format_validate(ln=cdslang, bfo=None, bft=None, bfe=None): """ Returns a page showing the status of an output format or format template or format element. This page is called from output formats management page or format template management page or format elements documentation. The page only shows the status of one of the format, depending on the specified one. If multiple are specified, shows the first one. @param ln language @param bfo an output format 6 chars code @param bft a format element filename @param bfe a format element name """ if bfo is not None: errors = check_output_format(bfo) messages = get_msgs_for_code_list(code_list = errors, ln=ln) elif bft is not None: errors = check_format_template(bft, checking=1) messages = get_msgs_for_code_list(code_list = errors, ln=ln) elif bfe is not None: errors = check_format_element(bfe) messages = get_msgs_for_code_list(code_list = errors, ln=ln) if messages is None: messages = [] messages = map(lambda x: encode_for_xml(x[1]), messages) return bibformat_templates.tmpl_admin_validate_format(ln, messages) def check_output_format(code): """ Returns the list of errors in the output format given by code The errors are the formatted errors defined in bibformat_config.py file. @param code the 6 chars code of the output format to check @return a list of errors """ errors = [] filename = bibformat_engine.resolve_output_format_filename(code) if can_read_output_format(code): path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename format = open(path) current_tag = '' i = 0 for line in format: i += 1 if line.strip() == "": # Ignore blank lines continue clean_line = line.rstrip("\n\r ") #remove spaces and eol if line.strip().endswith(":") or (line.strip().lower().startswith("tag") and line.find('---') == -1): # Check tag if not clean_line.endswith(":"): # Column misses at the end of line errors.append(("ERR_BIBFORMAT_OUTPUT_RULE_FIELD_COL", line, i)) if not clean_line.lower().startswith("tag"): # Tag keyword is missing errors.append(("ERR_BIBFORMAT_OUTPUT_TAG_MISSING", line, i)) elif not clean_line.startswith("tag"): # Tag was not lower case errors.append(("ERR_BIBFORMAT_OUTPUT_WRONG_TAG_CASE", line, i)) clean_line = clean_line.rstrip(": ") #remove : and spaces at the end of line current_tag = "".join(clean_line.split()[1:]).strip() #the tag starts at second position if len(clean_line.split()) > 2: #We should only have 'tag' keyword and tag errors.append(("ERR_BIBFORMAT_INVALID_OUTPUT_RULE_FIELD", i)) else: if len(check_tag(current_tag)) > 0: # Invalid tag errors.append(("ERR_BIBFORMAT_INVALID_OUTPUT_RULE_FIELD_tag", current_tag, i)) if not clean_line.startswith("tag"): errors.append(("ERR_BIBFORMAT_INVALID_OUTPUT_RULE_FIELD", i)) elif line.find('---') != -1: # Check condition if current_tag == "": errors.append(("ERR_BIBFORMAT_OUTPUT_CONDITION_OUTSIDE_FIELD", line, i)) words = line.split('---') if len(words) != 2: errors.append(("ERR_BIBFORMAT_INVALID_OUTPUT_CONDITION", line, i)) template = words[-1].strip() path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + template if not os.path.exists(path): errors.append(("ERR_BIBFORMAT_WRONG_OUTPUT_RULE_TEMPLATE_REF", template, i)) elif line.find(':') != -1 or (line.strip().lower().startswith("default") and line.find('---') == -1): # Check default template clean_line = line.strip() if line.find(':') == -1: # Column misses after default errors.append(("ERR_BIBFORMAT_OUTPUT_RULE_DEFAULT_COL", line, i)) if not clean_line.startswith("default"): # Default keyword is missing errors.append(("ERR_BIBFORMAT_OUTPUT_DEFAULT_MISSING", line, i)) if not clean_line.startswith("default"): # Default was not lower case errors.append(("ERR_BIBFORMAT_OUTPUT_WRONG_DEFAULT_CASE", line, i)) default = "".join(line.split(':')[1]).strip() path = CFG_BIBFORMAT_TEMPLATES_PATH + os.sep + default if not os.path.exists(path): errors.append(("ERR_BIBFORMAT_WRONG_OUTPUT_RULE_TEMPLATE_REF", default, i)) else: # Check others errors.append(("ERR_BIBFORMAT_WRONG_OUTPUT_LINE", line, i)) else: errors.append(("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", filename, "")) return errors def check_format_template(filename, checking=0): """ Returns the list of errors in the format template given by its filename The errors are the formatted errors defined in bibformat_config.py file. @param filename the filename of the format template to check @param checking the level of checking (0:basic, >=1 extensive (time-consuming)) @return a list of errors """ errors = [] if can_read_format_template(filename):#Can template be read? if filename.endswith('.'+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION): #format_template = bibformat_engine.get_format_template(filename, with_attributes=True) format = open("%s%s%s" % (CFG_BIBFORMAT_TEMPLATES_PATH, os.sep, filename)) code = format.read() format.close() # Look for name match = bibformat_engine.pattern_format_template_name.search(code) if match is None:#Is tag defined in template? errors.append(("ERR_BIBFORMAT_TEMPLATE_HAS_NO_NAME", filename)) # Look for description match = bibformat_engine.pattern_format_template_desc.search(code) if match is None:#Is tag defined in template? errors.append(("ERR_BIBFORMAT_TEMPLATE_HAS_NO_DESCRIPTION", filename)) format_template = bibformat_engine.get_format_template(filename, with_attributes=False) code = format_template['code'] # Look for calls to format elements # Check existence of elements and attributes used in call elements_call = bibformat_engine.pattern_tag.finditer(code) for element_match in elements_call: element_name = element_match.group("function_name") filename = bibformat_engine.resolve_format_element_filename(element_name) if filename is None and not bibformat_dblayer.tag_exists_for_name(element_name): #Is element defined? errors.append(("ERR_BIBFORMAT_TEMPLATE_CALLS_UNDEFINED_ELEM", filename, element_name)) else: format_element = bibformat_engine.get_format_element(element_name, with_built_in_params=True) if format_element is None:#Can element be loaded? if not can_read_format_element(element_name): errors.append(("ERR_BIBFORMAT_TEMPLATE_CALLS_UNREADABLE_ELEM", filename, element_name)) else: errors.append(("ERR_BIBFORMAT_TEMPLATE_CALLS_UNLOADABLE_ELEM", element_name, filename)) else: # Are the parameters used defined in element? params_call = bibformat_engine.pattern_function_params.finditer(element_match.group()) all_params = {} for param_match in params_call: param = param_match.group("param") value = param_match.group("value") all_params[param] = value allowed_params = [] # Built-in params for allowed_param in format_element['attrs']['builtin_params']: allowed_params.append(allowed_param['name']) # Params defined in element for allowed_param in format_element['attrs']['params']: allowed_params.append(allowed_param['name']) if not param in allowed_params: errors.append(("ERR_BIBFORMAT_TEMPLATE_WRONG_ELEM_ARG", element_name, param, filename)) # The following code is too much time consuming. Only do where really requested if checking > 0: # Try to evaluate, with any object and pattern recIDs = perform_request_search() if len(recIDs) > 0: recID = recIDs[0] bfo = bibformat_engine.BibFormatObject(recID, search_pattern="Test") (result, errors_) = bibformat_engine.eval_format_element(format_element, bfo, all_params, verbose=7) errors.extend(errors_) else:# Template cannot be read errors.append(("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", filename, "")) return errors def check_format_element(name): """ Returns the list of errors in the format element given by its name The errors are the formatted errors defined in bibformat_config.py file. @param name the name of the format element to check @return a list of errors """ errors = [] filename = bibformat_engine.resolve_format_element_filename(name) if filename is not None:#Can element be found in files? if can_read_format_element(name):#Can element be read? # Try to load try: module_name = filename if module_name.endswith(".py"): module_name = module_name[:-3] module = __import__("invenio.bibformat_elements."+module_name) function_format = module.bibformat_elements.__dict__[module_name].format # Try to evaluate, with any object and pattern recIDs = perform_request_search() if len(recIDs) > 0: recID = recIDs[0] bfo = bibformat_engine.BibFormatObject(recID, search_pattern="Test") element = bibformat_engine.get_format_element(name) (result, errors_) = bibformat_engine.eval_format_element(element, bfo, verbose=7) errors.extend(errors_) except Exception, e: errors.append(("ERR_BIBFORMAT_IN_FORMAT_ELEMENT", name, e)) else: errors.append(("ERR_BIBFORMAT_CANNOT_READ_ELEMENT_FILE", filename, "")) elif bibformat_dblayer.tag_exists_for_name(name):#Can element be found in database? pass else: errors.append(("ERR_BIBFORMAT_CANNOT_RESOLVE_ELEMENT_NAME", name)) return errors def check_tag(tag): """ Checks the validity of a tag """ errors = [] return errors def perform_request_dreamweaver_floater(): """ Returns a floater for Dreamweaver with all Format Elements avalaible. """ # Get format elements lists of attributes elements = bibformat_engine.get_format_elements(with_built_in_params=True) keys = elements.keys() keys.sort() elements = map(elements.get, keys) def filter_elem(element): """Keep element if is string representation contains all keywords of search_doc_pattern, and if its name does not start with a number (to remove 'garbage' from elements in tags table)""" if element['type'] != 'python' and \ element['attrs']['name'][0] in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']: return False else: return True elements = filter(filter_elem, elements) return bibformat_templates.tmpl_dreamweaver_floater(cdslang, elements) diff --git a/modules/bibformat/lib/elements/bfe_client_info.py b/modules/bibformat/lib/elements/bfe_client_info.py index 0309292e0..eb60775ab 100644 --- a/modules/bibformat/lib/elements/bfe_client_info.py +++ b/modules/bibformat/lib/elements/bfe_client_info.py @@ -1,43 +1,55 @@ ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -## General Public License for more details. +## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """BibFormat element - Prints client info """ __revision__ = "$Id$" def format(bfo, var=''): ''' Print several client specific variables. - @param var the name of the desired variable. Can be one of: ln, search_pattern, uid + @param var the name of the desired variable. Can be one of: ln, search_pattern, uid, referer, uri, nickname, email lang: the current language of the user search_pattern: the list of keywords used by the user uid: the current user id + referer: the url the user came from + uri: the current uri + nickname: the user nickname + email: the user email ''' - + if var == '': out = '' elif var == 'ln': out = bfo.lang elif var == 'search_pattern': out = ' '.join(bfo.search_pattern) elif var == 'uid': - out = bfo.uid + out = bfo.user_info['uid'] + elif var == 'referer': + out = bfo.user_info['referer'] + elif var == 'uri': + out = bfo.user_info['uri'] + elif var == 'nickname': + out = bfo.user_info['nickname'] + elif var == 'email': + out = bfo.user_info['email'] else: out = 'Unknown variable: %s' % (var) - + return out diff --git a/modules/bibformat/lib/elements/bfe_edit_record.py b/modules/bibformat/lib/elements/bfe_edit_record.py index 2009ad7c1..7b95c5668 100644 --- a/modules/bibformat/lib/elements/bfe_edit_record.py +++ b/modules/bibformat/lib/elements/bfe_edit_record.py @@ -1,61 +1,57 @@ # -*- coding: utf-8 -*- ## ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """BibFormat element - Prints a link to BibEdit """ __revision__ = "$Id$" def format(bfo, style): """ Prints a link to BibEdit, if authorization is granted @param style the CSS style to be applied to the link. """ from invenio.config import weburl from invenio.access_control_engine import acc_authorize_action out = "" - uid = bfo.uid - req = bfo.req - if uid is not None: - if req: - (auth_code, auth_message) = acc_authorize_action(req, 'runbibedit') - else: - (auth_code, auth_message) = acc_authorize_action(uid, 'runbibedit') - if auth_code == 0: - print_style = '' - if style != '': - print_style = 'style="' + style + '"' + user_info = bfo.user_info + (auth_code, auth_message) = acc_authorize_action(user_info['uid'], \ + 'runbibedit') + if auth_code == 0: + print_style = '' + if style != '': + print_style = 'style="' + style + '"' - out += 'Edit This Record' + out += 'Edit This Record' return out def escape_values(bfo): """ Called by BibFormat in order to check if output of this element should be escaped. """ return 0 diff --git a/modules/bibformat/web/admin/bibformatadmin.py b/modules/bibformat/web/admin/bibformatadmin.py index c819f0079..eefa8c5d0 100644 --- a/modules/bibformat/web/admin/bibformatadmin.py +++ b/modules/bibformat/web/admin/bibformatadmin.py @@ -1,1474 +1,1469 @@ ## $Id$ ## ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """CDS Invenio BibFormat Administrator Interface.""" __revision__ = "$Id$" __lastupdated__ = """$Date$""" import MySQLdb from invenio import bibformatadminlib, \ config, \ bibformat_dblayer,\ bibformat_engine from invenio.bibrankadminlib import check_user from invenio.webpage import page, create_error_box -from invenio.webuser import getUid, page_not_authorized +from invenio.webuser import getUid, page_not_authorized, collect_user_info from invenio.messages import wash_language, gettext_set_language from invenio.urlutils import wash_url_argument, redirect_to_url from invenio.search_engine import search_pattern, \ create_basic_search_units def index(req, ln=config.cdslang): """ Main BibFormat administration page. Displays a warning if we find out that etc/biformat dir is not writable by us (as most opeation of BibFormat must write in this directory). @param ln: language """ warnings = [] if not bibformatadminlib.can_write_etc_bibformat_dir(): warnings.append(("WRN_BIBFORMAT_CANNOT_WRITE_IN_ETC_BIBFORMAT")) ln = wash_language(ln) _ = gettext_set_language(ln) # Check if user is authorized to administer # If not, still display page but offer to log in try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: is_admin = True else: is_admin = False navtrail = '''%s''' % \ (config.weburl, _("Admin Area")) return page(title=_("BibFormat Admin"), body=bibformatadminlib.perform_request_index(ln=ln, warnings=warnings, is_admin=is_admin), language=ln, uid=uid, navtrail = navtrail, lastupdated=__lastupdated__, req=req, warnings=warnings) def output_formats_manage(req, ln=config.cdslang, sortby="code"): """ Main page for output formats management. Check for authentication and print output formats list. @param ln language @param sortby the sorting crieteria (can be 'code' or 'name') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail() try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: sortby = wash_url_argument(sortby, 'str') return page(title=_("Manage Output Formats"), body=bibformatadminlib.perform_request_output_formats_management(ln=ln, sortby=sortby), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def output_format_show(req, bfo, ln=config.cdslang, r_fld=[], r_val=[], r_tpl=[], default="", r_upd="", chosen_option="", **args): """ Show a single output format. Check for authentication and print output format settings. The page either shows the output format from file, or from user's POST session, as we want to let him edit the rules without saving. Policy is: r_fld, r_val, rules_tpl are list of attributes of the rules. If they are empty, load from file. Else use POST. The i th value of each list is one of the attributes of rule i. Rule i is the i th rule in order of evaluation. All list have the same number of item. r_upd contains an action that has to be performed on rules. It can composed of a number (i, the rule we want to modify) and an operator : "save" to save the rules, "add" or "del". syntax: operator [number] For eg: r_upd = _("Save Changes") saves all rules (no int should be specified). For eg: r_upd = _("Add New Rule") adds a rule (no int should be specified). For eg: r_upd = _("Remove Rule") + " 5" deletes rule at position 5. The number is used only for operation delete. An action can also be in **args. We must look there for string starting with '(+|-) [number]' to increase (+) or decrease (-) a rule given by its index (number). For example "+ 5" increase priority of rule 5 (put it at fourth position). The string in **args can be followed by some garbage that looks like .x or .y, as this is returned as the coordinate of the click on the . We HAVE to use args and reason on its keys, because for of type image, iexplorer does not return the value of the tag, but only the name. Action is executed only if we are working from user's POST session (means we must have loaded the output format first, which is totally normal and expected behaviour) @param ln language @param bfo the filename of the output format to show @param r_fld the list of 'field' attribute for each rule @param r_val the list of 'value' attribute for each rule @param r_tpl the list of 'template' attribute for each rule @param default the default format template used by this output format @param r_upd the rule that we want to increase/decrease in order of evaluation """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Output Formats"))) code = wash_url_argument(bfo, 'str') try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfo = wash_url_argument(bfo, 'str') default = wash_url_argument(default, 'str') r_upd = wash_url_argument(r_upd, 'str') if not bibformatadminlib.can_read_output_format(bfo): #No read permission return page(title=_("Restricted Output Format"), body = """You don't have permission to view this output format.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", bfo , "")], lastupdated=__lastupdated__, req=req) output_format = bibformat_engine.get_output_format(code=bfo, with_attributes=True) name = output_format['attrs']['names']['generic'] if name == "": name = bfo if not bibformatadminlib.can_write_output_format(bfo) and \ chosen_option == "":#No write permission return dialog_box(req=req, ln=ln, title="File Permission on %s" % name, message="You don't have write permission" \ "on %s.
    You can view the output " \ "format, but not edit it." % name, navtrail=navtrail_previous_links, options=[ _("Ok")]) return page(title=_('Output Format %s Rules' % name), body=bibformatadminlib.perform_request_output_format_show(bfo=bfo, ln=ln, r_fld=r_fld, r_val=r_val, r_tpl=r_tpl, default=default, r_upd=r_upd, args=args), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def output_format_show_attributes(req, bfo, ln=config.cdslang): """ Page for output format names and descrition attributes edition. @param ln language @param bfo the filename of the template to show """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln , _("Manage Output Formats"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfo = wash_url_argument(bfo, 'str') if not bibformatadminlib.can_read_output_format(bfo): #No read permission return page(title=_("Restricted Output Format"), body = """You don't have permission to view this output format.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", bfo ,"")], lastupdated=__lastupdated__, req=req) output_format = bibformat_engine.get_output_format(code=bfo, with_attributes=True) name = output_format['attrs']['names']['generic'] return page(title=_("Output Format %s Attributes" % name), body=bibformatadminlib.perform_request_output_format_show_attributes(bfo, ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links , lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg) def output_format_show_dependencies(req, bfo, ln=config.cdslang): """ Show the dependencies of the given output format. @param ln language @param bfo the filename of the output format to show """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s ''' % (config.weburl, ln, _("Manage Output Formats"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfo = wash_url_argument(bfo, 'str') if not bibformatadminlib.can_read_output_format(bfo): #No read permission return page(title=_("Restricted Output Format"), body = """You don't have permission to view this output format.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", bfo , "")], lastupdated=__lastupdated__, req=req) format_name = bibformat_engine.get_output_format_attrs(bfo)['names']['generic'] return page(title=_("Output Format %s Dependencies" % format_name), body=bibformatadminlib.perform_request_output_format_show_dependencies(bfo, ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg) def output_format_update_attributes(req, bfo, ln=config.cdslang, name = "", description="", code="", content_type="", names_trans=[], visibility="0"): """ Update the name, description and code of given output format @param ln language @param description the new description @param name the new name @param code the new short code (== new bfo) of the output format @param content_type the new content_type of the output format @param bfo the filename of the output format to update @param names_trans the translations in the same order as the languages from get_languages() @param visibility the visibility of the output format in the output formats list (public pages) """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: name = wash_url_argument(name, 'str') description = wash_url_argument(description, 'str') bfo = wash_url_argument(bfo, 'str') code = wash_url_argument(code, 'str') visibility = wash_url_argument(visibility, 'int') bfo = bibformatadminlib.update_output_format_attributes(bfo, name, description, code, content_type, names_trans, visibility) redirect_to_url(req, "output_format_show?ln=%(ln)s&bfo=%(bfo)s" % {'ln':ln, 'bfo':bfo, 'names_trans':names_trans}) else: return page_not_authorized(req=req, text=auth_msg, chosen_option="") def output_format_delete(req, bfo, ln=config.cdslang, chosen_option=""): """ Delete an output format @param bfo the filename of the output format to delete """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s > %s''' % (config.weburl, ln, _("Manage Output Formats"), _("Delete Output Format"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: #Ask confirmation to user if not already done chosen_option = wash_url_argument(chosen_option, 'str') if chosen_option == "": bfo = wash_url_argument(bfo, 'str') format_name = bibformat_dblayer.get_output_format_names(bfo)['generic'] return dialog_box(req=req, ln=ln, title="Delete %s"%format_name, message="Are you sure you want to" \ "delete output format %s?" % format_name, navtrail=navtrail_previous_links, options=[_("Cancel"), _("Delete")]) elif chosen_option==_("Delete"): bibformatadminlib.delete_output_format(bfo) redirect_to_url(req, "output_formats_manage?ln=%(ln)s"%{'ln':ln}) else: return page_not_authorized(req=req, text=auth_msg) def output_format_add(req, ln=config.cdslang): """ Adds a new output format """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfo = bibformatadminlib.add_output_format() redirect_to_url(req, "output_format_show_attributes?ln=%(ln)s&bfo=%(bfo)s" % {'ln':ln, 'bfo':bfo}) else: return page_not_authorized(req=req, text=auth_msg) def format_templates_manage(req, ln=config.cdslang, checking='0'): """ Main page for formats templates management. Check for authentication and print formats list. @param ln language @param checking if 0, basic checking. Else perform extensive checking (time-consuming) """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail() try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: checking_level = wash_url_argument(checking, 'int') return page(title=_("Manage Format Templates"), body=bibformatadminlib.perform_request_format_templates_management(ln=ln, checking=checking_level), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def format_template_show(req, bft, code=None, ln=config.cdslang, ln_for_preview=config.cdslang, pattern_for_preview="", content_type_for_preview="text/html", chosen_option=""): """ Main page for template edition. Check for authentication and print formats editor. @param ln language @param code the code being edited @param bft the name of the template to show @param ln_for_preview the language for the preview (for bfo) @param pattern_for_preview the search pattern to be used for the preview (for bfo) @param content_type_for_preview the (MIME) content type of the preview @param chosen_option returned value for dialog_box warning """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln , _("Manage Format Templates"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: format_template = wash_url_argument(bft, 'str') ln_preview = wash_language(ln_for_preview) pattern_preview = wash_url_argument(pattern_for_preview, 'str') if not bibformatadminlib.can_read_format_template(bft): #No read permission return page(title=_("Restricted Format Template"), body = """You don't have permission to view this format template.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", format_template , "")], lastupdated=__lastupdated__, req=req) format_name = bibformat_engine.get_format_template_attrs(bft)['name'] if not bibformatadminlib.can_write_format_template(bft) and \ chosen_option == "": #No write permission return dialog_box(req=req, ln=ln, title="File Permission on %s" % format_name, message="You don't have write permission " \ "on %s.
    You can view the template" \ ", but not edit it." % format_name, navtrail=navtrail_previous_links, options=[ _("Ok")]) if bft.endswith('.xsl'): format_name += ' (XSL)' return page(title=_("Format Template %s"%format_name), body=bibformatadminlib.perform_request_format_template_show(format_template, code=code, ln=ln, ln_for_preview=ln_preview, pattern_for_preview=pattern_preview, content_type_for_preview=content_type_for_preview), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def format_template_show_attributes(req, bft, ln=config.cdslang, new=0): """ Page for template name and descrition attributes edition. This is also the first page shown when a format template has just been added. In that case new is different from False and we can offer specific option to user (for ex let him make a duplicate of existing template). @param ln language @param bft the name of the template to show @param new if "False", the template has not just been added """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Format Templates"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: format_template = wash_url_argument(bft, 'str') format_name = bibformat_engine.get_format_template_attrs(bft)['name'] is_new = wash_url_argument(new, 'int') if not bibformatadminlib.can_read_format_template(bft): #No read permission return page(title=_("Restricted Format Template"), body = """You don't have permission to view this format template.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", format_template , "")], lastupdated=__lastupdated__, req=req) return page(title=_("Format Template %s Attributes"%format_name), body=bibformatadminlib.perform_request_format_template_show_attributes(bft, ln=ln, new=is_new), uid=uid, language=ln, navtrail = navtrail_previous_links , lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg) def format_template_show_dependencies(req, bft, ln=config.cdslang): """ Show the dependencies (on elements) of the given format. @param ln language @param bft the filename of the template to show """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Format Templates"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: format_template = wash_url_argument(bft, 'str') format_name = bibformat_engine.get_format_template_attrs(bft)['name'] return page(title=_("Format Template %s Dependencies" % format_name), body=bibformatadminlib.perform_request_format_template_show_dependencies(bft, ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg) def format_template_update_attributes(req, bft, ln=config.cdslang, name = "", description="", duplicate=None): """ Update the name and description of given format template @param ln language @param description the new description @param name the new name @param bft the filename of the template to update @param duplicate the filename of template that we want to copy (the code) """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: if duplicate is not None: duplicate = wash_url_argument(duplicate, 'str') name = wash_url_argument(name, 'str') description = wash_url_argument(description, 'str') bft = bibformatadminlib.update_format_template_attributes(bft, name, description, duplicate) redirect_to_url(req, "format_template_show?ln=%(ln)s&bft=%(bft)s" % {'ln':ln, 'bft':bft}) else: return page_not_authorized(req=req, text=auth_msg) def format_template_delete(req, bft, ln=config.cdslang, chosen_option=""): """ Delete a format template """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s > %s''' % (config.weburl, ln ,_("Manage Format Templates"),_("Delete Format Template"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: #Ask confirmation to user if not already done chosen_option = wash_url_argument(chosen_option, 'str') if chosen_option == "": format_template = wash_url_argument(bft, 'str') format_name = bibformat_engine.get_format_template_attrs(bft)['name'] return dialog_box(req=req, ln=ln, title="Delete %s" % format_name, message="Are you sure you want to delete" \ "format template %s?" % format_name, navtrail=navtrail_previous_links, options=[_("Cancel"), _("Delete")]) elif chosen_option==_("Delete"): bibformatadminlib.delete_format_template(bft) redirect_to_url(req, "format_templates_manage?ln=%(ln)s" % {'ln':ln}) else: return page_not_authorized(req=req, text=auth_msg) def format_template_add(req, ln=config.cdslang): """ Adds a new format template """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bft = bibformatadminlib.add_format_template() redirect_to_url(req, "format_template_show_attributes?ln=%(ln)s&bft=%(bft)s&new=1" % {'ln':ln, 'bft':bft}) else: return page_not_authorized(req=req, text=auth_msg) def format_template_show_preview_or_save(req, bft, ln=config.cdslang, code=None, ln_for_preview=config.cdslang, pattern_for_preview="", content_type_for_preview='text/html', save_action=None, navtrail=""): """ Print the preview of a record with a format template. To be included inside Format template editor. If the save_action has a value, then the code should also be saved at the same time @param code the code of a template to use for formatting @param ln_for_preview the language for the preview (for bfo) @param pattern_for_preview the search pattern to be used for the preview (for bfo) @param save_action has a value if the code has to be saved @param bft the filename of the template to save @param navtrail standard navtrail """ ln = wash_language(ln) _ = gettext_set_language(ln) - try: - uid = getUid(req) - except MySQLdb.Error, e: - return error_page(req) - (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: + user_info = collect_user_info(req) + uid = user_info['uid'] bft = wash_url_argument(bft, 'str') if save_action is not None and code is not None: #save bibformatadminlib.update_format_template_code(bft, code=code) bibformat_engine.clear_caches() if code is None: code = bibformat_engine.get_format_template(bft)['code'] ln_for_preview = wash_language(ln_for_preview) pattern_for_preview = wash_url_argument(pattern_for_preview, 'str') if pattern_for_preview == "": try: recID = search_pattern(p='-collection:DELETED').pop() except KeyError: return page(title="No Document Found", body="", uid=uid, language=ln_for_preview, navtrail = "", lastupdated=__lastupdated__, req=req, navmenuid='search') pattern_for_preview = "recid:%s" % recID else: try: recID = search_pattern(p=pattern_for_preview + \ ' -collection:DELETED').pop() except KeyError: return page(title="No Record Found for %s" % pattern_for_preview, body="", uid=uid, language=ln_for_preview, navtrail = "", lastupdated=__lastupdated__, req=req) units = create_basic_search_units(None, pattern_for_preview, None) keywords = [unit[1] for unit in units if unit[0] != '-'] - bfo = bibformat_engine.BibFormatObject(recID, - ln_for_preview, - keywords, - None, - getUid(req), req=req) + bfo = bibformat_engine.BibFormatObject(recID = recID, + ln = ln_for_preview, + search_pattern = keywords, + xml_record = None, + user_info = user_info) (body, errors) = bibformat_engine.format_with_format_template(bft, bfo, verbose=7, format_template_code=code) if content_type_for_preview == 'text/html': #Standard page display with CDS headers, etc. return page(title="", body=body, uid=uid, language=ln_for_preview, navtrail = navtrail, lastupdated=__lastupdated__, req=req, navmenuid='search') else: #Output with chosen content-type. req.content_type = content_type_for_preview req.send_http_header() req.write(body) else: return page_not_authorized(req=req, text=auth_msg) def format_template_show_short_doc(req, ln=config.cdslang, search_doc_pattern=""): """ Prints the format elements documentation in a brief way. To be included inside Format template editor. @param ln: language @param search_doc_pattern a search pattern that specified which elements to display @param bft the name of the template to show """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: search_doc_pattern = wash_url_argument(search_doc_pattern, 'str') return bibformatadminlib.perform_request_format_template_show_short_doc(ln=ln, search_doc_pattern=search_doc_pattern) else: return page_not_authorized(req=req, text=auth_msg) def format_elements_doc(req, ln=config.cdslang): """ Main page for format elements documentation. Check for authentication and print format elements list. @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail() try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: return page(title=_("Format Elements Documentation"), body=bibformatadminlib.perform_request_format_elements_documentation(ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def format_element_show_dependencies(req, bfe, ln=config.cdslang): """ Shows format element dependencies @param bfe the name of the bfe to show @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln , _("Format Elements Documentation"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfe = wash_url_argument(bfe, 'str') return page(title=_("Format Element %s Dependencies" % bfe), body=bibformatadminlib.perform_request_format_element_show_dependencies(bfe=bfe, ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def format_element_test(req, bfe, ln=config.cdslang, param_values=None): """ Allows user to test element with different parameters and check output 'param_values' is the list of values to pass to 'format' function of the element as parameters, in the order ... If params is None, this means that they have not be defined by user yet. @param bfe the name of the element to test @param ln language @param param_values the list of parameters to pass to element format function """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' %( config.weburl, ln , _("Format Elements Documentation"))) - try: - uid = getUid(req) - except MySQLdb.Error, e: - return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: bfe = wash_url_argument(bfe, 'str') + user_info = collect_user_info(req) + uid = user_info['uid'] return page(title=_("Test Format Element %s" % bfe), body=bibformatadminlib.perform_request_format_element_test(bfe=bfe, ln=ln, param_values=param_values, - uid=getUid(req), req=req), + user_info=user_info), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_manage(req, ln=config.cdslang): """ Main page for knowledge bases management. Check for authentication. @param ln: language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail() try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: return page(title=_("Manage Knowledge Bases"), body=bibformatadminlib.perform_request_knowledge_bases_management(ln=ln), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_show(req, kb, sortby="to", ln=config.cdslang): """ Shows the content of the given knowledge base id. Check for authentication and kb existence. Before displaying the content of the knowledge base, check if a form was submitted asking for adding, editing or removing a value. @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) return page(title=_("Knowledge Base %s" % kb_name), body=bibformatadminlib.perform_request_knowledge_base_show(ln=ln, kb_id=kb_id, sortby=sortby), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_show_attributes(req, kb, ln=config.cdslang, sortby="to"): """ Shows the attributes (name, description) of a given kb @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) return page(title=_("Knowledge Base %s Attributes" % kb_name), body=bibformatadminlib.perform_request_knowledge_base_show_attributes(ln=ln, kb_id=kb_id, sortby=sortby), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_show_dependencies(req, kb, ln=config.cdslang, sortby="to"): """ Shows the dependencies of a given kb @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) return page(title=_("Knowledge Base %s Dependencies" % kb_name), body=bibformatadminlib.perform_request_knowledge_base_show_dependencies(ln=ln, kb_id=kb_id, sortby=sortby), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_add_mapping(req, kb, mapFrom, mapTo, sortby="to", ln=config.cdslang): """ Adds a new mapping to a kb. @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) key = wash_url_argument(mapFrom, 'str') value = wash_url_argument(mapTo, 'str') bibformatadminlib.add_kb_mapping(kb_name, key, value) redirect_to_url(req, "kb_show?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby}) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_edit_mapping(req, kb, key, mapFrom, mapTo, update="", delete="", sortby="to", ln=config.cdslang): """ Edit a mapping to in kb. Edit can be "update old value" or "delete existing value" @param kb the knowledge base id to edit @param key the key of the mapping that will be modified @param mapFrom the new key of the mapping @param mapTo the new value of the mapping @param update contains a value if the mapping is to be updated @param delete contains a value if the mapping is to be deleted @param sortby the sorting criteria ('from' or 'to') """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) key = wash_url_argument(key, 'str') if delete != "": #Delete bibformatadminlib.remove_kb_mapping(kb_name, key) else: #Update new_key = wash_url_argument(mapFrom, 'str') new_value = wash_url_argument(mapTo, 'str') bibformatadminlib.update_kb_mapping(kb_name, key, new_key, new_value) redirect_to_url(req, "kb_show?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby}) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_update_attributes(req, kb="", name="", description="", sortby="to", ln=config.cdslang, chosen_option=None): """ Update the attributes of the kb @param ln language @param kb the kb id to update @param sortby the sorting criteria ('from' or 'to') @param name the new name of the kn @param description the new description of the kb @param chosen_option set to dialog box value """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') if chosen_option is not None: # Update could not be performed. # Redirect to kb attributes page redirect_to_url(req, "kb_show_attributes?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby}) kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) new_name = wash_url_argument(name, 'str') if kb_name != new_name and bibformatadminlib.kb_exists(new_name): #A knowledge base with that name already exist #Do not update return dialog_box(req=req, ln=ln, title="Name already in use", message="""%s cannot be renamed to %s: Another knowledge base already has that name.
    Please choose another name.""" % (kb_name, new_name), navtrail=navtrail_previous_links, options=[ _("Ok")]) new_desc = wash_url_argument(description, 'str') bibformatadminlib.update_kb_attributes(kb_name, new_name, new_desc) redirect_to_url(req, "kb_show?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby}) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_add(req, ln=config.cdslang, sortby="to"): """ Adds a new kb """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = bibformatadminlib.add_kb() redirect_to_url(req, "kb_show_attributes?ln=%(ln)s&kb=%(kb)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby}) else: navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"))) return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def kb_delete(req, kb, ln=config.cdslang, chosen_option="", sortby="to"): """ Deletes an existing kb @param kb the kb id to delete """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s > %s''' % (config.weburl, ln, _("Manage Knowledge Bases"), _("Delete Knowledge Base"))) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibformatadminlib.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) #Ask confirmation to user if not already done chosen_option = wash_url_argument(chosen_option, 'str') if chosen_option == "": return dialog_box(req=req, ln=ln, title="Delete %s" % kb_name, message="""Are you sure you want to delete knowledge base %s?""" % kb_name, navtrail=navtrail_previous_links, options=[_("Cancel"), _("Delete")]) elif chosen_option==_("Delete"): bibformatadminlib.delete_kb(kb_name) redirect_to_url(req, "kb_manage?ln=%(ln)s" % {'ln':ln}) else: navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s'''%(config.weburl, _("Manage Knowledge Bases"))) return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links) def validate_format(req, ln=config.cdslang, bfo=None, bft=None, bfe=None): """ Returns a page showing the status of an output format or format template or format element. This page is called from output formats management page or format template management page or format elements documentation. The page only shows the status of one of the format, depending on the specified one. If multiple are specified, shows the first one. @param ln language @param bfo an output format 6 chars code @param bft a format element filename @param bfe a format element name """ ln = wash_language(ln) _ = gettext_set_language(ln) try: uid = getUid(req) except MySQLdb.Error, e: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibformat') if not auth_code: if bfo is not None: #Output format validation bfo = wash_url_argument(bfo, 'str') navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s'''%(config.weburl, ln, _("Manage Output Formats"))) if not bibformatadminlib.can_read_output_format(bfo): #No read permission return page(title=_("Restricted Output Format"), body = """You don't have permission to view this output format.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_OUTPUT_FILE", bfo , "")], lastupdated=__lastupdated__, req=req) output_format = bibformat_engine.get_output_format(code=bfo, with_attributes=True) name = output_format['attrs']['names']['generic'] title = _("Validation of Output Format %s" % name) elif bft is not None: #Format template validation bft = wash_url_argument(bft, 'str') navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln, _("Manage Format Templates"))) if not bibformatadminlib.can_read_format_template(bft): #No read permission return page(title=_("Restricted Format Template"), body = """You don't have permission to view this format template.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_TEMPLATE_FILE", bft , "")], lastupdated=__lastupdated__, req=req) name = bibformat_engine.get_format_template_attrs(bft)['name'] title = _("Validation of Format Template %s" % name) elif bfe is not None: #Format element validation bfe = wash_url_argument(bfe, 'str') navtrail_previous_links = bibformatadminlib.getnavtrail(''' > %s''' % (config.weburl, ln , bfe.upper() , _("Format Elements Documentation"))) if not bibformatadminlib.can_read_format_element(bfe) and \ not bibformat_dblayer.tag_exists_for_name(bfe): #No read permission return page(title=_("Restricted Format Element"), body = """You don't have permission to view this format element.""", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_BIBFORMAT_CANNOT_READ_ELEMENT_FILE", bfe ,"")], lastupdated=__lastupdated__, req=req) title = _("Validation of Format Element %s" % bfe) else: #No format specified return page(title=_("Format Validation"), uid=uid, language=ln, errors = [("ERR_BIBFORMAT_VALIDATE_NO_FORMAT")], navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) return page(title=title, body=bibformatadminlib.perform_request_format_validate(ln=ln, bfo=bfo, bft=bft, bfe=bfe), uid=uid, language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: navtrail_previous_links = bibformatadminlib.getnavtrail(''' > formatting - MARC rule list -> Category Navigation - MARC tag used for issue numbers -> search (later in the format elements) Uses BibFormatObject and format_with_format_template to produce the required HTML. """ # init all the values we need from config.xml config_strings = get_xml_from_config(["index", "rule", "issue_number"], - journal_name) + journal_name) try: try: index_page_template = config_strings["index"][0] except: raise InvenioWebJournalNoIndexTemplateError(language, journal_name) except InvenioWebJournalNoIndexTemplateError, e: register_exception(req=req) return e.user_box() index_page_template_path = 'webjournal/%s' % (index_page_template) rule_list = config_strings["rule"] try: if len(rule_list) == 0: - raise InvenioWebJournalNoArticleRuleError(language, journal_name) - except InvenioWebJournalNoArticleRuleError, e: + raise InvenioWebJournalNoArticleRuleError(language, journal_name) + except InvenioWebJournalNoArticleRuleError, e: register_exception(req=req) return e.user_box() try: try: issue_number_tag = config_strings["issue_number"][0] except: raise InvenioWebJournalNoIssueNumberTagError(language, journal_name) except InvenioWebJournalNoIssueNumberTagError, e: register_exception(req=req) return e.user_box() # get the current category for index display current_category_in_list = 0 i = 0 if category != "": for rule_string in rule_list: category_from_config = rule_string.split(",")[0] if category_from_config.lower() == category.lower(): current_category_in_list = i i+=1 else: # add the first category to the url string as a default req.journal_defaults["category"] = rule_list[0].split(",")[0] - # get the important values for the category from the config file + # get the important values for the category from the config file rule_string = rule_list[current_category_in_list].replace(" ", "") category = rule_string.split(",")[0] rule = rule_string.split(",")[1] marc_datafield = rule.split(":")[0] rule_match = rule.split(":")[1] marc_tag = marc_datafield[:3] marc_ind1 = (str(marc_datafield[3]) == "_") and " " or marc_datafield[3] marc_ind2 = (str(marc_datafield[4]) == "_") and " " or marc_datafield[4] marc_subfield = marc_datafield[5] - # create a marc record, containing category and issue number + # create a marc record, containing category and issue number temp_marc = ''' 0 %s %s ''' % (issue_number_tag[:3], (issue_number_tag[3] == "_") and " " or issue_number_tag[3], (issue_number_tag[4] == "_") and " " or issue_number_tag[4], issue_number_tag[5], issue_number, marc_tag, marc_ind1, marc_ind2, marc_subfield, rule_match) #temp_marc = temp_marc.decode('utf-8').encode('utf-8') # create a record and get HTML back from bibformat - bfo = BibFormatObject(0, ln=language, xml_record=temp_marc, req=req) + user_info = collect_user_info(req) + bfo = BibFormatObject(0, ln=language, xml_record=temp_marc, user_info=user_info) + bfo.req = req html = format_with_format_template(index_page_template_path, bfo)[0] return html def perform_request_article(req, journal_name, issue_number, language, category, number, editor): """ Central logic function for article pages. Loads the format template for article display and displays the requested article using BibFormat. 'Editor' Mode genereates edit links on the article view page and disables caching. """ # init all the values we need from config.xml config_strings = get_xml_from_config(["detailed", "rule"], journal_name) try: - try: + try: index_page_template = config_strings["detailed"][0] except: raise InvenioWebJournalNoArticleTemplateError(language, journal_name) except InvenioWebJournalNoArticleTemplateError, e: register_exception(req=req) return e.user_box() index_page_template_path = 'webjournal/%s' % (index_page_template) rule_list = config_strings["rule"] try: if len(rule_list) == 0: - raise InvenioWebJournalNoArticleRuleError(language, journal_name) - except InvenioWebJournalNoArticleRuleError, e: + raise InvenioWebJournalNoArticleRuleError(language, journal_name) + except InvenioWebJournalNoArticleRuleError, e: register_exception(req=req) return e.user_box() # get the current category for index display current_category_in_list = 0 i = 0 if category != "": for rule_string in rule_list: category_from_config = rule_string.split(",")[0] if category_from_config.lower() == category.lower(): current_category_in_list = i - i+=1 + i+=1 rule_string = rule_list[current_category_in_list].replace(" ", "") rule = rule_string.split(",")[1] # try to get the page from the cache recid = get_recid_from_order_CERNBulletin(number, rule, issue_number) cached_html = get_article_page_from_cache(journal_name, category, recid, issue_number, language) if cached_html and editor == "False": return cached_html # create a record and get HTML back from bibformat - bfo = BibFormatObject(recid, ln=language, req=req) + user_info = collect_user_info(req) + bfo = BibFormatObject(recid, ln=language, user_info=user_info) + bfo.req = req html_out = format_with_format_template(index_page_template_path, bfo)[0] # cache if not in editor mode if editor == "False": cache_article_page(html_out, journal_name, category, recid, issue_number, language) - + return html_out def perform_request_administrate(journal_name, language): """ """ current_issue = get_current_issue(language, journal_name) current_publication = get_current_publication(journal_name, current_issue, language) issue_list = get_list_of_issues_for_publication(current_publication) next_issue_number = count_week_string_up(issue_list[-1]) return tmpl_webjournal_admin_interface(journal_name, current_issue, current_publication, issue_list, next_issue_number, language) - + def perform_request_alert(req, journal_name, issue_number, language, sent, plain_text, subject, recipients, html_mail, force): """ All the logic for alert emails. Messages are retrieved from templates. (should be migrated to msg class) Mails can be edited by an interface form. Sent in HTML/PlainText or only PlainText if wished so. """ subject = tmpl_webjournal_alert_subject_CERNBulletin(journal_name, issue_number) plain_text = tmpl_webjournal_alert_plain_text_CERNBulletin(journal_name, language, issue_number) plain_text = plain_text.encode('utf-8') - + if sent == "False": interface = tmpl_webjournal_alert_interface(language, journal_name, subject, plain_text) return page(title="alert system", body=interface) else: if was_alert_sent_for_issue(issue_number, journal_name, language) != False and force == "False": return tmpl_webjournal_alert_was_already_sent(language, journal_name, subject, plain_text, recipients, html_mail, issue_number) - if html_mail == "html": + if html_mail == "html": html_file = urlopen('%s/journal/?name=%s&ln=en' - % (weburl, journal_name)) + % (weburl, journal_name)) html_string = html_file.read() html_file.close() html_string = put_css_in_file(html_string, journal_name) else: html_string = plain_text.replace("\n", "
    ") - + message = createhtmlmail(html_string, plain_text, subject, recipients) server = smtplib.SMTP("localhost", 25) server.sendmail('Bulletin-Support@cern.ch', recipients, message) # todo: has to go to some messages config - update_DB_for_alert(issue_number, journal_name, language) + update_DB_for_alert(issue_number, journal_name, language) return tmpl_webjournal_alert_success_msg(language, journal_name) - + def perform_request_issue_control(req, journal_name, issue_numbers, language, add, action): """ Central logic for issue control. Regenerates the flat files current_issue and issue_group that control the which issue is currently active for the journal. Todo: move issue control to DB """ if action == "cfg" or action == "Refresh" or action == "Add_One": # find out if we are in update or release try: current_issue_time = get_current_issue_time(journal_name) all_issue_weeks = get_all_issue_weeks(current_issue_time, journal_name, language) except InvenioWebJournalIssueNotFoundDBError, e: register_exception(req=req) return e.user_box() except InvenioWebJournalJournalIdNotFoundDBError, e: register_exception(req=req) return e.user_box() if max(all_issue_weeks) > current_issue_time: # propose an update next_issue_week = None all_issue_weeks.sort() for issue_week in all_issue_weeks: if issue_week > current_issue_time: next_issue_week = issue_week break output = tmpl_webjournal_update_an_issue(language, journal_name, issue_times_to_week_strings([next_issue_week,])[0], - issue_times_to_week_strings([current_issue_time,])[0]) + issue_times_to_week_strings([current_issue_time,])[0]) else: # propose a release next_issues = get_next_journal_issues(current_issue_time, journal_name) next_issues = issue_times_to_week_strings(next_issues, language) if action == "Refresh": next_issues += issue_numbers next_issues = list(sets.Set(next_issues))# avoid double entries elif action == "Add_One": next_issues += issue_numbers next_issues = list(sets.Set(next_issues))# avoid double entries next_issues_times = issue_week_strings_to_times(next_issues, language) highest_issue_so_far = max(next_issues_times) one_more_issue = get_next_journal_issues(highest_issue_so_far, journal_name, language, 1) one_more_issue = issue_times_to_week_strings(one_more_issue, language) next_issues += one_more_issue next_issues = list(sets.Set(next_issues)) # avoid double entries next_issues.sort() else: # get the next (default 2) issue numbers to publish next_issues = get_next_journal_issues(current_issue_time, journal_name, language) next_issues = issue_times_to_week_strings(next_issues, language) output = tmpl_webjournal_issue_control_interface(language, journal_name, next_issues) elif action == "Publish": publish_issues = issue_numbers publish_issues = list(sets.Set(publish_issues)) # avoid double entries publish_issues.sort() try: release_journal_issue(publish_issues, journal_name, language) except InvenioWebJournalJournalIdNotFoundDBError, e: register_exception(req=req) return e.user_box() output = tmpl_webjournal_issue_control_success_msg(language, publish_issues, journal_name) - + elif action == "Update": try: try: - update_issue = issue_numbers[0] + update_issue = issue_numbers[0] except: raise InvenioWebJournalReleaseUpdateError(language, journal_name) except InvenioWebJournalReleaseUpdateError, e: register_exception(req=req) return e.user_box() try: release_journal_update(update_issue, journal_name, language) except InvenioWebJournalJournalIdNotFoundDBError, e: register_exception(req=req) return e.user_box() output = tmpl_webjournal_updated_issue_msg(language, update_issue, journal_name) - + return page(title="Publish System", body=output) def perform_request_popup(req, language, journal_name, type, record): """ """ - config_strings = get_xml_from_config(["popup"], journal_name) + config_strings = get_xml_from_config(["popup"], journal_name) try: try: popup_page_template = config_strings["popup"][0] except: raise InvenioWebJournalNoPopupTemplateError(language) except InvenioWebJournalNoPopupTemplateError, e: register_exception(req=req) return e.user_box() - + popup_page_template_path = 'webjournal/%s' % popup_page_template - bfo = BibFormatObject(record, ln=language, req=req) + user_info = collect_user_info(req) + bfo = BibFormatObject(record, ln=language, user_info=user_info) + bfo.req = req html = format_with_format_template(popup_page_template_path, bfo)[0] return html def perform_request_search(journal_name, language, req, issue, archive_year, archive_issue, archive_select, archive_date, archive_search): """ Logic for the search / archive page. """ config_strings = get_xml_from_config(["search", "issue_number", "rule"], journal_name) try: - try: + try: search_page_template = config_strings["search"][0] except: raise InvenioWebJournalNoSearchTemplateError(journal_name, language) except InvenioWebJournalNoSearchTemplateError, e: register_exception(req=req) return e.user_box() search_page_template_path = 'webjournal/%s' % (search_page_template) # just an empty buffer record, since all values are in req.journal_defaults - + if archive_select == "False" and archive_search == "False": temp_marc = ''' 0 ''' - - bfo = BibFormatObject(0, ln=language, xml_record=temp_marc, req=req) + + user_info = collect_user_info(req) + bfo = BibFormatObject(0, ln=language, xml_record=temp_marc, user_info=user_info) + bfo.req = req html = format_with_format_template(search_page_template_path, bfo)[0] return html elif archive_select == "Go": redirect_to_url(req, "%s/journal/?name=%s&issue=%s&ln=%s" % (weburl, journal_name, archive_issue, language)) elif archive_search == "Go": archive_issue_time = time.strptime(archive_date, "%d/%m/%Y") archive_issue_time = count_down_to_monday(archive_issue_time) archive_issue = issue_times_to_week_strings([archive_issue_time,])[0] redirect_to_url(req, "%s/journal/?name=%s&issue=%s&ln=%s" % (weburl, journal_name, archive_issue, language)) diff --git a/modules/websearch/lib/search_engine.py b/modules/websearch/lib/search_engine.py index 5d82bee6f..ec0e6246b 100644 --- a/modules/websearch/lib/search_engine.py +++ b/modules/websearch/lib/search_engine.py @@ -1,4058 +1,4058 @@ # -*- coding: utf-8 -*- ## $Id$ ## This file is part of CDS Invenio. ## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN. ## ## CDS Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## CDS Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with CDS Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. # pylint: disable-msg=C0301 """CDS Invenio Search Engine in mod_python.""" __lastupdated__ = """$Date$""" __revision__ = "$Id$" ## import general modules: import cgi import copy import string import os import re import time import urllib import zlib ## import CDS Invenio stuff: from invenio.config import \ CFG_CERN_SITE, \ CFG_OAI_ID_FIELD, \ CFG_WEBCOMMENT_ALLOW_REVIEWS, \ CFG_WEBSEARCH_CALL_BIBFORMAT, \ CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX, \ CFG_WEBSEARCH_FIELDS_CONVERT, \ CFG_WEBSEARCH_NB_RECORDS_TO_SORT, \ CFG_WEBSEARCH_SEARCH_CACHE_SIZE, \ CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS, \ CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS, \ cdslang, \ cdsname, \ logdir, \ weburl from invenio.search_engine_config import CFG_EXPERIMENTAL_FEATURES, InvenioWebSearchUnknownCollectionError from invenio.bibrank_record_sorter import get_bibrank_methods, rank_records from invenio.bibrank_downloads_similarity import register_page_view_event, calculate_reading_similarity_list from invenio.bibindex_engine_stemmer import stem from invenio.bibformat import format_record, format_records, get_output_format_content_type, create_excel from invenio.bibformat_config import CFG_BIBFORMAT_USE_OLD_BIBFORMAT from invenio.bibrank_downloads_grapher import create_download_history_graph_and_box from invenio.data_cacher import DataCacher from invenio.websearch_external_collections import print_external_results_overview, perform_external_collection_search from invenio.access_control_admin import acc_get_action_id from invenio.access_control_config import VIEWRESTRCOLL, \ CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS from invenio.websearchadminlib import get_detailed_page_tabs from invenio.intbitset import intbitset as HitSet from invenio.webinterface_handler import wash_urlargd from invenio.urlutils import make_canonical_urlargd from invenio.dbquery import DatabaseError from invenio.access_control_engine import acc_authorize_action import invenio.template webstyle_templates = invenio.template.load('webstyle') webcomment_templates = invenio.template.load('webcomment') from invenio.bibrank_citation_searcher import calculate_cited_by_list, calculate_co_cited_with_list, get_self_cited_in, get_self_cited_by from invenio.bibrank_citation_grapher import create_citation_history_graph_and_box from invenio.dbquery import run_sql, run_sql_cached, get_table_update_time, Error -from invenio.webuser import getUid +from invenio.webuser import getUid, collect_user_info from invenio.webpage import page, pageheaderonly, pagefooteronly, create_error_box from invenio.messages import gettext_set_language try: from mod_python import apache except ImportError, e: pass # ignore user personalisation, needed e.g. for command-line try: import invenio.template websearch_templates = invenio.template.load('websearch') except: pass ## global vars: search_cache = {} # will cache results of previous searches cfg_nb_browse_seen_records = 100 # limit of the number of records to check when browsing certain collection cfg_nicely_ordered_collection_list = 0 # do we propose collection list nicely ordered or alphabetical? collection_reclist_cache_timestamp = 0 field_i18nname_cache_timestamp = 0 collection_i18nname_cache_timestamp = 0 ## precompile some often-used regexp for speed reasons: re_word = re.compile('[\s]') re_quotes = re.compile('[\'\"]') re_doublequote = re.compile('\"') re_equal = re.compile('\=') re_logical_and = re.compile('\sand\s', re.I) re_logical_or = re.compile('\sor\s', re.I) re_logical_not = re.compile('\snot\s', re.I) re_operators = re.compile(r'\s([\+\-\|])\s') re_pattern_wildcards_at_beginning = re.compile(r'(\s)[\*\%]+') re_pattern_single_quotes = re.compile("'(.*?)'") re_pattern_double_quotes = re.compile("\"(.*?)\"") re_pattern_regexp_quotes = re.compile("\/(.*?)\/") re_pattern_short_words = re.compile(r'([\s\"]\w{1,3})[\*\%]+') re_pattern_space = re.compile("__SPACE__") re_pattern_today = re.compile("\$TODAY\$") re_unicode_lowercase_a = re.compile(unicode(r"(?u)[áàäâãå]", "utf-8")) re_unicode_lowercase_ae = re.compile(unicode(r"(?u)[æ]", "utf-8")) re_unicode_lowercase_e = re.compile(unicode(r"(?u)[éèëê]", "utf-8")) re_unicode_lowercase_i = re.compile(unicode(r"(?u)[íìïî]", "utf-8")) re_unicode_lowercase_o = re.compile(unicode(r"(?u)[óòöôõø]", "utf-8")) re_unicode_lowercase_u = re.compile(unicode(r"(?u)[úùüû]", "utf-8")) re_unicode_lowercase_y = re.compile(unicode(r"(?u)[ýÿ]", "utf-8")) re_unicode_lowercase_c = re.compile(unicode(r"(?u)[çć]", "utf-8")) re_unicode_lowercase_n = re.compile(unicode(r"(?u)[ñ]", "utf-8")) re_unicode_uppercase_a = re.compile(unicode(r"(?u)[ÁÀÄÂÃÅ]", "utf-8")) re_unicode_uppercase_ae = re.compile(unicode(r"(?u)[Æ]", "utf-8")) re_unicode_uppercase_e = re.compile(unicode(r"(?u)[ÉÈËÊ]", "utf-8")) re_unicode_uppercase_i = re.compile(unicode(r"(?u)[ÍÌÏÎ]", "utf-8")) re_unicode_uppercase_o = re.compile(unicode(r"(?u)[ÓÒÖÔÕØ]", "utf-8")) re_unicode_uppercase_u = re.compile(unicode(r"(?u)[ÚÙÜÛ]", "utf-8")) re_unicode_uppercase_y = re.compile(unicode(r"(?u)[Ý]", "utf-8")) re_unicode_uppercase_c = re.compile(unicode(r"(?u)[ÇĆ]", "utf-8")) re_unicode_uppercase_n = re.compile(unicode(r"(?u)[Ñ]", "utf-8")) re_latex_lowercase_a = re.compile("\\\\[\"H'`~^vu=k]\{?a\}?") re_latex_lowercase_ae = re.compile("\\\\ae\\{\\}?") re_latex_lowercase_e = re.compile("\\\\[\"H'`~^vu=k]\\{?e\\}?") re_latex_lowercase_i = re.compile("\\\\[\"H'`~^vu=k]\\{?i\\}?") re_latex_lowercase_o = re.compile("\\\\[\"H'`~^vu=k]\\{?o\\}?") re_latex_lowercase_u = re.compile("\\\\[\"H'`~^vu=k]\\{?u\\}?") re_latex_lowercase_y = re.compile("\\\\[\"']\\{?y\\}?") re_latex_lowercase_c = re.compile("\\\\['uc]\\{?c\\}?") re_latex_lowercase_n = re.compile("\\\\[c'~^vu]\\{?n\\}?") re_latex_uppercase_a = re.compile("\\\\[\"H'`~^vu=k]\\{?A\\}?") re_latex_uppercase_ae = re.compile("\\\\AE\\{?\\}?") re_latex_uppercase_e = re.compile("\\\\[\"H'`~^vu=k]\\{?E\\}?") re_latex_uppercase_i = re.compile("\\\\[\"H'`~^vu=k]\\{?I\\}?") re_latex_uppercase_o = re.compile("\\\\[\"H'`~^vu=k]\\{?O\\}?") re_latex_uppercase_u = re.compile("\\\\[\"H'`~^vu=k]\\{?U\\}?") re_latex_uppercase_y = re.compile("\\\\[\"']\\{?Y\\}?") re_latex_uppercase_c = re.compile("\\\\['uc]\\{?C\\}?") re_latex_uppercase_n = re.compile("\\\\[c'~^vu]\\{?N\\}?") class RestrictedCollectionDataCacher(DataCacher): def __init__(self): def cache_filler(): ret = [] try: viewcollid = acc_get_action_id(VIEWRESTRCOLL) res = run_sql("""SELECT DISTINCT ar.value FROM accROLE_accACTION_accARGUMENT raa JOIN accARGUMENT ar ON raa.id_accARGUMENT = ar.id WHERE ar.keyword = 'collection' AND raa.id_accACTION = %s""", (viewcollid,)) except Exception: # database problems, return empty cache return [] for coll in res: ret.append(coll[0]) return ret def timestamp_getter(): return max(get_table_update_time('accROLE_accACTION_accARGUMENT'), get_table_update_time('accARGUMENT')) DataCacher.__init__(self, cache_filler, timestamp_getter) def collection_restricted_p(collection): cache = restricted_collection_cache.get_cache() return collection in cache try: restricted_collection_cache.is_ok_p except Exception: restricted_collection_cache = RestrictedCollectionDataCacher() def check_user_can_view_record(user_info, recid): """Check if the user is authorized to view the given recid. The function grants access in two cases: either user has author rights on ths record, or he has view rights to the primary collection this record belongs to. Returns the same type as acc_authorize_action """ def _is_user_in_authorized_author_list_for_recid(user_info, recid): """Return True if the user have submitted the given record.""" authorized_emails = [] for tag in CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS: authorized_emails.extend(get_fieldvalues(recid, tag)) for email in authorized_emails: email = email.strip().lower() if user_info['email'].strip().lower() == email: return True return False record_primary_collection = guess_primary_collection_of_a_record(recid) if collection_restricted_p(record_primary_collection): (auth_code, auth_msg) = acc_authorize_action(user_info, VIEWRESTRCOLL, collection=record_primary_collection) if auth_code == 0 or _is_user_in_authorized_author_list_for_recid(user_info, recid): return (0, '') else: return (auth_code, auth_msg) else: return (0, '') class IndexStemmingDataCacher(DataCacher): def __init__(self): def cache_filler(): try: res = run_sql("""SELECT id, stemming_language FROM idxINDEX""") except DatabaseError: # database problems, return empty cache return {} return dict(res) def timestamp_getter(): return get_table_update_time('idxINDEX') DataCacher.__init__(self, cache_filler, timestamp_getter) def get_index_stemming_language(index_id): cache = index_stemming_cache.get_cache() return cache[index_id] try: index_stemming_cache.is_ok_p except Exception: index_stemming_cache = IndexStemmingDataCacher() class FieldI18nNameDataCacher(DataCacher): def __init__(self): def cache_filler(): ret = {} try: res = run_sql("SELECT f.name,fn.ln,fn.value FROM fieldname AS fn, field AS f WHERE fn.id_field=f.id AND fn.type='ln'") # ln=long name except Exception: # database problems, return empty cache return {} for f, ln, i18nname in res: if i18nname: if not ret.has_key(f): ret[f] = {} ret[f][ln] = i18nname return ret def timestamp_getter(): return get_table_update_time('fieldname') DataCacher.__init__(self, cache_filler, timestamp_getter) def get_field_i18nname(self, f, ln=cdslang): out = f try: out = self.get_cache()[f][ln] except KeyError: pass # translation in LN does not exist return out try: if not field_i18n_name_cache.is_ok_p: raise Exception except Exception: field_i18n_name_cache = FieldI18nNameDataCacher() class CollectionRecListDataCacher(DataCacher): def __init__(self): def cache_filler(): ret = {} try: res = run_sql("SELECT name,reclist FROM collection") except Exception: # database problems, return empty cache return {} for name, reclist in res: ret[name] = None # this will be filled later during runtime by calling get_collection_reclist(coll) return ret def timestamp_getter(): return get_table_update_time('collection') DataCacher.__init__(self, cache_filler, timestamp_getter) def get_collection_reclist(self, coll): cache = self.get_cache() if not cache[coll]: # not yet it the cache, so calculate it and fill the cache: set = HitSet() query = "SELECT nbrecs,reclist FROM collection WHERE name='%s'" % coll res = run_sql(query, None, 1) if res: try: set = HitSet(res[0][1]) except: pass self.cache[coll] = set cache[coll] = set # finally, return reclist: return cache[coll] try: if not collection_reclist_cache.is_ok_p: raise Exception except Exception: collection_reclist_cache = CollectionRecListDataCacher() class CollectionI18nDataCacher(DataCacher): def __init__(self): def cache_filler(): ret = {} try: res = run_sql("SELECT c.name,cn.ln,cn.value FROM collectionname AS cn, collection AS c WHERE cn.id_collection=c.id AND cn.type='ln'") # ln=long name except Exception: # database problems, return {} for c, ln, i18nname in res: if i18nname: if not ret.has_key(c): ret[c] = {} ret[c][ln] = i18nname return ret def timestamp_getter(): return get_table_update_time('collectionname') DataCacher.__init__(self, cache_filler, timestamp_getter) def get_coll_i18nname(self, c, ln=cdslang): """Return nicely formatted collection name (of name type 'ln', 'long name') for collection C in language LN.""" cache = self.get_cache() out = c try: out = cache[c][ln] except KeyError: pass # translation in LN does not exist return out try: if not collection_i18n_name_cache.is_ok_p: raise Exception except Exception: collection_i18n_name_cache = CollectionI18nDataCacher() def get_alphabetically_ordered_collection_list(level=0, ln=cdslang): """Returns nicely ordered (score respected) list of collections, more exactly list of tuples (collection name, printable collection name). Suitable for create_search_box().""" out = [] query = "SELECT id,name FROM collection ORDER BY name ASC" res = run_sql(query) for c_id, c_name in res: # make a nice printable name (e.g. truncate c_printable for # long collection names in given language): c_printable = get_coll_i18nname(c_name, ln) if len(c_printable)>30: c_printable = c_printable[:30] + "..." if level: c_printable = " " + level * '-' + " " + c_printable out.append([c_name, c_printable]) return out def get_nicely_ordered_collection_list(collid=1, level=0, ln=cdslang): """Returns nicely ordered (score respected) list of collections, more exactly list of tuples (collection name, printable collection name). Suitable for create_search_box().""" colls_nicely_ordered = [] query = "SELECT c.name,cc.id_son FROM collection_collection AS cc, collection AS c "\ " WHERE c.id=cc.id_son AND cc.id_dad='%s' ORDER BY score DESC" % collid res = run_sql(query) for c, cid in res: # make a nice printable name (e.g. truncate c_printable for # long collection names in given language): c_printable = get_coll_i18nname(c, ln) if len(c_printable)>30: c_printable = c_printable[:30] + "..." if level: c_printable = " " + level * '-' + " " + c_printable colls_nicely_ordered.append([c, c_printable]) colls_nicely_ordered = colls_nicely_ordered + get_nicely_ordered_collection_list(cid, level+1, ln=ln) return colls_nicely_ordered def get_index_id_from_field(field): """Returns first index id where the field code FIELD is indexed. Returns zero in case there is no table for this index. Example: field='author', output=4.""" out = 0 res = run_sql("""SELECT w.id FROM idxINDEX AS w, idxINDEX_field AS wf, field AS f WHERE f.code=%s AND wf.id_field=f.id AND w.id=wf.id_idxINDEX LIMIT 1""", (field,)) if res: out = res[0][0] return out def get_words_from_pattern(pattern): "Returns list of whitespace-separated words from pattern." words = {} for word in string.split(pattern): if not words.has_key(word): words[word] = 1; return words.keys() def create_basic_search_units(req, p, f, m=None, of='hb'): """Splits search pattern and search field into a list of independently searchable units. - A search unit consists of '(operator, pattern, field, type, hitset)' tuples where 'operator' is set union (|), set intersection (+) or set exclusion (-); 'pattern' is either a word (e.g. muon*) or a phrase (e.g. 'nuclear physics'); 'field' is either a code like 'title' or MARC tag like '100__a'; 'type' is the search type ('w' for word file search, 'a' for access file search). - Optionally, the function accepts the match type argument 'm'. If it is set (e.g. from advanced search interface), then it performs this kind of matching. If it is not set, then a guess is made. 'm' can have values: 'a'='all of the words', 'o'='any of the words', 'p'='phrase/substring', 'r'='regular expression', 'e'='exact value'. - Warnings are printed on req (when not None) in case of HTML output formats.""" opfts = [] # will hold (o,p,f,t,h) units ## check arguments: if matching type phrase/string/regexp, do we have field defined? if (m=='p' or m=='r' or m=='e') and not f: m = 'a' if of.startswith("h"): print_warning(req, "This matching type cannot be used within any field. I will perform a word search instead." ) print_warning(req, "If you want to phrase/substring/regexp search in a specific field, e.g. inside title, then please choose within title search option.") ## is desired matching type set? if m: ## A - matching type is known; good! if m == 'e': # A1 - exact value: opfts.append(['+', p, f, 'a']) # '+' since we have only one unit elif m == 'p': # A2 - phrase/substring: opfts.append(['+', "%" + p + "%", f, 'a']) # '+' since we have only one unit elif m == 'r': # A3 - regular expression: opfts.append(['+', p, f, 'r']) # '+' since we have only one unit elif m == 'a' or m == 'w': # A4 - all of the words: p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed for word in get_words_from_pattern(p): opfts.append(['+', word, f, 'w']) # '+' in all units elif m == 'o': # A5 - any of the words: p = strip_accents(p) # strip accents for 'w' mode, FIXME: delete when not needed for word in get_words_from_pattern(p): if len(opfts)==0: opfts.append(['+', word, f, 'w']) # '+' in the first unit else: opfts.append(['|', word, f, 'w']) # '|' in further units else: if of.startswith("h"): print_warning(req, "Matching type '%s' is not implemented yet." % m, "Warning") opfts.append(['+', "%" + p + "%", f, 'a']) else: ## B - matching type is not known: let us try to determine it by some heuristics if f and p[0] == '"' and p[-1] == '"': ## B0 - does 'p' start and end by double quote, and is 'f' defined? => doing ACC search opfts.append(['+', p[1:-1], f, 'a']) elif f and p[0] == "'" and p[-1] == "'": ## B0bis - does 'p' start and end by single quote, and is 'f' defined? => doing ACC search opfts.append(['+', '%' + p[1:-1] + '%', f, 'a']) elif f and p[0] == "/" and p[-1] == "/": ## B0ter - does 'p' start and end by a slash, and is 'f' defined? => doing regexp search opfts.append(['+', p[1:-1], f, 'r']) elif f and string.find(p, ',') >= 0: ## B1 - does 'p' contain comma, and is 'f' defined? => doing ACC search opfts.append(['+', p, f, 'a']) elif f and str(f[0:2]).isdigit(): ## B2 - does 'f' exist and starts by two digits? => doing ACC search opfts.append(['+', p, f, 'a']) else: ## B3 - doing WRD search, but maybe ACC too # search units are separated by spaces unless the space is within single or double quotes # so, let us replace temporarily any space within quotes by '__SPACE__' p = re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", p) p = re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", p) p = re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", p) # wash argument: p = re_equal.sub(":", p) p = re_logical_and.sub(" ", p) p = re_logical_or.sub(" |", p) p = re_logical_not.sub(" -", p) p = re_operators.sub(r' \1', p) for pi in string.split(p): # iterate through separated units (or items, as "pi" stands for "p item") pi = re_pattern_space.sub(" ", pi) # replace back '__SPACE__' by ' ' # firstly, determine set operator if pi[0] == '+' or pi[0] == '-' or pi[0] == '|': oi = pi[0] pi = pi[1:] else: # okay, there is no operator, so let us decide what to do by default oi = '+' # by default we are doing set intersection... # secondly, determine search pattern and field: if string.find(pi, ":") > 0: fi, pi = string.split(pi, ":", 1) else: fi, pi = f, pi # look also for old ALEPH field names: if fi and CFG_WEBSEARCH_FIELDS_CONVERT.has_key(string.lower(fi)): fi = CFG_WEBSEARCH_FIELDS_CONVERT[string.lower(fi)] # wash 'pi' argument: if re_quotes.match(pi): # B3a - quotes are found => do ACC search (phrase search) if fi: if pi[0] == '"' and pi[-1] == '"': pi = string.replace(pi, '"', '') # remove quote signs opfts.append([oi, pi, fi, 'a']) elif pi[0] == "'" and pi[-1] == "'": pi = string.replace(pi, "'", "") # remove quote signs opfts.append([oi, "%" + pi + "%", fi, 'a']) else: # unbalanced quotes, so do WRD query: opfts.append([oi, pi, fi, 'w']) else: # fi is not defined, look at where we are doing exact or subphrase search (single/double quotes): if pi[0] == '"' and pi[-1] == '"': opfts.append([oi, pi[1:-1], "anyfield", 'a']) if of.startswith("h"): print_warning(req, "Searching for an exact match inside any field may be slow. You may want to search for words instead, or choose to search within specific field.") else: # nope, subphrase in global index is not possible => change back to WRD search pi = strip_accents(pi) # strip accents for 'w' mode, FIXME: delete when not needed for pii in get_words_from_pattern(pi): # since there may be '-' and other chars that we do not index in WRD opfts.append([oi, pii, fi, 'w']) if of.startswith("h"): print_warning(req, "The partial phrase search does not work in any field. I'll do a boolean AND searching instead.") print_warning(req, "If you want to do a partial phrase search in a specific field, e.g. inside title, then please choose 'within title' search option.", "Tip") print_warning(req, "If you want to do exact phrase matching, then please use double quotes.", "Tip") elif fi and str(fi[0]).isdigit() and str(fi[0]).isdigit(): # B3b - fi exists and starts by two digits => do ACC search opfts.append([oi, pi, fi, 'a']) elif fi and not get_index_id_from_field(fi): # B3c - fi exists but there is no words table for fi => try ACC search opfts.append([oi, pi, fi, 'a']) elif fi and pi.startswith('/') and pi.endswith('/'): # B3d - fi exists and slashes found => try regexp search opfts.append([oi, pi[1:-1], fi, 'r']) else: # B3e - general case => do WRD search pi = strip_accents(pi) # strip accents for 'w' mode, FIXME: delete when not needed for pii in get_words_from_pattern(pi): opfts.append([oi, pii, fi, 'w']) ## sanity check: for i in range(0, len(opfts)): try: pi = opfts[i][1] if pi == '*': if of.startswith("h"): print_warning(req, "Ignoring standalone wildcard word.", "Warning") del opfts[i] if pi == '' or pi == ' ': fi = opfts[i][2] if fi: if of.startswith("h"): print_warning(req, "Ignoring empty %s search term." % fi, "Warning") del opfts[i] except: pass ## return search units: return opfts def page_start(req, of, cc, as, ln, uid, title_message=None, description='', keywords='', recID=-1, tab=''): "Start page according to given output format." _ = gettext_set_language(ln) if not title_message: title_message = _("Search Results") if not req: return # we were called from CLI content_type = get_output_format_content_type(of) if of.startswith('x'): if of == 'xr': # we are doing RSS output req.content_type = "application/rss+xml" req.send_http_header() req.write("""\n""") else: # we are doing XML output: req.content_type = "text/xml" req.send_http_header() req.write("""\n""") elif of.startswith('t') or str(of[0:3]).isdigit(): # we are doing plain text output: req.content_type = "text/plain" req.send_http_header() elif of == "id": pass # nothing to do, we shall only return list of recIDs elif content_type == 'text/html': # we are doing HTML output: req.content_type = "text/html" req.send_http_header() if not description: description = "%s %s." % (cc, _("Search Results")) if not keywords: keywords = "%s, WebSearch, %s" % (get_coll_i18nname(cdsname, ln), get_coll_i18nname(cc, ln)) argd = {} if req.args: argd = cgi.parse_qs(req.args) rssurl = websearch_templates.build_rss_url(argd) navtrail = create_navtrail_links(cc, as, ln) navtrail_append_title_p = 1 # FIXME: Find a good point to put this code. # This is a nice hack to trigger jsMath only when displaying single # records. if of.lower() in CFG_WEBSEARCH_USE_JSMATH_FOR_FORMATS: metaheaderadd = """ """ else: metaheaderadd = '' if tab != '' or ((of != '' or of.lower() != 'hd') and of != 'hb'): # If we are not in information tab in HD format, customize # the nav. trail to have a link back to main record. (Due # to the way perform_request_search() works, hb # (lowercase) is equal to hd) if (of != '' or of.lower() != 'hd') and of != 'hb': # Export format_name = of query = "SELECT name FROM format WHERE code=%s" res = run_sql(query, (of,)) if res: format_name = res[0][0] navtrail += ' >     %s > %s' % \ (weburl, recID, title_message, format_name) else: # Discussion, citations, etc. tabs tab_label = get_detailed_page_tabs(cc, ln=ln)[tab]['label'] navtrail += ' > %s > %s' % \ (weburl, recID, title_message, _(tab_label)) navtrail_append_title_p = 0 req.write(pageheaderonly(req=req, title=title_message, navtrail=navtrail, description=description, keywords=keywords, metaheaderadd=metaheaderadd, uid=uid, language=ln, navmenuid='search', navtrail_append_title_p=\ navtrail_append_title_p, rssurl=rssurl)) req.write(websearch_templates.tmpl_search_pagestart(ln=ln)) #else: # req.send_http_header() def page_end(req, of="hb", ln=cdslang): "End page according to given output format: e.g. close XML tags, add HTML footer, etc." if of == "id": return [] # empty recID list if not req: return # we were called from CLI if of.startswith('h'): req.write(websearch_templates.tmpl_search_pageend(ln = ln)) # pagebody end req.write(pagefooteronly(lastupdated=__lastupdated__, language=ln, req=req)) return "\n" def create_inputdate_box(name="d1", selected_year=0, selected_month=0, selected_day=0, ln=cdslang): "Produces 'From Date', 'Until Date' kind of selection box. Suitable for search options." _ = gettext_set_language(ln) box = "" # day box += """""" # month box += """""" # year box += """""" return box def create_search_box(cc, colls, p, f, rg, sf, so, sp, rm, of, ot, as, ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action=""): """Create search box for 'search again in the results page' functionality.""" # load the right message language _ = gettext_set_language(ln) # some computations cc_intl = get_coll_i18nname(cc, ln) cc_colID = get_colID(cc) colls_nicely_ordered = [] if cfg_nicely_ordered_collection_list: colls_nicely_ordered = get_nicely_ordered_collection_list(ln=ln) else: colls_nicely_ordered = get_alphabetically_ordered_collection_list(ln=ln) colls_nice = [] for (cx, cx_printable) in colls_nicely_ordered: if not cx.startswith("Unnamed collection"): colls_nice.append({ 'value' : cx, 'text' : cx_printable }) coll_selects = [] if colls and colls[0] != cdsname: # some collections are defined, so print these first, and only then print 'add another collection' heading: for c in colls: if c: temp = [] temp.append({ 'value' : '', 'text' : '*** %s ***' % _("any collection") }) for val in colls_nice: # print collection: if not cx.startswith("Unnamed collection"): temp.append({ 'value' : val['value'], 'text' : val['text'], 'selected' : (c == re.sub("^[\s\-]*","", val['value'])) }) coll_selects.append(temp) coll_selects.append([{ 'value' : '', 'text' : '*** %s ***' % _("add another collection") }] + colls_nice) else: # we searched in CDSNAME, so print 'any collection' heading coll_selects.append([{ 'value' : '', 'text' : '*** %s ***' % _("any collection") }] + colls_nice) sort_fields = [{ 'value' : '', 'text' : _("latest first") }] query = """SELECT DISTINCT(f.code),f.name FROM field AS f, collection_field_fieldvalue AS cff WHERE cff.type='soo' AND cff.id_field=f.id ORDER BY cff.score DESC, f.name ASC""" res = run_sql(query) for code, name in res: sort_fields.append({ 'value' : code, 'text' : name, }) ## ranking methods ranks = [{ 'value' : '', 'text' : "- %s %s -" % (_("OR").lower (), _("rank by")), }] for (code, name) in get_bibrank_methods(cc_colID, ln): # propose found rank methods: ranks.append({ 'value' : code, 'text' : name, }) formats = [] query = """SELECT code,name FROM format WHERE visibility='1' ORDER BY name ASC""" res = run_sql(query) if res: # propose found formats: for code, name in res: formats.append({ 'value' : code, 'text' : name }) else: formats.append({'value' : 'hb', 'text' : _("HTML brief") }) return websearch_templates.tmpl_search_box( ln = ln, as = as, cc_intl = cc_intl, cc = cc, ot = ot, sp = sp, action = action, fieldslist = get_searchwithin_fields(ln=ln, colID=cc_colID), f1 = f1, f2 = f2, f3 = f3, m1 = m1, m2 = m2, m3 = m3, p1 = p1, p2 = p2, p3 = p3, op1 = op1, op2 = op2, rm = rm, p = p, f = f, coll_selects = coll_selects, d1y = d1y, d2y = d2y, d1m = d1m, d2m = d2m, d1d = d1d, d2d = d2d, dt = dt, sort_fields = sort_fields, sf = sf, so = so, ranks = ranks, sc = sc, rg = rg, formats = formats, of = of, pl = pl, jrec = jrec, ec = ec, ) def create_navtrail_links(cc=cdsname, as=0, ln=cdslang, self_p=1, tab=''): """Creates navigation trail links, i.e. links to collection ancestors (except Home collection). If as==1, then links to Advanced Search interfaces; otherwise Simple Search. """ dads = [] for dad in get_coll_ancestors(cc): if dad != cdsname: # exclude Home collection dads.append ((dad, get_coll_i18nname (dad, ln))) if self_p and cc != cdsname: dads.append((cc, get_coll_i18nname(cc, ln))) return websearch_templates.tmpl_navtrail_links( as=as, ln=ln, dads=dads) def get_searchwithin_fields(ln='en', colID=None): """Retrieves the fields name used in the 'search within' selection box for the collection ID colID.""" res = None if colID: res = run_sql_cached("""SELECT f.code,f.name FROM field AS f, collection_field_fieldvalue AS cff WHERE cff.type='sew' AND cff.id_collection=%s AND cff.id_field=f.id ORDER BY cff.score DESC, f.name ASC""", (colID,)) if not res: res = run_sql_cached("SELECT code,name FROM field ORDER BY name ASC") fields = [{ 'value' : '', 'text' : get_field_i18nname("any field", ln) }] for field_code, field_name in res: if field_code and field_code != "anyfield": fields.append({ 'value' : field_code, 'text' : get_field_i18nname(field_name, ln) }) return fields def create_andornot_box(name='op', value='', ln='en'): "Returns HTML code for the AND/OR/NOT selection box." _ = gettext_set_language(ln) out = """ """ % (name, is_selected('a', value), _("AND"), is_selected('o', value), _("OR"), is_selected('n', value), _("AND NOT")) return out def create_matchtype_box(name='m', value='', ln='en'): "Returns HTML code for the 'match type' selection box." _ = gettext_set_language(ln) out = """ """ % (name, is_selected('a', value), _("All of the words:"), is_selected('o', value), _("Any of the words:"), is_selected('e', value), _("Exact phrase:"), is_selected('p', value), _("Partial phrase:"), is_selected('r', value), _("Regular expression:")) return out def is_selected(var, fld): "Checks if the two are equal, and if yes, returns ' selected'. Useful for select boxes." if type(var) is int and type(fld) is int: if var == fld: return " selected" elif str(var) == str(fld): return " selected" elif fld and len(fld)==3 and fld[0] == "w" and var == fld[1:]: return " selected" return "" def wash_colls(cc, c, split_colls=0): """Wash collection list by checking whether user has deselected anything under 'Narrow search'. Checks also if cc is a list or not. Return list of cc, colls_to_display, colls_to_search since the list of collections to display is different from that to search in. This is because users might have chosen 'split by collection' functionality. The behaviour of "collections to display" depends solely whether user has deselected a particular collection: e.g. if it started from 'Articles and Preprints' page, and deselected 'Preprints', then collection to display is 'Articles'. If he did not deselect anything, then collection to display is 'Articles & Preprints'. The behaviour of "collections to search in" depends on the 'split_colls' parameter: * if is equal to 1, then we can wash the colls list down and search solely in the collection the user started from; * if is equal to 0, then we are splitting to the first level of collections, i.e. collections as they appear on the page we started to search from; The function raises exception InvenioWebSearchUnknownCollectionError if cc or one of c collections is not known. """ colls_out = [] colls_out_for_display = [] # check what type is 'cc': if type(cc) is list: for ci in cc: if collection_reclist_cache.has_key(ci): # yes this collection is real, so use it: cc = ci break else: # check once if cc is real: if not collection_reclist_cache.has_key(cc): if cc: raise InvenioWebSearchUnknownCollectionError(cc) else: cc = cdsname # cc is not set, so replace it with Home collection # check type of 'c' argument: if type(c) is list: colls = c else: colls = [c] # remove all 'unreal' collections: colls_real = [] for coll in colls: if collection_reclist_cache.has_key(coll): colls_real.append(coll) else: if coll: raise InvenioWebSearchUnknownCollectionError(coll) colls = colls_real # check if some real collections remain: if len(colls)==0: colls = [cc] # then let us check the list of non-restricted "real" sons of 'cc' and compare it to 'coll': res = run_sql("""SELECT c.name FROM collection AS c, collection_collection AS cc, collection AS ccc WHERE c.id=cc.id_son AND cc.id_dad=ccc.id AND ccc.name=%s AND cc.type='r' AND c.restricted IS NULL""", (cc,)) l_cc_nonrestricted_sons = [] l_c = colls for row in res: l_cc_nonrestricted_sons.append(row[0]) l_c.sort() l_cc_nonrestricted_sons.sort() if l_cc_nonrestricted_sons == l_c: colls_out_for_display = [cc] # yep, washing permitted, it is sufficient to display 'cc' else: colls_out_for_display = colls # nope, we need to display all 'colls' successively # remove duplicates: colls_out_for_display_nondups=filter(lambda x, colls_out_for_display=colls_out_for_display: colls_out_for_display[x-1] not in colls_out_for_display[x:], range(1, len(colls_out_for_display)+1)) colls_out_for_display = map(lambda x, colls_out_for_display=colls_out_for_display:colls_out_for_display[x-1], colls_out_for_display_nondups) # second, let us decide on collection splitting: if split_colls == 0: # type A - no sons are wanted colls_out = colls_out_for_display # elif split_colls == 1: else: # type B - sons (first-level descendants) are wanted for coll in colls_out_for_display: coll_sons = get_coll_sons(coll) if coll_sons == []: colls_out.append(coll) else: colls_out = colls_out + coll_sons # remove duplicates: colls_out_nondups=filter(lambda x, colls_out=colls_out: colls_out[x-1] not in colls_out[x:], range(1, len(colls_out)+1)) colls_out = map(lambda x, colls_out=colls_out:colls_out[x-1], colls_out_nondups) return (cc, colls_out_for_display, colls_out) def strip_accents(x): """Strip accents in the input phrase X (assumed in UTF-8) by replacing accented characters with their unaccented cousins (e.g. é by e). Return such a stripped X.""" x = re_latex_lowercase_a.sub("a", x) x = re_latex_lowercase_ae.sub("ae", x) x = re_latex_lowercase_e.sub("e", x) x = re_latex_lowercase_i.sub("i", x) x = re_latex_lowercase_o.sub("o", x) x = re_latex_lowercase_u.sub("u", x) x = re_latex_lowercase_y.sub("x", x) x = re_latex_lowercase_c.sub("c", x) x = re_latex_lowercase_n.sub("n", x) x = re_latex_uppercase_a.sub("A", x) x = re_latex_uppercase_ae.sub("AE", x) x = re_latex_uppercase_e.sub("E", x) x = re_latex_uppercase_i.sub("I", x) x = re_latex_uppercase_o.sub("O", x) x = re_latex_uppercase_u.sub("U", x) x = re_latex_uppercase_y.sub("Y", x) x = re_latex_uppercase_c.sub("C", x) x = re_latex_uppercase_n.sub("N", x) # convert input into Unicode string: try: y = unicode(x, "utf-8") except: return x # something went wrong, probably the input wasn't UTF-8 # asciify Latin-1 lowercase characters: y = re_unicode_lowercase_a.sub("a", y) y = re_unicode_lowercase_ae.sub("ae", y) y = re_unicode_lowercase_e.sub("e", y) y = re_unicode_lowercase_i.sub("i", y) y = re_unicode_lowercase_o.sub("o", y) y = re_unicode_lowercase_u.sub("u", y) y = re_unicode_lowercase_y.sub("y", y) y = re_unicode_lowercase_c.sub("c", y) y = re_unicode_lowercase_n.sub("n", y) # asciify Latin-1 uppercase characters: y = re_unicode_uppercase_a.sub("A", y) y = re_unicode_uppercase_ae.sub("AE", y) y = re_unicode_uppercase_e.sub("E", y) y = re_unicode_uppercase_i.sub("I", y) y = re_unicode_uppercase_o.sub("O", y) y = re_unicode_uppercase_u.sub("U", y) y = re_unicode_uppercase_y.sub("Y", y) y = re_unicode_uppercase_c.sub("C", y) y = re_unicode_uppercase_n.sub("N", y) # return UTF-8 representation of the Unicode string: return y.encode("utf-8") def wash_index_term(term, max_char_length=50): """ Return washed form of the index term TERM that would be suitable for storing into idxWORD* tables. I.e., lower the TERM, and truncate it safely to MAX_CHAR_LENGTH UTF-8 characters (meaning, in principle, 4*MAX_CHAR_LENGTH bytes). The function works by an internal conversion of TERM, when needed, from its input Python UTF-8 binary string format into Python Unicode format, and then truncating it safely to the given number of TF-8 characters, without possible mis-truncation in the middle of a multi-byte UTF-8 character that could otherwise happen if we would have been working with UTF-8 binary representation directly. Note that MAX_CHAR_LENGTH corresponds to the length of the term column in idxINDEX* tables. """ washed_term = unicode(term, 'utf-8').lower() if len(washed_term) <= max_char_length: # no need to truncate the term, because it will fit # nicely even if it uses four-byte UTF-8 characters return washed_term.encode('utf-8') else: # truncate the term in a safe position: return washed_term[:max_char_length].encode('utf-8') def wash_pattern(p): """Wash pattern passed by URL. Check for sanity of the wildcard by removing wildcards if they are appended to extremely short words (1-3 letters). TODO: instead of this approximative treatment, it will be much better to introduce a temporal limit, e.g. to kill a query if it does not finish in 10 seconds.""" # strip accents: # p = strip_accents(p) # FIXME: when available, strip accents all the time # add leading/trailing whitespace for the two following wildcard-sanity checking regexps: p = " " + p + " " # get rid of wildcards at the beginning of words: p = re_pattern_wildcards_at_beginning.sub("\\1", p) # replace spaces within quotes by __SPACE__ temporarily: p = re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", p) p = re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", p) p = re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", p) # get rid of extremely short words (1-3 letters with wildcards): p = re_pattern_short_words.sub("\\1", p) # replace back __SPACE__ by spaces: p = re_pattern_space.sub(" ", p) # replace special terms: p = re_pattern_today.sub(time.strftime("%Y-%m-%d", time.localtime()), p) # remove unnecessary whitespace: p = string.strip(p) return p def wash_field(f): """Wash field passed by URL.""" # get rid of unnecessary whitespace: f = string.strip(f) # wash old-style CDS Invenio/ALEPH 'f' field argument, e.g. replaces 'wau' and 'au' by 'author' if CFG_WEBSEARCH_FIELDS_CONVERT.has_key(string.lower(f)): f = CFG_WEBSEARCH_FIELDS_CONVERT[f] return f def wash_dates(d1="", d1y=0, d1m=0, d1d=0, d2="", d2y=0, d2m=0, d2d=0): """ Take user-submitted date arguments D1 (full datetime string) or (D1Y, D1M, D1Y) year, month, day tuple and D2 or (D2Y, D2M, D2Y) and return (YYY1-M1-D2 H1:M1:S2, YYY2-M2-D2 H2:M2:S2) datetime strings in the YYYY-MM-DD HH:MM:SS format suitable for time restricted searching. Note that when both D1 and (D1Y, D1M, D1D) parameters are present, the precedence goes to D1. Ditto for D2*. Note that when (D1Y, D1M, D1D) are taken into account, some values may be missing and are completed e.g. to 01 or 12 according to whether it is the starting or the ending date. """ datetext1, datetext2 = "", "" # sanity checking: if d1 == "" and d1y == 0 and d1m == 0 and d1d == 0 and d2 == "" and d2y == 0 and d2m == 0 and d2d == 0: return ("", "") # nothing selected, so return empty values # wash first (starting) date: if d1: # full datetime string takes precedence: datetext1 = d1 else: # okay, first date passed as (year,month,day): if d1y: datetext1 += "%04d" % d1y else: datetext1 += "0000" if d1m: datetext1 += "-%02d" % d1m else: datetext1 += "-01" if d1d: datetext1 += "-%02d" % d1d else: datetext1 += "-01" datetext1 += " 00:00:00" # wash second (ending) date: if d2: # full datetime string takes precedence: datetext2 = d2 else: # okay, second date passed as (year,month,day): if d2y: datetext2 += "%04d" % d2y else: datetext2 += "9999" if d2m: datetext2 += "-%02d" % d2m else: datetext2 += "-12" if d2d: datetext2 += "-%02d" % d2d else: datetext2 += "-31" # NOTE: perhaps we should add max(datenumber) in # given month, but for our quering it's not # needed, 31 will always do datetext2 += " 00:00:00" # okay, return constructed YYYY-MM-DD HH:MM:SS datetexts: return (datetext1, datetext2) def get_colID(c): "Return collection ID for collection name C. Return None if no match found." colID = None res = run_sql("SELECT id FROM collection WHERE name=%s", (c,), 1) if res: colID = res[0][0] return colID def get_coll_i18nname(c, ln=cdslang): """Return nicely formatted collection name (of name type 'ln', 'long name') for collection C in language LN.""" global collection_i18nname_cache global collection_i18nname_cache_timestamp # firstly, check whether the collectionname table was modified: if get_table_update_time('collectionname') > collection_i18nname_cache_timestamp: # yes it was, cache clear-up needed: collection_i18nname_cache = create_collection_i18nname_cache() # secondly, read i18n name from either the cache or return common name: out = c try: out = collection_i18nname_cache[c][ln] except KeyError: pass # translation in LN does not exist return out def get_field_i18nname(f, ln=cdslang): """Return nicely formatted field name (of type 'ln', 'long name') for field F in language LN.""" global field_i18nname_cache global field_i18nname_cache_timestamp # firstly, check whether the fieldname table was modified: if get_table_update_time('fieldname') > field_i18nname_cache_timestamp: # yes it was, cache clear-up needed: field_i18nname_cache = create_field_i18nname_cache() # secondly, read i18n name from either the cache or return common name: out = f try: out = field_i18nname_cache[f][ln] except KeyError: pass # translation in LN does not exist return out def get_coll_ancestors(coll): "Returns a list of ancestors for collection 'coll'." coll_ancestors = [] coll_ancestor = coll while 1: res = run_sql("""SELECT c.name FROM collection AS c LEFT JOIN collection_collection AS cc ON c.id=cc.id_dad LEFT JOIN collection AS ccc ON ccc.id=cc.id_son WHERE ccc.name=%s ORDER BY cc.id_dad ASC LIMIT 1""", (coll_ancestor,)) if res: coll_name = res[0][0] coll_ancestors.append(coll_name) coll_ancestor = coll_name else: break # ancestors found, return reversed list: coll_ancestors.reverse() return coll_ancestors def get_coll_sons(coll, type='r', public_only=1): """Return a list of sons (first-level descendants) of type 'type' for collection 'coll'. If public_only, then return only non-restricted son collections. """ coll_sons = [] query = "SELECT c.name FROM collection AS c "\ "LEFT JOIN collection_collection AS cc ON c.id=cc.id_son "\ "LEFT JOIN collection AS ccc ON ccc.id=cc.id_dad "\ "WHERE cc.type=%s AND ccc.name=%s" if public_only: query += " AND c.restricted IS NULL " query += " ORDER BY cc.score DESC" res = run_sql(query, (type, coll)) for name in res: coll_sons.append(name[0]) return coll_sons def get_coll_real_descendants(coll): """Return a list of all descendants of collection 'coll' that are defined by a 'dbquery'. IOW, we need to decompose compound collections like "A & B" into "A" and "B" provided that "A & B" has no associated database query defined. """ coll_sons = [] res = run_sql("""SELECT c.name,c.dbquery FROM collection AS c LEFT JOIN collection_collection AS cc ON c.id=cc.id_son LEFT JOIN collection AS ccc ON ccc.id=cc.id_dad WHERE ccc.name=%s ORDER BY cc.score DESC""", (coll,)) for name, dbquery in res: if dbquery: # this is 'real' collection, so return it: coll_sons.append(name) else: # this is 'composed' collection, so recurse: coll_sons.extend(get_coll_real_descendants(name)) return coll_sons def get_collection_reclist(coll): """Return hitset of recIDs that belong to the collection 'coll'. But firstly check the last updated date of the collection table. If it's newer than the cache timestamp, then empty the cache, since new records could have been added.""" global collection_reclist_cache global collection_reclist_cache_timestamp # firstly, check whether the collection table was modified: if get_table_update_time('collection') > collection_reclist_cache_timestamp: # yes it was, cache clear-up needed: collection_reclist_cache = create_collection_reclist_cache() # secondly, read reclist from either the cache or the database: if not collection_reclist_cache[coll]: # not yet it the cache, so calculate it and fill the cache: query = "SELECT nbrecs,reclist FROM collection WHERE name='%s'" % coll res = run_sql(query, None, 1) if res: try: set = HitSet(res[0][1]) except: set = HitSet() collection_reclist_cache[coll] = set # finally, return reclist: return collection_reclist_cache[coll] def coll_restricted_p(coll): "Predicate to test if the collection coll is restricted or not." if not coll: return 0 res = run_sql("SELECT restricted FROM collection WHERE name=%s", (coll,)) if res and res[0][0] is not None: return 1 else: return 0 def coll_restricted_group(coll): "Return Apache group to which the collection is restricted. Return None if it's public." if not coll: return None res = run_sql("SELECT restricted FROM collection WHERE name=%s", (coll,)) if res: return res[0][0] else: return None def create_collection_reclist_cache(): """Creates list of records belonging to collections. Called on startup and used later for intersecting search results with collection universe.""" global collection_reclist_cache_timestamp # populate collection reclist cache: collrecs = {} try: res = run_sql("SELECT name,reclist FROM collection") except Error: # database problems, set timestamp to zero and return empty cache collection_reclist_cache_timestamp = 0 return collrecs for name, reclist in res: collrecs[name] = None # this will be filled later during runtime by calling get_collection_reclist(coll) # update timestamp: try: collection_reclist_cache_timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) except NameError: collection_reclist_cache_timestamp = 0 return collrecs try: collection_reclist_cache.has_key(cdsname) except: try: collection_reclist_cache = create_collection_reclist_cache() except: collection_reclist_cache = {} def create_collection_i18nname_cache(): """Create cache of I18N collection names of type 'ln' (=long name). Called on startup and used later during the search time.""" global collection_i18nname_cache_timestamp # populate collection I18N name cache: names = {} try: res = run_sql("SELECT c.name,cn.ln,cn.value FROM collectionname AS cn, collection AS c WHERE cn.id_collection=c.id AND cn.type='ln'") # ln=long name except Error: # database problems, set timestamp to zero and return empty cache collection_i18nname_cache_timestamp = 0 return names for c, ln, i18nname in res: if i18nname: if not names.has_key(c): names[c] = {} names[c][ln] = i18nname # update timestamp: try: collection_i18nname_cache_timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) except NameError: collection_i18nname_cache_timestamp = 0 return names try: collection_i18nname_cache.has_key(cdsname) except: try: collection_i18nname_cache = create_collection_i18nname_cache() except: collection_i18nname_cache = {} def create_field_i18nname_cache(): """Create cache of I18N field names of type 'ln' (=long name). Called on startup and used later during the search time.""" global field_i18nname_cache_timestamp # populate field I18 name cache: names = {} try: res = run_sql("SELECT f.name,fn.ln,fn.value FROM fieldname AS fn, field AS f WHERE fn.id_field=f.id AND fn.type='ln'") # ln=long name except Error: # database problems, set timestamp to zero and return empty cache field_i18nname_cache_timestamp = 0 return names for f, ln, i18nname in res: if i18nname: if not names.has_key(f): names[f] = {} names[f][ln] = i18nname # update timestamp: try: field_i18nname_cache_timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) except NameError: field_i18nname_cache_timestamp = 0 return names try: field_i18nname_cache.has_key(cdsname) except: try: field_i18nname_cache = create_field_i18nname_cache() except: field_i18nname_cache = {} def browse_pattern(req, colls, p, f, rg, ln=cdslang): """Browse either biliographic phrases or words indexes, and display it.""" # load the right message language _ = gettext_set_language(ln) ## do we search in words indexes? if not f: return browse_in_bibwords(req, p, f) ## is p enclosed in quotes? (coming from exact search) if p.startswith('"') and p.endswith('"'): p = p[1:-1] p_orig = p ## okay, "real browse" follows: browsed_phrases = get_nearest_terms_in_bibxxx(p, f, rg, 1) while not browsed_phrases: # try again and again with shorter and shorter pattern: try: p = p[:-1] browsed_phrases = get_nearest_terms_in_bibxxx(p, f, rg, 1) except: # probably there are no hits at all: req.write(_("No values found.")) return ## try to check hits in these particular collection selection: browsed_phrases_in_colls = [] if 0: for phrase in browsed_phrases: phrase_hitset = HitSet() phrase_hitsets = search_pattern("", phrase, f, 'e') for coll in colls: phrase_hitset.union_update(phrase_hitsets[coll]) if len(phrase_hitset) > 0: # okay, this phrase has some hits in colls, so add it: browsed_phrases_in_colls.append([phrase, len(phrase_hitset)]) ## were there hits in collections? if browsed_phrases_in_colls == []: if browsed_phrases != []: #print_warning(req, """

    No match close to %s found in given collections. #Please try different term.

    Displaying matches in any collection...""" % p_orig) ## try to get nbhits for these phrases in any collection: for phrase in browsed_phrases: browsed_phrases_in_colls.append([phrase, get_nbhits_in_bibxxx(phrase, f)]) ## display results now: out = websearch_templates.tmpl_browse_pattern( f=f, fn=get_field_i18nname(f, ln), ln=ln, browsed_phrases_in_colls=browsed_phrases_in_colls, colls=colls, ) req.write(out) return def browse_in_bibwords(req, p, f, ln=cdslang): """Browse inside words indexes.""" if not p: return _ = gettext_set_language(ln) urlargd = {} urlargd.update(req.argd) urlargd['action'] = 'search' nearest_box = create_nearest_terms_box(urlargd, p, f, 'w', ln=ln, intro_text_p=0) req.write(websearch_templates.tmpl_search_in_bibwords( p = p, f = f, ln = ln, nearest_box = nearest_box )) return def search_pattern(req=None, p=None, f=None, m=None, ap=0, of="id", verbose=0, ln=cdslang): """Search for complex pattern 'p' within field 'f' according to matching type 'm'. Return hitset of recIDs. The function uses multi-stage searching algorithm in case of no exact match found. See the Search Internals document for detailed description. The 'ap' argument governs whether an alternative patterns are to be used in case there is no direct hit for (p,f,m). For example, whether to replace non-alphanumeric characters by spaces if it would give some hits. See the Search Internals document for detailed description. (ap=0 forbits the alternative pattern usage, ap=1 permits it.) The 'of' argument governs whether to print or not some information to the user in case of no match found. (Usually it prints the information in case of HTML formats, otherwise it's silent). The 'verbose' argument controls the level of debugging information to be printed (0=least, 9=most). All the parameters are assumed to have been previously washed. This function is suitable as a mid-level API. """ _ = gettext_set_language(ln) hitset_empty = HitSet() # sanity check: if not p: hitset_full = HitSet(trailing_bits=1) hitset_full.discard(0) # no pattern, so return all universe return hitset_full # search stage 1: break up arguments into basic search units: if verbose and of.startswith("h"): t1 = os.times()[4] basic_search_units = create_basic_search_units(req, p, f, m, of) if verbose and of.startswith("h"): t2 = os.times()[4] print_warning(req, "Search stage 1: basic search units are: %s" % basic_search_units) print_warning(req, "Search stage 1: execution took %.2f seconds." % (t2 - t1)) # search stage 2: do search for each search unit and verify hit presence: if verbose and of.startswith("h"): t1 = os.times()[4] basic_search_units_hitsets = [] for idx_unit in range(0, len(basic_search_units)): bsu_o, bsu_p, bsu_f, bsu_m = basic_search_units[idx_unit] basic_search_unit_hitset = search_unit(bsu_p, bsu_f, bsu_m) if verbose >= 9 and of.startswith("h"): print_warning(req, "Search stage 1: pattern %s gave hitlist %s" % (bsu_p, list(basic_search_unit_hitset))) if len(basic_search_unit_hitset) > 0 or \ ap==0 or \ bsu_o=="|" or \ ((idx_unit+1) 0: # we retain the new unit instead if of.startswith('h'): print_warning(req, _("No exact match found for %(x_query1)s, using %(x_query2)s instead...") % \ {'x_query1': "" + cgi.escape(bsu_p) + "", 'x_query2': "" + cgi.escape(bsu_pn) + ""}) basic_search_units[idx_unit][1] = bsu_pn basic_search_units_hitsets.append(basic_search_unit_hitset) else: # stage 2-3: no hits found either, propose nearest indexed terms: if of.startswith('h'): if req: if bsu_f == "recid": print_warning(req, "Requested record does not seem to exist.") else: print_warning(req, create_nearest_terms_box(req.argd, bsu_p, bsu_f, bsu_m, ln=ln)) return hitset_empty else: # stage 2-3: no hits found either, propose nearest indexed terms: if of.startswith('h'): if req: if bsu_f == "recid": print_warning(req, "Requested record does not seem to exist.") else: print_warning(req, create_nearest_terms_box(req.argd, bsu_p, bsu_f, bsu_m, ln=ln)) return hitset_empty if verbose and of.startswith("h"): t2 = os.times()[4] for idx_unit in range(0, len(basic_search_units)): print_warning(req, "Search stage 2: basic search unit %s gave %d hits." % (basic_search_units[idx_unit][1:], len(basic_search_units_hitsets[idx_unit]))) print_warning(req, "Search stage 2: execution took %.2f seconds." % (t2 - t1)) # search stage 3: apply boolean query for each search unit: if verbose and of.startswith("h"): t1 = os.times()[4] # let the initial set be the complete universe: hitset_in_any_collection = HitSet(trailing_bits=1) hitset_in_any_collection.discard(0) for idx_unit in range(0, len(basic_search_units)): this_unit_operation = basic_search_units[idx_unit][0] this_unit_hitset = basic_search_units_hitsets[idx_unit] if this_unit_operation == '+': hitset_in_any_collection.intersection_update(this_unit_hitset) elif this_unit_operation == '-': hitset_in_any_collection.difference_update(this_unit_hitset) elif this_unit_operation == '|': hitset_in_any_collection.union_update(this_unit_hitset) else: if of.startswith("h"): print_warning(req, "Invalid set operation %s." % this_unit_operation, "Error") if len(hitset_in_any_collection) == 0: # no hits found, propose alternative boolean query: if of.startswith('h'): nearestterms = [] for idx_unit in range(0, len(basic_search_units)): bsu_o, bsu_p, bsu_f, bsu_m = basic_search_units[idx_unit] if bsu_p.startswith("%") and bsu_p.endswith("%"): bsu_p = "'" + bsu_p[1:-1] + "'" bsu_nbhits = len(basic_search_units_hitsets[idx_unit]) # create a similar query, but with the basic search unit only argd = {} argd.update(req.argd) argd['p'] = bsu_p argd['f'] = bsu_f nearestterms.append((bsu_p, bsu_nbhits, argd)) text = websearch_templates.tmpl_search_no_boolean_hits( ln=ln, nearestterms=nearestterms) print_warning(req, text) if verbose and of.startswith("h"): t2 = os.times()[4] print_warning(req, "Search stage 3: boolean query gave %d hits." % len(hitset_in_any_collection)) print_warning(req, "Search stage 3: execution took %.2f seconds." % (t2 - t1)) return hitset_in_any_collection def search_unit(p, f=None, m=None): """Search for basic search unit defined by pattern 'p' and field 'f' and matching type 'm'. Return hitset of recIDs. All the parameters are assumed to have been previously washed. 'p' is assumed to be already a ``basic search unit'' so that it is searched as such and is not broken up in any way. Only wildcard and span queries are being detected inside 'p'. This function is suitable as a low-level API. """ ## create empty output results set: set = HitSet() if not p: # sanity checking return set if m == 'a' or m == 'r': # we are doing either direct bibxxx search or phrase search or regexp search set = search_unit_in_bibxxx(p, f, m) else: # we are doing bibwords search by default set = search_unit_in_bibwords(p, f) return set def search_unit_in_bibwords(word, f, decompress=zlib.decompress): """Searches for 'word' inside bibwordsX table for field 'f' and returns hitset of recIDs.""" set = HitSet() # will hold output result set set_used = 0 # not-yet-used flag, to be able to circumvent set operations # deduce into which bibwordsX table we will search: stemming_language = get_index_stemming_language(get_index_id_from_field("anyfield")) bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield") if f: index_id = get_index_id_from_field(f) if index_id: bibwordsX = "idxWORD%02dF" % index_id stemming_language = get_index_stemming_language(index_id) else: return HitSet() # word index f does not exist # wash 'word' argument and run query: word = string.replace(word, '*', '%') # we now use '*' as the truncation character words = string.split(word, "->", 1) # check for span query if len(words) == 2: word0 = re_word.sub('', words[0]) word1 = re_word.sub('', words[1]) if stemming_language: word0 = stem(word0, stemming_language) word1 = stem(word1, stemming_language) res = run_sql("SELECT term,hitlist FROM %s WHERE term BETWEEN %%s AND %%s" % bibwordsX, (wash_index_term(word0), wash_index_term(word1))) else: word = re_word.sub('', word) if stemming_language: word = stem(word, stemming_language) if string.find(word, '%') >= 0: # do we have wildcard in the word? res = run_sql("SELECT term,hitlist FROM %s WHERE term LIKE %%s" % bibwordsX, (wash_index_term(word),)) else: res = run_sql("SELECT term,hitlist FROM %s WHERE term=%%s" % bibwordsX, (wash_index_term(word),)) # fill the result set: for word, hitlist in res: hitset_bibwrd = HitSet(hitlist) # add the results: if set_used: set.union_update(hitset_bibwrd) else: set = hitset_bibwrd set_used = 1 # okay, return result set: return set def search_unit_in_bibxxx(p, f, type): """Searches for pattern 'p' inside bibxxx tables for field 'f' and returns hitset of recIDs found. The search type is defined by 'type' (e.g. equals to 'r' for a regexp search).""" p_orig = p # saving for eventual future 'no match' reporting query_addons = "" # will hold additional SQL code for the query query_params = () # will hold parameters for the query (their number may vary depending on TYPE argument) # wash arguments: f = string.replace(f, '*', '%') # replace truncation char '*' in field definition if type == 'r': query_addons = "REGEXP %s" query_params = (p,) else: p = string.replace(p, '*', '%') # we now use '*' as the truncation character ps = string.split(p, "->", 1) # check for span query: if len(ps) == 2: query_addons = "BETWEEN %s AND %s" query_params = (ps[0], ps[1]) else: if string.find(p, '%') > -1: query_addons = "LIKE %s" query_params = (ps[0],) else: query_addons = "= %s" query_params = (ps[0],) # construct 'tl' which defines the tag list (MARC tags) to search in: tl = [] if str(f[0]).isdigit() and str(f[1]).isdigit(): tl.append(f) # 'f' seems to be okay as it starts by two digits else: # convert old ALEPH tag names, if appropriate: (TODO: get rid of this before entering this function) if CFG_WEBSEARCH_FIELDS_CONVERT.has_key(string.lower(f)): f = CFG_WEBSEARCH_FIELDS_CONVERT[string.lower(f)] # deduce desired MARC tags on the basis of chosen 'f' tl = get_field_tags(f) if not tl: # f index does not exist, nevermind pass # okay, start search: l = [] # will hold list of recID that matched for t in tl: # deduce into which bibxxx table we will search: digit1, digit2 = int(t[0]), int(t[1]) bx = "bib%d%dx" % (digit1, digit2) bibx = "bibrec_bib%d%dx" % (digit1, digit2) # construct and run query: if t == "001": res = run_sql("SELECT id FROM bibrec WHERE id %s" % query_addons, query_params) else: query = "SELECT bibx.id_bibrec FROM %s AS bx LEFT JOIN %s AS bibx ON bx.id=bibx.id_bibxxx WHERE bx.value %s" % \ (bx, bibx, query_addons) if len(t) != 6 or t[-1:]=='%': # wildcard query, or only the beginning of field 't' # is defined, so add wildcard character: query += " AND bx.tag LIKE %s" res = run_sql(query, query_params + (t + '%',)) else: # exact query for 't': query += " AND bx.tag=%s" res = run_sql(query, query_params + (t,)) # fill the result set: for id_bibrec in res: if id_bibrec[0]: l.append(id_bibrec[0]) # check no of hits found: nb_hits = len(l) # okay, return result set: set = HitSet(l) return set def search_unit_in_bibrec(datetext1, datetext2, type='c'): """ Return hitset of recIDs found that were either created or modified (according to 'type' arg being 'c' or 'm') from datetext1 until datetext2, inclusive. Does not pay attention to pattern, collection, anything. Useful to intersect later on with the 'real' query. """ set = HitSet() if type.startswith("m"): type = "modification_date" else: type = "creation_date" # by default we are searching for creation dates res = run_sql("SELECT id FROM bibrec WHERE %s>=%%s AND %s<=%%s" % (type, type), (datetext1, datetext2)) for row in res: set += row[0] return set def intersect_results_with_collrecs(req, hitset_in_any_collection, colls, ap=0, of="hb", verbose=0, ln=cdslang): """Return dict of hitsets given by intersection of hitset with the collection universes.""" _ = gettext_set_language(ln) # search stage 4: intersect with the collection universe: if verbose and of.startswith("h"): t1 = os.times()[4] results = {} results_nbhits = 0 for coll in colls: results[coll] = hitset_in_any_collection & get_collection_reclist(coll) results_nbhits += len(results[coll]) if results_nbhits == 0: # no hits found, try to search in Home: results_in_Home = hitset_in_any_collection & get_collection_reclist(cdsname) if len(results_in_Home) > 0: # some hits found in Home, so propose this search: if of.startswith("h"): url = websearch_templates.build_search_url(req.argd, cc=cdsname, c=[]) print_warning(req, _("No match found in collection %(x_collection)s. Other public collections gave %(x_url_open)s%(x_nb_hits)d hits%(x_url_close)s.") %\ {'x_collection': '' + string.join([get_coll_i18nname(coll, ln) for coll in colls], ', ') + '', 'x_url_open': '' % (url), 'x_nb_hits': len(results_in_Home), 'x_url_close': ''}) results = {} else: # no hits found in Home, recommend different search terms: if of.startswith("h"): print_warning(req, _("No public collection matched your query. " "If you were looking for a non-public document, please choose " "the desired restricted collection first.")) results = {} if verbose and of.startswith("h"): t2 = os.times()[4] print_warning(req, "Search stage 4: intersecting with collection universe gave %d hits." % results_nbhits) print_warning(req, "Search stage 4: execution took %.2f seconds." % (t2 - t1)) return results def intersect_results_with_hitset(req, results, hitset, ap=0, aptext="", of="hb"): """Return intersection of search 'results' (a dict of hitsets with collection as key) with the 'hitset', i.e. apply 'hitset' intersection to each collection within search 'results'. If the final 'results' set is to be empty, and 'ap' (approximate pattern) is true, and then print the `warningtext' and return the original 'results' set unchanged. If 'ap' is false, then return empty results set. """ if ap: results_ap = copy.deepcopy(results) else: results_ap = {} # will return empty dict in case of no hits found nb_total = 0 for coll in results.keys(): results[coll].intersection_update(hitset) nb_total += len(results[coll]) if nb_total == 0: if of.startswith("h"): print_warning(req, aptext) results = results_ap return results def create_similarly_named_authors_link_box(author_name, ln=cdslang): """Return a box similar to ``Not satisfied...'' one by proposing author searches for similar names. Namely, take AUTHOR_NAME and the first initial of the firstame (after comma) and look into author index whether authors with e.g. middle names exist. Useful mainly for CERN Library that sometimes contains name forms like Ellis-N, Ellis-Nick, Ellis-Nicolas all denoting the same person. The box isn't proposed if no similarly named authors are found to exist. """ # return nothing if not configured: if CFG_WEBSEARCH_CREATE_SIMILARLY_NAMED_AUTHORS_LINK_BOX == 0: return "" # return empty box if there is no initial: if re.match(r'[^ ,]+, [^ ]', author_name) is None: return "" # firstly find name comma initial: author_name_to_search = re.sub(r'^([^ ,]+, +[^ ,]).*$', '\\1', author_name) # secondly search for similar name forms: similar_author_names = {} for name in author_name_to_search, strip_accents(author_name_to_search): for tag in get_field_tags("author"): # deduce into which bibxxx table we will search: digit1, digit2 = int(tag[0]), int(tag[1]) bx = "bib%d%dx" % (digit1, digit2) bibx = "bibrec_bib%d%dx" % (digit1, digit2) if len(tag) != 6 or tag[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value LIKE %%s AND bx.tag LIKE %%s""" % bx, (name + "%", tag + "%")) else: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value LIKE %%s AND bx.tag=%%s""" % bx, (name + "%", tag)) for row in res: similar_author_names[row[0]] = 1 # remove the original name and sort the list: try: del similar_author_names[author_name] except KeyError: pass # thirdly print the box: out = "" if similar_author_names: out_authors = similar_author_names.keys() out_authors.sort() tmp_authors = [] for out_author in out_authors: nbhits = get_nbhits_in_bibxxx(out_author, "author") if nbhits: tmp_authors.append((out_author, nbhits)) out += websearch_templates.tmpl_similar_author_names( authors=tmp_authors, ln=ln) return out def create_nearest_terms_box(urlargd, p, f, t='w', n=5, ln=cdslang, intro_text_p=True): """Return text box containing list of 'n' nearest terms above/below 'p' for the field 'f' for matching type 't' (words/phrases) in language 'ln'. Propose new searches according to `urlargs' with the new words. If `intro_text_p' is true, then display the introductory message, otherwise print only the nearest terms in the box content. """ # load the right message language _ = gettext_set_language(ln) out = "" nearest_terms = [] if not p: # sanity check p = "." # look for nearest terms: if t == 'w': nearest_terms = get_nearest_terms_in_bibwords(p, f, n, n) if not nearest_terms: return "%s %s." % (_("No words index available for"), get_field_i18nname(f, ln)) else: nearest_terms = get_nearest_terms_in_bibxxx(p, f, n, n) if not nearest_terms: return "%s %s." % (_("No phrase index available for"), get_field_i18nname(f, ln)) terminfo = [] for term in nearest_terms: if t == 'w': hits = get_nbhits_in_bibwords(term, f) else: hits = get_nbhits_in_bibxxx(term, f) argd = {} argd.update(urlargd) # check which fields contained the requested parameter, and replace it. for (px, fx) in ('p', 'f'), ('p1', 'f1'), ('p2', 'f2'), ('p3', 'f3'): if px in argd: if f == argd[fx] or f == "anyfield" or f == "": if string.find(argd[px], p) > -1: argd[px] = string.replace(argd[px], p, term) break else: if string.find(argd[px], f+':'+p) > -1: argd[px] = string.replace(argd[px], f+':'+p, f+':'+term) break elif string.find(argd[px], f+':"'+p+'"') > -1: argd[px] = string.replace(argd[px], f+':"'+p+'"', f+':"'+term+'"') break terminfo.append((term, hits, argd)) intro = "" if intro_text_p: # add full leading introductory text if f: intro = _("Search term %(x_term)s inside index %(x_index)s did not match any record. Nearest terms in any collection are:") % \ {'x_term': "" + cgi.escape(p.startswith("%") and p.endswith("%") and p[1:-1] or p) + "", 'x_index': "" + cgi.escape(get_field_i18nname(f, ln)) + ""} else: intro = _("Search term %s did not match any record. Nearest terms in any collection are:") % \ ("" + cgi.escape(p.startswith("%") and p.endswith("%") and p[1:-1] or p) + "") return websearch_templates.tmpl_nearest_term_box(p=p, ln=ln, f=f, terminfo=terminfo, intro=intro) def get_nearest_terms_in_bibwords(p, f, n_below, n_above): """Return list of +n -n nearest terms to word `p' in index for field `f'.""" nearest_words = [] # will hold the (sorted) list of nearest words to return # deduce into which bibwordsX table we will search: bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield") if f: index_id = get_index_id_from_field(f) if index_id: bibwordsX = "idxWORD%02dF" % index_id else: return nearest_words # firstly try to get `n' closest words above `p': res = run_sql("SELECT term FROM %s WHERE term<%%s ORDER BY term DESC LIMIT %%s" % bibwordsX, (p, n_above)) for row in res: nearest_words.append(row[0]) nearest_words.reverse() # secondly insert given word `p': nearest_words.append(p) # finally try to get `n' closest words below `p': res = run_sql("SELECT term FROM %s WHERE term>%%s ORDER BY term ASC LIMIT %%s" % bibwordsX, (p, n_below)) for row in res: nearest_words.append(row[0]) return nearest_words def get_nearest_terms_in_bibxxx(p, f, n_below, n_above): """Browse (-n_above, +n_below) closest bibliographic phrases for the given pattern p in the given field f, regardless of collection. Return list of [phrase1, phrase2, ... , phrase_n].""" ## determine browse field: if not f and string.find(p, ":") > 0: # does 'p' contain ':'? f, p = string.split(p, ":", 1) ## We are going to take max(n_below, n_above) as the number of ## values to ferch from bibXXx. This is needed to work around ## MySQL UTF-8 sorting troubles in 4.0.x. Proper solution is to ## use MySQL 4.1.x or our own idxPHRASE in the future. n_fetch = 2*max(n_below, n_above) ## construct 'tl' which defines the tag list (MARC tags) to search in: tl = [] if str(f[0]).isdigit() and str(f[1]).isdigit(): tl.append(f) # 'f' seems to be okay as it starts by two digits else: # deduce desired MARC tags on the basis of chosen 'f' tl = get_field_tags(f) ## start browsing to fetch list of hits: browsed_phrases = {} # will hold {phrase1: 1, phrase2: 1, ..., phraseN: 1} dict of browsed phrases (to make them unique) # always add self to the results set: browsed_phrases[p.startswith("%") and p.endswith("%") and p[1:-1] or p] = 1 for t in tl: # deduce into which bibxxx table we will search: digit1, digit2 = int(t[0]), int(t[1]) bx = "bib%d%dx" % (digit1, digit2) bibx = "bibrec_bib%d%dx" % (digit1, digit2) # firstly try to get `n' closest phrases above `p': if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value<%%s AND bx.tag LIKE %%s ORDER BY bx.value DESC LIMIT %%s""" % bx, (p, t + "%", n_fetch)) else: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value<%%s AND bx.tag=%%s ORDER BY bx.value DESC LIMIT %%s""" % bx, (p, t, n_fetch)) for row in res: browsed_phrases[row[0]] = 1 # secondly try to get `n' closest phrases equal to or below `p': if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value>=%%s AND bx.tag LIKE %%s ORDER BY bx.value ASC LIMIT %%s""" % bx, (p, t + "%", n_fetch)) else: res = run_sql("""SELECT bx.value FROM %s AS bx WHERE bx.value>=%%s AND bx.tag=%%s ORDER BY bx.value ASC LIMIT %%s""" % bx, (p, t, n_fetch)) for row in res: browsed_phrases[row[0]] = 1 # select first n words only: (this is needed as we were searching # in many different tables and so aren't sure we have more than n # words right; this of course won't be needed when we shall have # one ACC table only for given field): phrases_out = browsed_phrases.keys() phrases_out.sort(lambda x, y: cmp(string.lower(strip_accents(x)), string.lower(strip_accents(y)))) # find position of self: try: idx_p = phrases_out.index(p) except: idx_p = len(phrases_out)/2 # return n_above and n_below: return phrases_out[max(0, idx_p-n_above):idx_p+n_below] def get_nbhits_in_bibwords(word, f): """Return number of hits for word 'word' inside words index for field 'f'.""" out = 0 # deduce into which bibwordsX table we will search: bibwordsX = "idxWORD%02dF" % get_index_id_from_field("anyfield") if f: index_id = get_index_id_from_field(f) if index_id: bibwordsX = "idxWORD%02dF" % index_id else: return 0 if word: res = run_sql("SELECT hitlist FROM %s WHERE term=%%s" % bibwordsX, (word,)) for hitlist in res: out += len(HitSet(hitlist[0])) return out def get_nbhits_in_bibxxx(p, f): """Return number of hits for word 'word' inside words index for field 'f'.""" ## determine browse field: if not f and string.find(p, ":") > 0: # does 'p' contain ':'? f, p = string.split(p, ":", 1) ## construct 'tl' which defines the tag list (MARC tags) to search in: tl = [] if str(f[0]).isdigit() and str(f[1]).isdigit(): tl.append(f) # 'f' seems to be okay as it starts by two digits else: # deduce desired MARC tags on the basis of chosen 'f' tl = get_field_tags(f) # start searching: recIDs = {} # will hold dict of {recID1: 1, recID2: 1, ..., } (unique recIDs, therefore) for t in tl: # deduce into which bibxxx table we will search: digit1, digit2 = int(t[0]), int(t[1]) bx = "bib%d%dx" % (digit1, digit2) bibx = "bibrec_bib%d%dx" % (digit1, digit2) if len(t) != 6 or t[-1:]=='%': # only the beginning of field 't' is defined, so add wildcard character: res = run_sql("""SELECT bibx.id_bibrec FROM %s AS bibx, %s AS bx WHERE bx.value=%%s AND bx.tag LIKE %%s AND bibx.id_bibxxx=bx.id""" % (bibx, bx), (p, t + "%")) else: res = run_sql("""SELECT bibx.id_bibrec FROM %s AS bibx, %s AS bx WHERE bx.value=%%s AND bx.tag=%%s AND bibx.id_bibxxx=bx.id""" % (bibx, bx), (p, t)) for row in res: recIDs[row[0]] = 1 return len(recIDs) def get_mysql_recid_from_aleph_sysno(sysno): """Returns DB's recID for ALEPH sysno passed in the argument (e.g. "002379334CER"). Returns None in case of failure.""" out = None res = run_sql("""SELECT bb.id_bibrec FROM bibrec_bib97x AS bb, bib97x AS b WHERE b.value=%s AND b.tag='970__a' AND bb.id_bibxxx=b.id""", (sysno,)) if res: out = res[0][0] return out def guess_primary_collection_of_a_record(recID): """Return primary collection name a record recid belongs to, by testing 980 identifier. May lead to bad guesses when a collection is defined dynamically bia dbquery. In that case, return 'cdsname'.""" out = cdsname dbcollids = get_fieldvalues(recID, "980__a") if dbcollids: dbquery = "collection:" + dbcollids[0] res = run_sql("SELECT name FROM collection WHERE dbquery=%s", (dbquery,)) if res: out = res[0][0] return out def get_tag_name(tag_value, prolog="", epilog=""): """Return tag name from the known tag value, by looking up the 'tag' table. Return empty string in case of failure. Example: input='100__%', output=first author'.""" out = "" res = run_sql("SELECT name FROM tag WHERE value=%s", (tag_value,)) if res: out = prolog + res[0][0] + epilog return out def get_fieldcodes(): """Returns a list of field codes that may have been passed as 'search options' in URL. Example: output=['subject','division'].""" out = [] res = run_sql("SELECT DISTINCT(code) FROM field") for row in res: out.append(row[0]) return out def get_field_tags(field): """Returns a list of MARC tags for the field code 'field'. Returns empty list in case of error. Example: field='author', output=['100__%','700__%'].""" out = [] query = """SELECT t.value FROM tag AS t, field_tag AS ft, field AS f WHERE f.code=%s AND ft.id_field=f.id AND t.id=ft.id_tag ORDER BY ft.score DESC""" res = run_sql(query, (field, )) for val in res: out.append(val[0]) return out def get_fieldvalues(recID, tag): """Return list of field values for field TAG inside record RECID.""" out = [] if tag == "001___": # we have asked for recID that is not stored in bibXXx tables out.append(str(recID)) else: # we are going to look inside bibXXx tables digits = tag[0:2] try: intdigits = int(digits) if intdigits < 0 or intdigits > 99: raise ValueError except ValueError: # invalid tag value asked for return [] bx = "bib%sx" % digits bibx = "bibrec_bib%sx" % digits query = "SELECT bx.value FROM %s AS bx, %s AS bibx " \ " WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag LIKE '%s' " \ " ORDER BY bibx.field_number, bx.tag ASC" % (bx, bibx, recID, tag) res = run_sql(query) for row in res: out.append(row[0]) return out def get_fieldvalues_alephseq_like(recID, tags_in): """Return buffer of ALEPH sequential-like textual format with fields found in the list TAGS_IN for record RECID.""" out = "" if type(tags_in) is not list: tags_in = [tags_in,] if len(tags_in) == 1 and len(tags_in[0]) == 6: ## case A: one concrete subfield asked, so print its value if found ## (use with care: can false you if field has multiple occurrences) out += string.join(get_fieldvalues(recID, tags_in[0]),"\n") else: ## case B: print our "text MARC" format; works safely all the time # find out which tags to output: dict_of_tags_out = {} if not tags_in: for i in range(0, 10): for j in range(0, 10): dict_of_tags_out["%d%d%%" % (i, j)] = 1 else: for tag in tags_in: if len(tag) == 0: for i in range(0, 10): for j in range(0, 10): dict_of_tags_out["%d%d%%" % (i, j)] = 1 elif len(tag) == 1: for j in range(0, 10): dict_of_tags_out["%s%d%%" % (tag, j)] = 1 elif len(tag) < 5: dict_of_tags_out["%s%%" % tag] = 1 elif tag >= 6: dict_of_tags_out[tag[0:5]] = 1 tags_out = dict_of_tags_out.keys() tags_out.sort() # search all bibXXx tables as needed: for tag in tags_out: digits = tag[0:2] try: intdigits = int(digits) if intdigits < 0 or intdigits > 99: raise ValueError except ValueError: # invalid tag value asked for continue if tag.startswith("001") or tag.startswith("00%"): if out: out += "\n" out += "%09d %s %d" % (recID, "001__", recID) bx = "bib%sx" % digits bibx = "bibrec_bib%sx" % digits query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\ "WHERE bb.id_bibrec='%s' AND b.id=bb.id_bibxxx AND b.tag LIKE '%s%%' "\ "ORDER BY bb.field_number, b.tag ASC" % (bx, bibx, recID, tag) res = run_sql(query) # go through fields: field_number_old = -999 field_old = "" for row in res: field, value, field_number = row[0], row[1], row[2] ind1, ind2 = field[3], field[4] if ind1 == "_": ind1 = "" if ind2 == "_": ind2 = "" # print field tag if field_number != field_number_old or field[:-1] != field_old[:-1]: if out: out += "\n" out += "%09d %s " % (recID, field[:5]) field_number_old = field_number field_old = field # print subfield value if field[0:2] == "00" and field[-1:] == "_": out += value else: out += "$$%s%s" % (field[-1:], value) return out def record_exists(recID): """Return 1 if record RECID exists. Return 0 if it doesn't exist. Return -1 if it exists but is marked as deleted.""" out = 0 query = "SELECT id FROM bibrec WHERE id='%s'" % recID res = run_sql(query, None, 1) if res: # record exists; now check whether it isn't marked as deleted: dbcollids = get_fieldvalues(recID, "980__%") if ("DELETED" in dbcollids) or (CFG_CERN_SITE and "DUMMY" in dbcollids): out = -1 # exists, but marked as deleted else: out = 1 # exists fine return out def record_public_p(recID): """Return 1 if the record is public, i.e. if it can be found in the Home collection. Return 0 otherwise. """ return recID in get_collection_reclist(cdsname) def get_creation_date(recID, fmt="%Y-%m-%d"): "Returns the creation date of the record 'recID'." out = "" res = run_sql("SELECT DATE_FORMAT(creation_date,%s) FROM bibrec WHERE id=%s", (fmt, recID), 1) if res: out = res[0][0] return out def get_modification_date(recID, fmt="%Y-%m-%d"): "Returns the date of last modification for the record 'recID'." out = "" res = run_sql("SELECT DATE_FORMAT(modification_date,%s) FROM bibrec WHERE id=%s", (fmt, recID), 1) if res: out = res[0][0] return out def print_warning(req, msg, type='', prologue='
    ', epilogue='
    '): "Prints warning message and flushes output." if req and msg: req.write(websearch_templates.tmpl_print_warning( msg = msg, type = type, prologue = prologue, epilogue = epilogue, )) return def print_search_info(p, f, sf, so, sp, rm, of, ot, collection=cdsname, nb_found=-1, jrec=1, rg=10, as=0, ln=cdslang, p1="", p2="", p3="", f1="", f2="", f3="", m1="", m2="", m3="", op1="", op2="", sc=1, pl_in_url="", d1y=0, d1m=0, d1d=0, d2y=0, d2m=0, d2d=0, dt="", cpu_time=-1, middle_only=0): """Prints stripe with the information on 'collection' and 'nb_found' results and CPU time. Also, prints navigation links (beg/next/prev/end) inside the results set. If middle_only is set to 1, it will only print the middle box information (beg/netx/prev/end/etc) links. This is suitable for displaying navigation links at the bottom of the search results page.""" out = "" # sanity check: if jrec < 1: jrec = 1 if jrec > nb_found: jrec = max(nb_found-rg+1, 1) return websearch_templates.tmpl_print_search_info( ln = ln, weburl = weburl, collection = collection, as = as, collection_name = get_coll_i18nname(collection, ln), collection_id = get_colID(collection), middle_only = middle_only, rg = rg, nb_found = nb_found, sf = sf, so = so, rm = rm, of = of, ot = ot, p = p, f = f, p1 = p1, p2 = p2, p3 = p3, f1 = f1, f2 = f2, f3 = f3, m1 = m1, m2 = m2, m3 = m3, op1 = op1, op2 = op2, pl_in_url = pl_in_url, d1y = d1y, d1m = d1m, d1d = d1d, d2y = d2y, d2m = d2m, d2d = d2d, dt = dt, jrec = jrec, sc = sc, sp = sp, all_fieldcodes = get_fieldcodes(), cpu_time = cpu_time, ) def print_results_overview(req, colls, results_final_nb_total, results_final_nb, cpu_time, ln=cdslang, ec=[]): """Prints results overview box with links to particular collections below.""" out = "" new_colls = [] for coll in colls: new_colls.append({ 'id': get_colID(coll), 'code': coll, 'name': get_coll_i18nname(coll, ln), }) return websearch_templates.tmpl_print_results_overview( ln = ln, weburl = weburl, results_final_nb_total = results_final_nb_total, results_final_nb = results_final_nb, cpu_time = cpu_time, colls = new_colls, ec = ec, ) def sort_records(req, recIDs, sort_field='', sort_order='d', sort_pattern='', verbose=0, of='hb', ln=cdslang): """Sort records in 'recIDs' list according sort field 'sort_field' in order 'sort_order'. If more than one instance of 'sort_field' is found for a given record, try to choose that that is given by 'sort pattern', for example "sort by report number that starts by CERN-PS". Note that 'sort_field' can be field code like 'author' or MARC tag like '100__a' directly.""" _ = gettext_set_language(ln) ## check arguments: if not sort_field: return recIDs if len(recIDs) > CFG_WEBSEARCH_NB_RECORDS_TO_SORT: if of.startswith('h'): print_warning(req, _("Sorry, sorting is allowed on sets of up to %d records only. Using default sort order.") % CFG_WEBSEARCH_NB_RECORDS_TO_SORT, "Warning") return recIDs sort_fields = string.split(sort_field, ",") recIDs_dict = {} recIDs_out = [] ## first deduce sorting MARC tag out of the 'sort_field' argument: tags = [] for sort_field in sort_fields: if sort_field and str(sort_field[0:2]).isdigit(): # sort_field starts by two digits, so this is probably a MARC tag already tags.append(sort_field) else: # let us check the 'field' table query = """SELECT DISTINCT(t.value) FROM tag AS t, field_tag AS ft, field AS f WHERE f.code='%s' AND ft.id_field=f.id AND t.id=ft.id_tag ORDER BY ft.score DESC""" % sort_field res = run_sql(query) if res: for row in res: tags.append(row[0]) else: if of.startswith('h'): print_warning(req, _("Sorry, %s does not seem to be a valid sort option. Choosing title sort instead.") % sort_field, "Error") tags.append("245__a") if verbose >= 3: print_warning(req, "Sorting by tags %s." % tags) if sort_pattern: print_warning(req, "Sorting preferentially by %s." % sort_pattern) ## check if we have sorting tag defined: if tags: # fetch the necessary field values: for recID in recIDs: val = "" # will hold value for recID according to which sort vals = [] # will hold all values found in sorting tag for recID for tag in tags: vals.extend(get_fieldvalues(recID, tag)) if sort_pattern: # try to pick that tag value that corresponds to sort pattern bingo = 0 for v in vals: if v.lower().startswith(sort_pattern.lower()): # bingo! bingo = 1 val = v break if not bingo: # sort_pattern not present, so add other vals after spaces val = sort_pattern + " " + string.join(vals) else: # no sort pattern defined, so join them all together val = string.join(vals) val = strip_accents(val.lower()) # sort values regardless of accents and case if recIDs_dict.has_key(val): recIDs_dict[val].append(recID) else: recIDs_dict[val] = [recID] # sort them: recIDs_dict_keys = recIDs_dict.keys() recIDs_dict_keys.sort() # now that keys are sorted, create output array: for k in recIDs_dict_keys: for s in recIDs_dict[k]: recIDs_out.append(s) # ascending or descending? if sort_order == 'a': recIDs_out.reverse() # okay, we are done return recIDs_out else: # good, no sort needed return recIDs def print_records(req, recIDs, jrec=1, rg=10, format='hb', ot='', ln=cdslang, relevances=[], relevances_prologue="(", relevances_epilogue="%%)", decompress=zlib.decompress, search_pattern='', print_records_prologue_p=True, print_records_epilogue_p=True, verbose=0, tab=''): """ - Prints list of records 'recIDs' formatted accoding to 'format' in + Prints list of records 'recIDs' formatted according to 'format' in groups of 'rg' starting from 'jrec'. Assumes that the input list 'recIDs' is sorted in reverse order, so it counts records from tail to head. A value of 'rg=-9999' means to print all records: to be used with care. Print also list of RELEVANCES for each record (if defined), in between RELEVANCE_PROLOGUE and RELEVANCE_EPILOGUE. Print prologue and/or epilogue specific to 'format' if 'print_records_prologue_p' and/or print_records_epilogue_p' are True. """ # load the right message language _ = gettext_set_language(ln) # sanity checking: if req is None: return - # get user id (for formatting based on priviledge) - uid = getUid(req) + # get user_info (for formatting based on user) + user_info = collect_user_info(req) if len(recIDs): nb_found = len(recIDs) if rg == -9999: # print all records rg = nb_found else: rg = abs(rg) if jrec < 1: # sanity checks jrec = 1 if jrec > nb_found: jrec = max(nb_found-rg+1, 1) # will print records from irec_max to irec_min excluded: irec_max = nb_found - jrec irec_min = nb_found - jrec - rg if irec_min < 0: irec_min = -1 if irec_max >= nb_found: irec_max = nb_found - 1 #req.write("%s:%d-%d" % (recIDs, irec_min, irec_max)) if format.startswith('x'): # print header if needed if print_records_prologue_p: print_records_prologue(req, format) # print records recIDs_to_print = [recIDs[x] for x in range(irec_max, irec_min, -1)] format_records(recIDs_to_print, format, ln=ln, search_pattern=search_pattern, record_separator="\n", - uid=uid, + user_info=user_info, req=req) # print footer if needed if print_records_epilogue_p: print_records_epilogue(req, format) elif format.startswith('t') or str(format[0:3]).isdigit(): # we are doing plain text output: for irec in range(irec_max, irec_min, -1): x = print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) req.write(x) if x: req.write('\n') elif format == 'excel': recIDs_to_print = [recIDs[x] for x in range(irec_max, irec_min, -1)] create_excel(recIDs=recIDs_to_print, req=req, ln=ln) else: # we are doing HTML output: if format == 'hp' or format.startswith("hb_") or format.startswith("hd_"): # portfolio and on-the-fly formats: for irec in range(irec_max, irec_min, -1): req.write(print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose)) + user_info=user_info, verbose=verbose)) elif format.startswith("hb"): # HTML brief format: req.write(websearch_templates.tmpl_record_format_htmlbrief_header( ln = ln)) for irec in range(irec_max, irec_min, -1): row_number = jrec+irec_max-irec recid = recIDs[irec] if relevances and relevances[irec]: relevance = relevances[irec] else: relevance = '' record = print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) req.write(websearch_templates.tmpl_record_format_htmlbrief_body( ln = ln, recid = recid, row_number = row_number, relevance = relevance, record = record, relevances_prologue = relevances_prologue, relevances_epilogue = relevances_epilogue, )) req.write(websearch_templates.tmpl_record_format_htmlbrief_footer( ln = ln)) elif format.startswith("hd"): # HTML detailed format: for irec in range(irec_max, irec_min, -1): unordered_tabs = get_detailed_page_tabs(get_colID(guess_primary_collection_of_a_record(recIDs[irec])), recIDs[irec], ln=ln) ordered_tabs_id = [(tab_id, values['order']) for (tab_id, values) in unordered_tabs.iteritems()] ordered_tabs_id.sort(lambda x,y: cmp(x[1],y[1])) link_ln = '' if ln != cdslang: link_ln = '?ln=%s' % ln tabs = [(unordered_tabs[tab_id]['label'], \ '%s/record/%s/%s%s' % (weburl, recIDs[irec], tab_id, link_ln), \ tab_id == tab, unordered_tabs[tab_id]['enabled']) \ for (tab_id, order) in ordered_tabs_id if unordered_tabs[tab_id]['visible'] == True] content = '' # load content if tab == 'usage': r = calculate_reading_similarity_list(recIDs[irec], "downloads") downloadsimilarity = None downloadhistory = None #if r: # downloadsimilarity = r if CFG_BIBRANK_SHOW_DOWNLOAD_GRAPHS: downloadhistory = create_download_history_graph_and_box(recIDs[irec], ln) r = calculate_reading_similarity_list(recIDs[irec], "pageviews") viewsimilarity = None if r: viewsimilarity = r content = websearch_templates.tmpl_detailed_record_statistics(recIDs[irec], ln, downloadsimilarity=downloadsimilarity, downloadhistory=downloadhistory, viewsimilarity=viewsimilarity) req.write(webstyle_templates.detailed_record_container(content, recIDs[irec], tabs, ln)) elif tab == 'citations': citinglist = [] citationhistory = None recid = recIDs[irec] selfcited = get_self_cited_by(recid) r = calculate_cited_by_list(recid) if r: citinglist = r citationhistory = create_citation_history_graph_and_box(recid, ln) r = calculate_co_cited_with_list(recid) cociting = None if r: cociting = r content = websearch_templates.tmpl_detailed_record_citations(recid, ln, citinglist=citinglist, citationhistory=citationhistory, cociting=cociting, selfcited=selfcited) req.write(webstyle_templates.detailed_record_container(content, recid, tabs, ln)) elif tab == 'references': - content = format_record(recIDs[irec], 'HDREF', ln=ln, uid=uid, verbose=verbose) + content = format_record(recIDs[irec], 'HDREF', ln=ln, user_info=user_info, verbose=verbose) req.write(webstyle_templates.detailed_record_container(content, recIDs[irec], tabs, ln)) else: # Metadata tab content = print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) creationdate = None modifydate = None if record_exists(recIDs[irec]) == 1: creationdate = get_creation_date(recIDs[irec]) modifydate = get_modification_date(recIDs[irec]) content = websearch_templates.tmpl_detailed_record_metadata( recID = recIDs[irec], ln = ln, format = format, creationdate = creationdate, modifydate = modifydate, content = content) req.write(webstyle_templates.detailed_record_container(content, recIDs[irec], tabs, ln=ln, creationdate=creationdate, modifydate=modifydate, show_short_rec_p=False)) if len(tabs) > 0: # Add the mini box at bottom of the page if CFG_WEBCOMMENT_ALLOW_REVIEWS: from invenio.webcomment import get_mini_reviews reviews = get_mini_reviews(recid = recIDs[irec], ln=ln) else: reviews = '' - actions = format_record(recIDs[irec], 'HDACT', ln=ln, uid=uid, verbose=verbose) - files = format_record(recIDs[irec], 'HDFILE', ln=ln, uid=uid, verbose=verbose) + actions = format_record(recIDs[irec], 'HDACT', ln=ln, user_info=user_info, verbose=verbose) + files = format_record(recIDs[irec], 'HDFILE', ln=ln, user_info=user_info, verbose=verbose) req.write(webstyle_templates.detailed_record_mini_panel(recIDs[irec], ln, format, files=files, reviews=reviews, actions=actions)) else: # Other formats for irec in range(irec_max, irec_min, -1): req.write(print_record(recIDs[irec], format, ot, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose)) + user_info=user_info, verbose=verbose)) else: print_warning(req, _("Use different search terms.")) def print_records_prologue(req, format): """ Print the appropriate prologue for list of records in the given format. """ prologue = "" # no prologue needed for HTML or Text formats if format.startswith('xm'): prologue = websearch_templates.tmpl_xml_marc_prologue() elif format.startswith('xn'): prologue = websearch_templates.tmpl_xml_nlm_prologue() elif format.startswith('xr'): prologue = websearch_templates.tmpl_xml_rss_prologue() elif format.startswith('x'): prologue = websearch_templates.tmpl_xml_default_prologue() req.write(prologue) def print_records_epilogue(req, format): """ Print the appropriate epilogue for list of records in the given format. """ epilogue = "" # no epilogue needed for HTML or Text formats if format.startswith('xm'): epilogue = websearch_templates.tmpl_xml_marc_epilogue() elif format.startswith('xn'): epilogue = websearch_templates.tmpl_xml_nlm_epilogue() elif format.startswith('xr'): epilogue = websearch_templates.tmpl_xml_rss_epilogue() elif format.startswith('x'): epilogue = websearch_templates.tmpl_xml_default_epilogue() req.write(epilogue) def print_record(recID, format='hb', ot='', ln=cdslang, decompress=zlib.decompress, - search_pattern=None, uid=None, verbose=0): + search_pattern=None, user_info=None, verbose=0): """Prints record 'recID' formatted accoding to 'format'.""" _ = gettext_set_language(ln) out = "" # sanity check: record_exist_p = record_exists(recID) if record_exist_p == 0: # doesn't exist return out # New Python BibFormat procedure for formatting # Old procedure follows further below # We must still check some special formats, but these # should disappear when BibFormat improves. if not (CFG_BIBFORMAT_USE_OLD_BIBFORMAT \ or format.lower().startswith('t') \ or format.lower().startswith('hm') \ or str(format[0:3]).isdigit() \ or ot): # Unspecified format is hd if format == '': format = 'hd' if record_exist_p == -1 and get_output_format_content_type(format) == 'text/html': # HTML output displays a default value for deleted records. # Other format have to deal with it. out += _("The record has been deleted.") else: out += call_bibformat(recID, format, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) # at the end of HTML brief mode, print the "Detailed record" functionality: if format.lower().startswith('hb') and \ format.lower() != 'hb_p': out += websearch_templates.tmpl_print_record_brief_links( ln = ln, recID = recID, weburl = weburl ) return out # Old PHP BibFormat procedure for formatting # print record opening tags, if needed: if format == "marcxml" or format == "oai_dc": out += " \n" out += "

    \n" for oai_id in get_fieldvalues(recID, CFG_OAI_ID_FIELD): out += " %s\n" % oai_id out += " %s\n" % get_modification_date(recID) out += "
    \n" out += " \n" if format.startswith("xm") or format == "marcxml": # look for detailed format existence: query = "SELECT value FROM bibfmt WHERE id_bibrec='%s' AND format='%s'" % (recID, format) res = run_sql(query, None, 1) if res and record_exist_p == 1: # record 'recID' is formatted in 'format', so print it out += "%s" % decompress(res[0][0]) else: # record 'recID' is not formatted in 'format' -- they are not in "bibfmt" table; so fetch all the data from "bibXXx" tables: if format == "marcxml": out += """ \n""" out += " %d\n" % int(recID) elif format.startswith("xm"): out += """ \n""" out += " %d\n" % int(recID) if record_exist_p == -1: # deleted record, so display only OAI ID and 980: oai_ids = get_fieldvalues(recID, CFG_OAI_ID_FIELD) if oai_ids: out += "%s\n" % \ (CFG_OAI_ID_FIELD[0:3], CFG_OAI_ID_FIELD[3:4], CFG_OAI_ID_FIELD[4:5], CFG_OAI_ID_FIELD[5:6], oai_ids[0]) out += "DELETED\n" else: # controlfields query = "SELECT b.tag,b.value,bb.field_number FROM bib00x AS b, bibrec_bib00x AS bb "\ "WHERE bb.id_bibrec='%s' AND b.id=bb.id_bibxxx AND b.tag LIKE '00%%' "\ "ORDER BY bb.field_number, b.tag ASC" % recID res = run_sql(query) for row in res: field, value = row[0], row[1] value = encode_for_xml(value) out += """ %s\n""" % \ (encode_for_xml(field[0:3]), value) # datafields i = 1 # Do not process bib00x and bibrec_bib00x, as # they are controlfields. So start at bib01x and # bibrec_bib00x (and set i = 0 at the end of # first loop) for digit1 in range(0, 10): for digit2 in range(i, 10): bx = "bib%d%dx" % (digit1, digit2) bibx = "bibrec_bib%d%dx" % (digit1, digit2) query = "SELECT b.tag,b.value,bb.field_number FROM %s AS b, %s AS bb "\ "WHERE bb.id_bibrec='%s' AND b.id=bb.id_bibxxx AND b.tag LIKE '%s%%' "\ "ORDER BY bb.field_number, b.tag ASC" % (bx, bibx, recID, str(digit1)+str(digit2)) res = run_sql(query) field_number_old = -999 field_old = "" for row in res: field, value, field_number = row[0], row[1], row[2] ind1, ind2 = field[3], field[4] if ind1 == "_" or ind1 == "": ind1 = " " if ind2 == "_" or ind2 == "": ind2 = " " # print field tag if field_number != field_number_old or field[:-1] != field_old[:-1]: if field_number_old != -999: out += """ \n""" out += """ \n""" % \ (encode_for_xml(field[0:3]), encode_for_xml(ind1), encode_for_xml(ind2)) field_number_old = field_number field_old = field # print subfield value value = encode_for_xml(value) out += """ %s\n""" % \ (encode_for_xml(field[-1:]), value) # all fields/subfields printed in this run, so close the tag: if field_number_old != -999: out += """ \n""" i = 0 # Next loop should start looking at bib%0 and bibrec_bib00x # we are at the end of printing the record: out += " \n" elif format == "xd" or format == "oai_dc": # XML Dublin Core format, possibly OAI -- select only some bibXXx fields: out += """ \n""" if record_exist_p == -1: out += "" else: for f in get_fieldvalues(recID, "041__a"): out += " %s\n" % f for f in get_fieldvalues(recID, "100__a"): out += " %s\n" % encode_for_xml(f) for f in get_fieldvalues(recID, "700__a"): out += " %s\n" % encode_for_xml(f) for f in get_fieldvalues(recID, "245__a"): out += " %s\n" % encode_for_xml(f) for f in get_fieldvalues(recID, "65017a"): out += " %s\n" % encode_for_xml(f) for f in get_fieldvalues(recID, "8564_u"): out += " %s\n" % encode_for_xml(f) for f in get_fieldvalues(recID, "520__a"): out += " %s\n" % encode_for_xml(f) out += " %s\n" % get_creation_date(recID) out += " \n" elif str(format[0:3]).isdigit(): # user has asked to print some fields only if format == "001": out += "%s\n" % (format, recID, format) else: vals = get_fieldvalues(recID, format) for val in vals: out += "%s\n" % (format, val, format) elif format.startswith('t'): ## user directly asked for some tags to be displayed only if record_exist_p == -1: out += get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"]) else: out += get_fieldvalues_alephseq_like(recID, ot) elif format == "hm": if record_exist_p == -1: out += "
    " + cgi.escape(get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"])) + "
    " else: out += "
    " + cgi.escape(get_fieldvalues_alephseq_like(recID, ot)) + "
    " elif format.startswith("h") and ot: ## user directly asked for some tags to be displayed only if record_exist_p == -1: out += "
    " + get_fieldvalues_alephseq_like(recID, ["001", CFG_OAI_ID_FIELD, "980"]) + "
    " else: out += "
    " + get_fieldvalues_alephseq_like(recID, ot) + "
    " elif format == "hd": # HTML detailed format if record_exist_p == -1: out += _("The record has been deleted.") else: # look for detailed format existence: query = "SELECT value FROM bibfmt WHERE id_bibrec='%s' AND format='%s'" % (recID, format) res = run_sql(query, None, 1) if res: # record 'recID' is formatted in 'format', so print it out += "%s" % decompress(res[0][0]) else: # record 'recID' is not formatted in 'format', so try to call BibFormat on the fly or use default format: out_record_in_format = call_bibformat(recID, format, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) if out_record_in_format: out += out_record_in_format else: out += websearch_templates.tmpl_print_record_detailed( ln = ln, recID = recID, weburl = weburl, ) elif format.startswith("hb_") or format.startswith("hd_"): # underscore means that HTML brief/detailed formats should be called on-the-fly; suitable for testing formats if record_exist_p == -1: out += _("The record has been deleted.") else: out += call_bibformat(recID, format, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) elif format.startswith("hx"): # BibTeX format, called on the fly: if record_exist_p == -1: out += _("The record has been deleted.") else: out += call_bibformat(recID, format, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) elif format.startswith("hs"): # for citation/download similarity navigation links: if record_exist_p == -1: out += _("The record has been deleted.") else: out += '' % websearch_templates.build_search_url(recid=recID, ln=ln) # firstly, title: titles = get_fieldvalues(recID, "245__a") if titles: for title in titles: out += "%s" % title else: # usual title not found, try conference title: titles = get_fieldvalues(recID, "111__a") if titles: for title in titles: out += "%s" % title else: # just print record ID: out += "%s %d" % (get_field_i18nname("record ID", ln), recID) out += "" # secondly, authors: authors = get_fieldvalues(recID, "100__a") + get_fieldvalues(recID, "700__a") if authors: out += " - %s" % authors[0] if len(authors) > 1: out += " et al" # thirdly publication info: publinfos = get_fieldvalues(recID, "773__s") if not publinfos: publinfos = get_fieldvalues(recID, "909C4s") if not publinfos: publinfos = get_fieldvalues(recID, "037__a") if not publinfos: publinfos = get_fieldvalues(recID, "088__a") if publinfos: out += " - %s" % publinfos[0] else: # fourthly publication year (if not publication info): years = get_fieldvalues(recID, "773__y") if not years: years = get_fieldvalues(recID, "909C4y") if not years: years = get_fieldvalues(recID, "260__c") if years: out += " (%s)" % years[0] else: # HTML brief format by default if record_exist_p == -1: out += _("The record has been deleted.") else: query = "SELECT value FROM bibfmt WHERE id_bibrec='%s' AND format='%s'" % (recID, format) res = run_sql(query) if res: # record 'recID' is formatted in 'format', so print it out += "%s" % decompress(res[0][0]) else: # record 'recID' is not formatted in 'format', so try to call BibFormat on the fly: or use default format: if CFG_WEBSEARCH_CALL_BIBFORMAT: out_record_in_format = call_bibformat(recID, format, ln, search_pattern=search_pattern, - uid=uid, verbose=verbose) + user_info=user_info, verbose=verbose) if out_record_in_format: out += out_record_in_format else: out += websearch_templates.tmpl_print_record_brief( ln = ln, recID = recID, weburl = weburl, ) else: out += websearch_templates.tmpl_print_record_brief( ln = ln, recID = recID, weburl = weburl, ) # at the end of HTML brief mode, print the "Detailed record" functionality: if format == 'hp' or format.startswith("hb_") or format.startswith("hd_"): pass # do nothing for portfolio and on-the-fly formats else: out += websearch_templates.tmpl_print_record_brief_links( ln = ln, recID = recID, weburl = weburl, ) # print record closing tags, if needed: if format == "marcxml" or format == "oai_dc": out += "     \n" out += " \n" return out def encode_for_xml(s): "Encode special chars in string so that it would be XML-compliant." s = string.replace(s, '&', '&') s = string.replace(s, '<', '<') return s -def call_bibformat(recID, format="HD", ln=cdslang, search_pattern=None, uid=None, verbose=0): +def call_bibformat(recID, format="HD", ln=cdslang, search_pattern=None, user_info=None, verbose=0): """ Calls BibFormat and returns formatted record. BibFormat will decide by itself if old or new BibFormat must be used. """ keywords = [] if search_pattern is not None: units = create_basic_search_units(None, str(search_pattern), None) keywords = [unit[1] for unit in units if unit[0] != '-'] return format_record(recID, of=format, ln=ln, search_pattern=keywords, - uid=uid, + user_info=user_info, verbose=verbose) def log_query(hostname, query_args, uid=-1): """ Log query into the query and user_query tables. Return id_query or None in case of problems. """ id_query = None if uid > 0: # log the query only if uid is reasonable res = run_sql("SELECT id FROM query WHERE urlargs=%s", (query_args,), 1) try: id_query = res[0][0] except: id_query = run_sql("INSERT INTO query (type, urlargs) VALUES ('r', %s)", (query_args,)) if id_query: run_sql("INSERT INTO user_query (id_user, id_query, hostname, date) VALUES (%s, %s, %s, %s)", (uid, id_query, hostname, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) return id_query def log_query_info(action, p, f, colls, nb_records_found_total=-1): """Write some info to the log file for later analysis.""" try: log = open(logdir + "/search.log", "a") log.write(time.strftime("%Y%m%d%H%M%S#", time.localtime())) log.write(action+"#") log.write(p+"#") log.write(f+"#") for coll in colls[:-1]: log.write("%s," % coll) log.write("%s#" % colls[-1]) log.write("%d" % nb_records_found_total) log.write("\n") log.close() except: pass return def wash_url_argument(var, new_type): """Wash list argument into 'new_type', that can be 'list', 'str', or 'int'. Useful for washing mod_python passed arguments, that are all lists of strings (URL args may be multiple), but we sometimes want only to take the first value, and sometimes to represent it as string or numerical value.""" out = [] if new_type == 'list': # return lst if type(var) is list: out = var else: out = [var] elif new_type == 'str': # return str if type(var) is list: try: out = "%s" % var[0] except: out = "" elif type(var) is str: out = var else: out = "%s" % var elif new_type == 'int': # return int if type(var) is list: try: out = string.atoi(var[0]) except: out = 0 elif type(var) is int: out = var elif type(var) is str: try: out = string.atoi(var) except: out = 0 else: out = 0 return out ### CALLABLES def perform_request_search(req=None, cc=cdsname, c=None, p="", f="", rg=10, sf="", so="d", sp="", rm="", of="id", ot="", as=0, p1="", f1="", m1="", op1="", p2="", f2="", m2="", op2="", p3="", f3="", m3="", sc=0, jrec=0, recid=-1, recidb=-1, sysno="", id=-1, idb=-1, sysnb="", action="", d1="", d1y=0, d1m=0, d1d=0, d2="", d2y=0, d2m=0, d2d=0, dt="", verbose=0, ap=0, ln=cdslang, ec=None, tab=""): """Perform search or browse request, without checking for authentication. Return list of recIDs found, if of=id. Otherwise create web page. The arguments are as follows: req - mod_python Request class instance. cc - current collection (e.g. "ATLAS"). The collection the user started to search/browse from. c - collection list (e.g. ["Theses", "Books"]). The collections user may have selected/deselected when starting to search from 'cc'. p - pattern to search for (e.g. "ellis and muon or kaon"). f - field to search within (e.g. "author"). rg - records in groups of (e.g. "10"). Defines how many hits per collection in the search results page are displayed. sf - sort field (e.g. "title"). so - sort order ("a"=ascending, "d"=descending). sp - sort pattern (e.g. "CERN-") -- in case there are more values in a sort field, this argument tells which one to prefer rm - ranking method (e.g. "jif"). Defines whether results should be ranked by some known ranking method. of - output format (e.g. "hb"). Usually starting "h" means HTML output (and "hb" for HTML brief, "hd" for HTML detailed), "x" means XML output, "t" means plain text output, "id" means no output at all but to return list of recIDs found. (Suitable for high-level API.) ot - output only these MARC tags (e.g. "100,700,909C0b"). Useful if only some fields are to be shown in the output, e.g. for library to control some fields. as - advanced search ("0" means no, "1" means yes). Whether search was called from within the advanced search interface. p1 - first pattern to search for in the advanced search interface. Much like 'p'. f1 - first field to search within in the advanced search interface. Much like 'f'. m1 - first matching type in the advanced search interface. ("a" all of the words, "o" any of the words, "e" exact phrase, "p" partial phrase, "r" regular expression). op1 - first operator, to join the first and the second unit in the advanced search interface. ("a" add, "o" or, "n" not). p2 - second pattern to search for in the advanced search interface. Much like 'p'. f2 - second field to search within in the advanced search interface. Much like 'f'. m2 - second matching type in the advanced search interface. ("a" all of the words, "o" any of the words, "e" exact phrase, "p" partial phrase, "r" regular expression). op2 - second operator, to join the second and the third unit in the advanced search interface. ("a" add, "o" or, "n" not). p3 - third pattern to search for in the advanced search interface. Much like 'p'. f3 - third field to search within in the advanced search interface. Much like 'f'. m3 - third matching type in the advanced search interface. ("a" all of the words, "o" any of the words, "e" exact phrase, "p" partial phrase, "r" regular expression). sc - split by collection ("0" no, "1" yes). Governs whether we want to present the results in a single huge list, or splitted by collection. jrec - jump to record (e.g. "234"). Used for navigation inside the search results. recid - display record ID (e.g. "20000"). Do not search/browse but go straight away to the Detailed record page for the given recID. recidb - display record ID bis (e.g. "20010"). If greater than 'recid', then display records from recid to recidb. Useful for example for dumping records from the database for reformatting. sysno - display old system SYS number (e.g. ""). If you migrate to CDS Invenio from another system, and store your old SYS call numbers, you can use them instead of recid if you wish so. id - the same as recid, in case recid is not set. For backwards compatibility. idb - the same as recid, in case recidb is not set. For backwards compatibility. sysnb - the same as sysno, in case sysno is not set. For backwards compatibility. action - action to do. "SEARCH" for searching, "Browse" for browsing. Default is to search. d1 - first datetime in full YYYY-mm-dd HH:MM:DD format (e.g. "1998-08-23 12:34:56"). Useful for search limits on creation/modification date (see 'dt' argument below). Note that 'd1' takes precedence over d1y, d1m, d1d if these are defined. d1y - first date's year (e.g. "1998"). Useful for search limits on creation/modification date. d1m - first date's month (e.g. "08"). Useful for search limits on creation/modification date. d1d - first date's day (e.g. "23"). Useful for search limits on creation/modification date. d2 - second datetime in full YYYY-mm-dd HH:MM:DD format (e.g. "1998-09-02 12:34:56"). Useful for search limits on creation/modification date (see 'dt' argument below). Note that 'd2' takes precedence over d2y, d2m, d2d if these are defined. d2y - second date's year (e.g. "1998"). Useful for search limits on creation/modification date. d2m - second date's month (e.g. "09"). Useful for search limits on creation/modification date. d2d - second date's day (e.g. "02"). Useful for search limits on creation/modification date. dt - first and second date's type (e.g. "c"). Specifies whether to search in creation dates ("c") or in modification dates ("m"). When dt is not set and d1* and d2* are set, the default is "c". verbose - verbose level (0=min, 9=max). Useful to print some internal information on the searching process in case something goes wrong. ap - alternative patterns (0=no, 1=yes). In case no exact match is found, the search engine can try alternative patterns e.g. to replace non-alphanumeric characters by a boolean query. ap defines if this is wanted. ln - language of the search interface (e.g. "en"). Useful for internationalization. ec - list of external search engines to search as well (e.g. "SPIRES HEP"). """ selected_external_collections_infos = None # wash all arguments requiring special care try: (cc, colls_to_display, colls_to_search) = wash_colls(cc, c, sc) # which colls to search and to display? except InvenioWebSearchUnknownCollectionError, exc: colname = exc.colname if of.startswith("h"): page_start(req, of, cc, as, ln, getUid(req), websearch_templates.tmpl_collection_not_found_page_title(colname, ln)) req.write(websearch_templates.tmpl_collection_not_found_page_body(colname, ln)) return page_end(req, of, ln) elif of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) else: return page_end(req, of, ln) p = wash_pattern(p) f = wash_field(f) p1 = wash_pattern(p1) f1 = wash_field(f1) p2 = wash_pattern(p2) f2 = wash_field(f2) p3 = wash_pattern(p3) f3 = wash_field(f3) datetext1, datetext2 = wash_dates(d1, d1y, d1m, d1d, d2, d2y, d2m, d2d) _ = gettext_set_language(ln) # backwards compatibility: id, idb, sysnb -> recid, recidb, sysno (if applicable) if sysnb != "" and sysno == "": sysno = sysnb if id > 0 and recid == -1: recid = id if idb > 0 and recidb == -1: recidb = idb # TODO deduce passed search limiting criterias (if applicable) pl, pl_in_url = "", "" # no limits by default if action != "browse" and req and req.args: # we do not want to add options while browsing or while calling via command-line fieldargs = cgi.parse_qs(req.args) for fieldcode in get_fieldcodes(): if fieldargs.has_key(fieldcode): for val in fieldargs[fieldcode]: pl += "+%s:\"%s\" " % (fieldcode, val) pl_in_url += "&%s=%s" % (urllib.quote(fieldcode), urllib.quote(val)) # deduce recid from sysno argument (if applicable): if sysno: # ALEPH SYS number was passed, so deduce DB recID for the record: recid = get_mysql_recid_from_aleph_sysno(sysno) # deduce collection we are in (if applicable): if recid > 0: cc = guess_primary_collection_of_a_record(recid) # deduce user id (if applicable): try: uid = getUid(req) except: uid = 0 ## 0 - start output if recid > 0: ## 1 - detailed record display title, description, keywords = \ websearch_templates.tmpl_record_page_header_content(req, recid, ln) page_start(req, of, cc, as, ln, uid, title, description, keywords, recid, tab) # Default format is hb but we are in detailed -> change 'of' if of == "hb": of = "hd" if record_exists(recid): if recidb <= recid: # sanity check recidb = recid + 1 if of == "id": return [recidx for recidx in range(recid, recidb) if record_exists(recidx)] else: print_records(req, range(recid, recidb), -1, -9999, of, ot, ln, search_pattern=p, verbose=verbose, tab=tab) if req and of.startswith("h"): # register detailed record page view event client_ip_address = str(req.get_remote_host(apache.REMOTE_NOLOOKUP)) register_page_view_event(recid, uid, client_ip_address) else: # record does not exist if of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) elif of.startswith("h"): print_warning(req, "Requested record does not seem to exist.") elif action == "browse": ## 2 - browse needed page_start(req, of, cc, as, ln, uid, _("Browse")) if of.startswith("h"): req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, as, ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action)) try: if as == 1 or (p1 or p2 or p3): browse_pattern(req, colls_to_search, p1, f1, rg, ln) browse_pattern(req, colls_to_search, p2, f2, rg, ln) browse_pattern(req, colls_to_search, p3, f3, rg, ln) else: browse_pattern(req, colls_to_search, p, f, rg, ln) except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) elif rm and p.startswith("recid:"): ## 3-ter - similarity search needed page_start(req, of, cc, as, ln, uid, _("Search Results")) if of.startswith("h"): req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, as, ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action)) if record_exists(p[6:]) != 1: # record does not exist if of.startswith("h"): print_warning(req, "Requested record does not seem to exist.") if of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) else: # record well exists, so find similar ones to it t1 = os.times()[4] results_similar_recIDs, results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, results_similar_comments = \ rank_records(rm, 0, get_collection_reclist(cdsname), string.split(p), verbose) if results_similar_recIDs: t2 = os.times()[4] cpu_time = t2 - t1 if of.startswith("h"): req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, cdsname, len(results_similar_recIDs), jrec, rg, as, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2, sc, pl_in_url, d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time)) print_warning(req, results_similar_comments) print_records(req, results_similar_recIDs, jrec, rg, of, ot, ln, results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, search_pattern=p, verbose=verbose) elif of=="id": return results_similar_recIDs elif of.startswith("x"): print_records(req, results_similar_recIDs, jrec, rg, of, ot, ln, results_similar_relevances, results_similar_relevances_prologue, results_similar_relevances_epilogue, search_pattern=p, verbose=verbose) else: # rank_records failed and returned some error message to display: if of.startswith("h"): print_warning(req, results_similar_relevances_prologue) print_warning(req, results_similar_relevances_epilogue) print_warning(req, results_similar_comments) if of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) elif p.startswith("cocitedwith:"): #WAS EXPERIMENTAL ## 3-terter - cited by search needed page_start(req, of, cc, as, ln, uid, _("Search Results")) if of.startswith("h"): req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, as, ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action)) recID = p[12:] if record_exists(recID) != 1: # record does not exist if of.startswith("h"): print_warning(req, "Requested record does not seem to exist.") if of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) else: # record well exists, so find co-cited ones: t1 = os.times()[4] results_cocited_recIDs = map(lambda x: x[0], calculate_co_cited_with_list(int(recID))) if results_cocited_recIDs: t2 = os.times()[4] cpu_time = t2 - t1 if of.startswith("h"): req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, cdsname, len(results_cocited_recIDs), jrec, rg, as, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2, sc, pl_in_url, d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time)) print_records(req, results_cocited_recIDs, jrec, rg, of, ot, ln, search_pattern=p, verbose=verbose) elif of=="id": return results_cocited_recIDs elif of.startswith("x"): print_records(req, results_cocited_recIDs, jrec, rg, of, ot, ln, search_pattern=p, verbose=verbose) else: # cited rank_records failed and returned some error message to display: if of.startswith("h"): print_warning(req, "nothing found") if of == "id": return [] elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) else: ## 3 - common search needed page_start(req, of, cc, as, ln, uid, _("Search Results")) if of.startswith("h"): req.write(create_search_box(cc, colls_to_display, p, f, rg, sf, so, sp, rm, of, ot, as, ln, p1, f1, m1, op1, p2, f2, m2, op2, p3, f3, m3, sc, pl, d1y, d1m, d1d, d2y, d2m, d2d, dt, jrec, ec, action)) t1 = os.times()[4] results_in_any_collection = HitSet() if as == 1 or (p1 or p2 or p3): ## 3A - advanced search try: results_in_any_collection = search_pattern(req, p1, f1, m1, ap=ap, of=of, verbose=verbose, ln=ln) if len(results_in_any_collection) == 0: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) if p2: results_tmp = search_pattern(req, p2, f2, m2, ap=ap, of=of, verbose=verbose, ln=ln) if op1 == "a": # add results_in_any_collection.intersection_update(results_tmp) elif op1 == "o": # or results_in_any_collection.union_update(results_tmp) elif op1 == "n": # not results_in_any_collection.difference_update(results_tmp) else: if of.startswith("h"): print_warning(req, "Invalid set operation %s." % op1, "Error") if len(results_in_any_collection) == 0: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) if p3: results_tmp = search_pattern(req, p3, f3, m3, ap=ap, of=of, verbose=verbose, ln=ln) if op2 == "a": # add results_in_any_collection.intersection_update(results_tmp) elif op2 == "o": # or results_in_any_collection.union_update(results_tmp) elif op2 == "n": # not results_in_any_collection.difference_update(results_tmp) else: if of.startswith("h"): print_warning(req, "Invalid set operation %s." % op2, "Error") except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) else: ## 3B - simple search try: results_in_any_collection = search_pattern(req, p, f, ap=ap, of=of, verbose=verbose, ln=ln) except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) if len(results_in_any_collection) == 0: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) # search_cache_key = p+"@"+f+"@"+string.join(colls_to_search,",") # if search_cache.has_key(search_cache_key): # is the result in search cache? # results_final = search_cache[search_cache_key] # else: # results_final = search_pattern(req, p, f, colls_to_search) # search_cache[search_cache_key] = results_final # if len(search_cache) > CFG_WEBSEARCH_SEARCH_CACHE_SIZE: # is the cache full? (sanity cleaning) # search_cache.clear() # search stage 4: intersection with collection universe: try: results_final = intersect_results_with_collrecs(req, results_in_any_collection, colls_to_search, ap, of, verbose, ln) except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) if results_final == {}: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) if of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) # search stage 5: apply search option limits and restrictions: if datetext1 != "": if verbose and of.startswith("h"): print_warning(req, "Search stage 5: applying time limits, from %s until %s..." % (datetext1, datetext2)) try: results_final = intersect_results_with_hitset(req, results_final, search_unit_in_bibrec(datetext1, datetext2, dt), ap, aptext= _("No match within your time limits, " "discarding this condition..."), of=of) except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) if results_final == {}: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) if pl: pl = wash_pattern(pl) if verbose and of.startswith("h"): print_warning(req, "Search stage 5: applying search pattern limit %s..." % (pl,)) try: results_final = intersect_results_with_hitset(req, results_final, search_pattern(req, pl, ap=0, ln=ln), ap, aptext=_("No match within your search limits, " "discarding this condition..."), of=of) except: if of.startswith("h"): req.write(create_error_box(req, verbose=verbose, ln=ln)) perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) if results_final == {}: if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) if of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) return page_end(req, of, ln) t2 = os.times()[4] cpu_time = t2 - t1 ## search stage 6: display results: results_final_nb_total = 0 results_final_nb = {} # will hold number of records found in each collection # (in simple dict to display overview more easily) for coll in results_final.keys(): results_final_nb[coll] = len(results_final[coll]) #results_final_nb_total += results_final_nb[coll] # Now let us calculate results_final_nb_total more precisely, # in order to get the total number of "distinct" hits across # searched collections; this is useful because a record might # have been attributed to more than one primary collection; so # we have to avoid counting it multiple times. The price to # pay for this accuracy of results_final_nb_total is somewhat # increased CPU time. if results_final.keys() == 1: # only one collection; no need to union them results_final_for_all_selected_colls = results_final.values()[0] results_final_nb_total = results_final_nb.values()[0] else: # okay, some work ahead to union hits across collections: results_final_for_all_selected_colls = HitSet() for coll in results_final.keys(): results_final_for_all_selected_colls.union_update(results_final[coll]) results_final_nb_total = len(results_final_for_all_selected_colls) if results_final_nb_total == 0: if of.startswith('h'): print_warning(req, "No match found, please enter different search terms.") elif of.startswith("x"): # Print empty, but valid XML print_records_prologue(req, of) print_records_epilogue(req, of) else: # yes, some hits found: good! # collection list may have changed due to not-exact-match-found policy so check it out: for coll in results_final.keys(): if coll not in colls_to_search: colls_to_search.append(coll) # print results overview: if of == "id": # we have been asked to return list of recIDs recIDs = list(results_final_for_all_selected_colls) if sf: # do we have to sort? recIDs = sort_records(req, recIDs, sf, so, sp, verbose, of) elif rm: # do we have to rank? results_final_for_all_colls_rank_records_output = rank_records(rm, 0, results_final_for_all_selected_colls, string.split(p) + string.split(p1) + string.split(p2) + string.split(p3), verbose) if results_final_for_all_colls_rank_records_output[0]: recIDs = results_final_for_all_colls_rank_records_output[0] return recIDs elif of.startswith("h"): req.write(print_results_overview(req, colls_to_search, results_final_nb_total, results_final_nb, cpu_time, ln, ec)) selected_external_collections_infos = print_external_results_overview(req, cc, [p, p1, p2, p3], f, ec, verbose, ln) # print number of hits found for XML outputs: if of.startswith("x"): req.write("\n" % results_final_nb_total) # print records: if len(colls_to_search)>1: cpu_time = -1 # we do not want to have search time printed on each collection print_records_prologue(req, of) for coll in colls_to_search: if results_final.has_key(coll) and len(results_final[coll]): if of.startswith("h"): req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, coll, results_final_nb[coll], jrec, rg, as, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2, sc, pl_in_url, d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time)) results_final_recIDs = list(results_final[coll]) results_final_relevances = [] results_final_relevances_prologue = "" results_final_relevances_epilogue = "" if sf: # do we have to sort? results_final_recIDs = sort_records(req, results_final_recIDs, sf, so, sp, verbose, of) elif rm: # do we have to rank? results_final_recIDs_ranked, results_final_relevances, results_final_relevances_prologue, results_final_relevances_epilogue, results_final_comments = \ rank_records(rm, 0, results_final[coll], string.split(p) + string.split(p1) + string.split(p2) + string.split(p3), verbose) if of.startswith("h"): print_warning(req, results_final_comments) if results_final_recIDs_ranked: results_final_recIDs = results_final_recIDs_ranked else: # rank_records failed and returned some error message to display: print_warning(req, results_final_relevances_prologue) print_warning(req, results_final_relevances_epilogue) print_records(req, results_final_recIDs, jrec, rg, of, ot, ln, results_final_relevances, results_final_relevances_prologue, results_final_relevances_epilogue, search_pattern=p, print_records_prologue_p=False, print_records_epilogue_p=False, verbose=verbose) if of.startswith("h"): req.write(print_search_info(p, f, sf, so, sp, rm, of, ot, coll, results_final_nb[coll], jrec, rg, as, ln, p1, p2, p3, f1, f2, f3, m1, m2, m3, op1, op2, sc, pl_in_url, d1y, d1m, d1d, d2y, d2m, d2d, dt, cpu_time, 1)) print_records_epilogue(req, of) if f == "author" and of.startswith("h"): req.write(create_similarly_named_authors_link_box(p, ln)) # log query: try: id_query = log_query(req.get_remote_host(), req.args, uid) if of.startswith("h") and id_query: # Alert/RSS teaser: req.write(websearch_templates.tmpl_alert_rss_teaser_box_for_query(id_query, ln=ln)) except: # do not log query if req is None (used by CLI interface) pass log_query_info("ss", p, f, colls_to_search, results_final_nb_total) # External searches if of.startswith("h"): perform_external_collection_search(req, cc, [p, p1, p2, p3], f, ec, verbose, ln, selected_external_collections_infos) return page_end(req, of, ln) def perform_request_cache(req, action="show"): """Manipulates the search engine cache.""" global search_cache global collection_reclist_cache global collection_reclist_cache_timestamp global field_i18nname_cache global field_i18nname_cache_timestamp global collection_i18nname_cache global collection_i18nname_cache_timestamp req.content_type = "text/html" req.send_http_header() out = "" out += "

    Search Cache

    " # clear cache if requested: if action == "clear": search_cache = {} collection_reclist_cache = create_collection_reclist_cache() # show collection reclist cache: out += "

    Collection reclist cache

    " out += "- collection table last updated: %s" % get_table_update_time('collection') out += "
    - reclist cache timestamp: %s" % collection_reclist_cache_timestamp out += "
    - reclist cache contents:" out += "
    " for coll in collection_reclist_cache.keys(): if collection_reclist_cache[coll]: out += "%s (%d)
    " % (coll, len(get_collection_reclist(coll))) out += "
    " # show search cache: out += "

    Search Cache

    " out += "
    " if len(search_cache): out += """""" out += "" % \ ("Pattern", "Field", "Collection", "Number of Hits") for search_cache_key in search_cache.keys(): p, f, c = string.split(search_cache_key, "@", 2) # find out about length of cached data: l = 0 for coll in search_cache[search_cache_key]: l += len(search_cache[search_cache_key][coll]) out += "" % (p, f, c, l) out += "
    %s%s%s%s
    %s%s%s%d
    " else: out += "

    Search cache is empty." out += "

    " out += """

    clear cache""" % weburl # show field i18nname cache: out += "

    Field I18N names cache

    " out += "- fieldname table last updated: %s" % get_table_update_time('fieldname') out += "
    - i18nname cache timestamp: %s" % field_i18nname_cache_timestamp out += "
    - i18nname cache contents:" out += "
    " for field in field_i18nname_cache.keys(): for ln in field_i18nname_cache[field].keys(): out += "%s, %s = %s
    " % (field, ln, field_i18nname_cache[field][ln]) out += "
    " # show collection i18nname cache: out += "

    Collection I18N names cache

    " out += "- collectionname table last updated: %s" % get_table_update_time('collectionname') out += "
    - i18nname cache timestamp: %s" % collection_i18nname_cache_timestamp out += "
    - i18nname cache contents:" out += "
    " for coll in collection_i18nname_cache.keys(): for ln in collection_i18nname_cache[coll].keys(): out += "%s, %s = %s
    " % (coll, ln, collection_i18nname_cache[coll][ln]) out += "
    " req.write("") req.write(out) req.write("") return "\n" def perform_request_log(req, date=""): """Display search log information for given date.""" req.content_type = "text/html" req.send_http_header() req.write("") req.write("

    Search Log

    ") if date: # case A: display stats for a day yyyymmdd = string.atoi(date) req.write("

    Date: %d

    " % yyyymmdd) req.write("""""") req.write("" % ("No.", "Time", "Pattern", "Field", "Collection", "Number of Hits")) # read file: p = os.popen("grep ^%d %s/search.log" % (yyyymmdd, logdir), 'r') lines = p.readlines() p.close() # process lines: i = 0 for line in lines: try: datetime, as, p, f, c, nbhits = string.split(line,"#") i += 1 req.write("" \ % (i, datetime[8:10], datetime[10:12], datetime[12:], p, f, c, nbhits)) except: pass # ignore eventual wrong log lines req.write("
    %s%s%s%s%s%s
    #%d%s:%s:%s%s%s%s%s
    ") else: # case B: display summary stats per day yyyymm01 = int(time.strftime("%Y%m01", time.localtime())) yyyymmdd = int(time.strftime("%Y%m%d", time.localtime())) req.write("""""") req.write("" % ("Day", "Number of Queries")) for day in range(yyyymm01, yyyymmdd + 1): p = os.popen("grep -c ^%d %s/search.log" % (day, logdir), 'r') for line in p.readlines(): req.write("""""" % \ (day, weburl, day, line)) p.close() req.write("
    %s%s
    %s%s
    ") req.write("") return "\n" def profile(p="", f="", c=cdsname): """Profile search time.""" import profile import pstats profile.run("perform_request_search(p='%s',f='%s', c='%s')" % (p, f, c), "perform_request_search_profile") p = pstats.Stats("perform_request_search_profile") p.strip_dirs().sort_stats("cumulative").print_stats() return 0 ## test cases: #print wash_colls(cdsname,"Library Catalogue", 0) #print wash_colls("Periodicals & Progress Reports",["Periodicals","Progress Reports"], 0) #print wash_field("wau") #print print_record(20,"tm","001,245") #print create_opft_search_units(None, "PHE-87-13","reportnumber") #print ":"+wash_pattern("* and % doo * %")+":\n" #print ":"+wash_pattern("*")+":\n" #print ":"+wash_pattern("ellis* ell* e*%")+":\n" #print run_sql("SELECT name,dbquery from collection") #print get_index_id("author") #print get_coll_ancestors("Theses") #print get_coll_sons("Articles & Preprints") #print get_coll_real_descendants("Articles & Preprints") #print get_collection_reclist("Theses") #print log(sys.stdin) #print search_unit_in_bibrec('2002-12-01','2002-12-12') #print type(wash_url_argument("-1",'int')) #print get_nearest_terms_in_bibxxx("ellis", "author", 5, 5) #print call_bibformat(68, "HB_FLY") #print create_collection_i18nname_cache() #print get_fieldvalues(10, "980__a") #print get_fieldvalues_alephseq_like(10,"001___") #print get_fieldvalues_alephseq_like(10,"980__a") #print get_fieldvalues_alephseq_like(10,"foo") #print get_fieldvalues_alephseq_like(10,"-1") #print get_fieldvalues_alephseq_like(10,"99") #print get_fieldvalues_alephseq_like(10,["001", "980"]) ## profiling: #profile("of the this") #print perform_request_search(p="ellis")