diff --git a/modules/websearch/lib/search_engine_query_parser.py b/modules/websearch/lib/search_engine_query_parser.py
index 731b7b2cc..6e6142f15 100644
--- a/modules/websearch/lib/search_engine_query_parser.py
+++ b/modules/websearch/lib/search_engine_query_parser.py
@@ -1,1322 +1,1326 @@
# -*- coding: utf-8 -*-
## This file is part of Invenio.
## Copyright (C) 2008, 2010, 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable=C0301
"""Invenio Search Engine query parsers."""
import re
import string
from datetime import datetime
try:
import dateutil
if not hasattr(dateutil, '__version__') or dateutil.__version__ != '2.0':
from dateutil import parser as du_parser
from dateutil.relativedelta import relativedelta as du_delta
from dateutil import relativedelta
GOT_DATEUTIL = True
else:
from warnings import warn
warn("Not using dateutil module because the version %s is not compatible with Python-2.x" % dateutil.__version__)
GOT_DATEUTIL = False
except ImportError:
# Ok, no date parsing is possible, but continue anyway,
# since this package is only recommended, not mandatory.
GOT_DATEUTIL = False
from invenio.bibindex_engine_tokenizer import BibIndexFuzzyNameTokenizer as FNT
from invenio.logicutils import to_cnf
from invenio.config import CFG_WEBSEARCH_SPIRES_SYNTAX
NameScanner = FNT()
class InvenioWebSearchMismatchedParensError(Exception):
"""Exception for parse errors caused by mismatched parentheses."""
def __init__(self, message):
"""Initialization."""
self.message = message
def __str__(self):
"""String representation."""
return repr(self.message)
class SearchQueryParenthesisedParser(object):
"""Search query parser that handles arbitrarily-nested parentheses
Parameters:
* substitution_dict: a dictionary mapping strings to other strings. By
default, maps 'and', 'or' and 'not' to '+', '|', and '-'. Dictionary
values will be treated as valid operators for output.
A note (valkyrie 25.03.2011):
Based on looking through the prod search logs, it is evident that users,
when they are using parentheses to do searches, only run word characters
up against parens when they intend the parens to be part of the word (e.g.
U(1)), and when they are using parentheses to combine operators, they put
a space before and after them. As of writing, this is the behavior that
SQPP now expects, in order that it be able to handle such queries as
e(+)e(-) that contain operators in parentheses that should be interpreted
as words.
"""
def __init__(self, substitution_dict = {'and': '+', 'or': '|', 'not': '-'}):
self.substitution_dict = substitution_dict
self.specials = set(['(', ')', '+', '|', '-', '+ -'])
self.__tl_idx = 0
self.__tl_len = 0
# I think my names are both concise and clear
# pylint: disable=C0103
def _invenio_to_python_logical(self, q):
"""Translate the + and - in invenio query strings into & and ~."""
p = q
p = re.sub('\+ -', '&~', p)
p = re.sub('\+', '&', p)
p = re.sub('-', '~', p)
p = re.sub(' ~', ' & ~', p)
return p
def _python_logical_to_invenio(self, q):
"""Translate the & and ~ in logical expression strings into + and -."""
p = q
p = re.sub('\& ~', '-', p)
p = re.sub('~', '-', p)
p = re.sub('\&', '+', p)
return p
# pylint: enable=C0103
def parse_query(self, query):
"""Make query into something suitable for search_engine.
This is the main entry point of the class.
Given an expression of the form:
"expr1 or expr2 (expr3 not (expr4 or expr5))"
produces annoted list output suitable for consumption by search_engine,
of the form:
['+', 'expr1', '|', 'expr2', '+', 'expr3 - expr4 | expr5']
parse_query() is a wrapper for self.tokenize() and self.parse().
"""
toklist = self.tokenize(query)
depth, balanced, dummy_d0_p = self.nesting_depth_and_balance(toklist)
if not balanced:
raise SyntaxError("Mismatched parentheses in "+str(toklist))
toklist, var_subs = self.substitute_variables(toklist)
if depth > 1:
toklist = self.tokenize(self.logically_reduce(toklist))
return self.parse(toklist, var_subs)
def substitute_variables(self, toklist):
"""Given a token list, return a copy of token list in which all free
variables are bound with boolean variable names of the form 'pN'.
Additionally, all the substitutable logical operators are exchanged
for their symbolic form and implicit ands are made explicit
e.g., ((author:'ellis, j' and title:quark) or author:stevens jones)
becomes:
((p0 + p1) | p2 + p3)
with the substitution table:
{'p0': "author:'ellis, j'", 'p1': "title:quark",
'p2': "author:stevens", 'p3': "jones" }
Return value is the substituted token list and a copy of the
substitution table.
"""
def labels():
i = 0
while True:
yield 'p'+str(i)
i += 1
def filter_front_ands(toklist):
"""Filter out extra logical connectives and whitespace from the front."""
while toklist[0] == '+' or toklist[0] == '|' or toklist[0] == '':
toklist = toklist[1:]
return toklist
var_subs = {}
labeler = labels()
new_toklist = ['']
cannot_be_anded = self.specials.difference((')',))
for token in toklist:
token = token.lower()
if token in self.substitution_dict:
if token == 'not' and new_toklist[-1] == '+':
new_toklist[-1] = '-'
else:
new_toklist.append(self.substitution_dict[token])
elif token == '(':
if new_toklist[-1] not in self.specials:
new_toklist.append('+')
new_toklist.append(token)
elif token not in self.specials:
# apparently generators are hard for pylint to figure out
# Turns off msg about labeler not having a 'next' method
# pylint: disable=E1101
label = labeler.next()
# pylint: enable=E1101
var_subs[label] = token
if new_toklist[-1] not in cannot_be_anded:
new_toklist.append('+')
new_toklist.append(label)
else:
if token == '-' and new_toklist[-1] == '+':
new_toklist[-1] = '-'
else:
new_toklist.append(token)
return filter_front_ands(new_toklist), var_subs
def nesting_depth_and_balance(self, token_list):
"""Checks that parentheses are balanced and counts how deep they nest"""
depth = 0
maxdepth = 0
depth0_pairs = 0
good_depth = True
for i in range(len(token_list)):
token = token_list[i]
if token == '(':
if depth == 0:
depth0_pairs += 1
depth += 1
if depth > maxdepth:
maxdepth += 1
elif token == ')':
depth -= 1
if depth == -1: # can only happen with unmatched )
good_depth = False # so force depth check to fail
depth = 0 # but keep maxdepth in good range
return maxdepth, depth == 0 and good_depth, depth0_pairs
def logically_reduce(self, token_list):
"""Return token_list in conjunctive normal form as a string.
CNF has the property that there will only ever be one level of
parenthetical nesting, and all distributable operators (such as
the not in -(p | q) will be fully distributed (as -p + -q).
"""
maxdepth, dummy_balanced, d0_p = self.nesting_depth_and_balance(token_list)
s = ' '.join(token_list)
s = self._invenio_to_python_logical(s)
last_maxdepth = 0
while maxdepth != last_maxdepth: # XXX: sometimes NaryExpr doesn't
try: # fully flatten Expr; but it usually
s = str(to_cnf(s)) # does in 2 passes FIXME: diagnose
except SyntaxError:
raise SyntaxError(str(s)+" couldn't be converted to a logic expression.")
last_maxdepth = maxdepth
maxdepth, dummy_balanced, d0_p = self.nesting_depth_and_balance(self.tokenize(s))
if d0_p == 1 and s[0] == '(' and s[-1] == ')': # s can come back with extra parens
s = s[1:-1]
s = self._python_logical_to_invenio(s)
return s
def tokenize(self, query):
"""Given a query string, return a list of tokens from that string.
* Isolates meaningful punctuation: ( ) + | -
* Keeps single- and double-quoted strings together without interpretation.
* Splits everything else on whitespace.
i.e.:
"expr1|expr2 (expr3-(expr4 or expr5))"
becomes:
['expr1', '|', 'expr2', '(', 'expr3', '-', '(', 'expr4', 'or', 'expr5', ')', ')']
special case:
"e(+)e(-)" interprets '+' and '-' as word characters since they are in parens with
word characters run up against them.
it becomes:
['e(+)e(-)']
"""
###
# Invariants:
# * Query is never modified
# * In every loop iteration, querytokens grows to the right
# * The only return point is at the bottom of the function, and the only
# return value is querytokens
###
def get_tokens(s):
"""
Given string s, return a list of s's tokens.
Adds space around special punctuation, then splits on whitespace.
"""
s = ' '+s
s = s.replace('->', '####DATE###RANGE##OP#') # XXX: Save '->'
s = re.sub('(?P<outside>[a-zA-Z0-9_,=:]+)\((?P<inside>[a-zA-Z0-9_,+-/]*)\)',
'#####\g<outside>####PAREN###\g<inside>##PAREN#', s) # XXX: Save U(1) and SL(2,Z)
s = re.sub('####PAREN###(?P<content0>[.0-9/-]*)(?P<plus>[+])(?P<content1>[.0-9/-]*)##PAREN#',
'####PAREN###\g<content0>##PLUS##\g<content1>##PAREN#', s)
s = re.sub('####PAREN###(?P<content0>([.0-9/]|##PLUS##)*)(?P<minus>[-])' +\
'(?P<content1>([.0-9/]|##PLUS##)*)##PAREN#',
'####PAREN###\g<content0>##MINUS##\g<content1>##PAREN#', s) # XXX: Save e(+)e(-)
for char in self.specials:
if char == '-':
s = s.replace(' -', ' - ')
s = s.replace(')-', ') - ')
s = s.replace('-(', ' - (')
else:
s = s.replace(char, ' '+char+' ')
s = re.sub('##PLUS##', '+', s)
s = re.sub('##MINUS##', '-', s) # XXX: Restore e(+)e(-)
s = re.sub('#####(?P<outside>[a-zA-Z0-9_,=:]+)####PAREN###(?P<inside>[a-zA-Z0-9_,+-/]*)##PAREN#',
'\g<outside>(\g<inside>)', s) # XXX: Restore U(1) and SL(2,Z)
s = s.replace('####DATE###RANGE##OP#', '->') # XXX: Restore '->'
return s.split()
querytokens = []
current_position = 0
re_quotes_match = re.compile(r'(?![\\])(".*?[^\\]")' + r"|(?![\\])('.*?[^\\]')")
for match in re_quotes_match.finditer(query):
match_start = match.start()
quoted_region = match.group(0).strip()
# clean the content after the previous quotes and before current quotes
unquoted = query[current_position : match_start]
querytokens.extend(get_tokens(unquoted))
# XXX: In case we end up with e.g. title:, "compton scattering", make it
# title:"compton scattering"
if querytokens and querytokens[0] and querytokens[-1][-1] == ':':
querytokens[-1] += quoted_region
# XXX: In case we end up with e.g. "expr1",->,"expr2", make it
# "expr1"->"expr2"
elif len(querytokens) >= 2 and querytokens[-1] == '->':
arrow = querytokens.pop()
querytokens[-1] += arrow + quoted_region
else:
# add our newly tokenized content to the token list
querytokens.extend([quoted_region])
# move current position to the end of the tokenized content
current_position = match.end()
# get tokens from the last appearance of quotes until the query end
unquoted = query[current_position : len(query)]
querytokens.extend(get_tokens(unquoted))
return querytokens
def parse(self, token_list, variable_substitution_dict=None):
"""Make token_list consumable by search_engine.
Turns a list of tokens and a variable mapping into a grouped list
of subexpressions in the format suitable for use by search_engine,
e.g.:
['+', 'searchterm', '-', 'searchterm to exclude', '|', 'another term']
Incidentally, this works recursively so parens can cause arbitrarily
deep nestings. But since the search_engine doesn't know about nested
structures, we need to flatten the input structure first.
"""
###
# Invariants:
# * Token list is never modified
# * Balanced parens remain balanced; unbalanced parens are an error
# * Individual tokens may only be exchanged for items in the variable
# substitution dict; otherwise they pass through unmolested
# * Return value is built up mostly as a stack
###
op_symbols = self.substitution_dict.values()
self.__tl_idx = 0
self.__tl_len = len(token_list)
def inner_parse(token_list, open_parens=False):
'''
although it's not in the API, it seems sensible to comment
this function a bit.
dist_token here is a token (e.g. a second-order operator)
which needs to be distributed across other tokens inside
the inner parens
'''
if open_parens:
parsed_values = []
else:
parsed_values = ['+']
i = 0
while i < len(token_list):
token = token_list[i]
if i > 0 and parsed_values[-1] not in op_symbols:
parsed_values.append('+')
if token == '(':
# if we need to distribute something over the tokens inside the parens
# we will know it because... it will end in a :
# that part of the list will be 'px', '+', '('
distributing = (len(parsed_values) > 2 and parsed_values[-2].endswith(':') and parsed_values[-1] == '+')
if distributing:
# we don't need the + if we are distributing
parsed_values = parsed_values[:-1]
offset = self.__tl_len - len(token_list)
inner_value = inner_parse(token_list[i+1:], True)
inner_value = ' '.join(inner_value)
if distributing:
if len(self.tokenize(inner_value)) == 1:
parsed_values[-1] = parsed_values[-1] + inner_value
elif "'" in inner_value:
parsed_values[-1] = parsed_values[-1] + '"' + inner_value + '"'
elif '"' in inner_value:
parsed_values[-1] = parsed_values[-1] + "'" + inner_value + "'"
else:
parsed_values[-1] = parsed_values[-1] + '"' + inner_value + '"'
else:
parsed_values.append(inner_value)
self.__tl_idx += 1
i = self.__tl_idx - offset
elif token == ')':
if parsed_values[-1] in op_symbols:
parsed_values = parsed_values[:-1]
if len(parsed_values) > 1 and parsed_values[0] == '+' and parsed_values[1] in op_symbols:
parsed_values = parsed_values[1:]
return parsed_values
elif token in op_symbols:
if len(parsed_values) > 0:
parsed_values[-1] = token
else:
parsed_values = [token]
else:
if variable_substitution_dict != None and token in variable_substitution_dict:
token = variable_substitution_dict[token]
parsed_values.append(token)
i += 1
self.__tl_idx += 1
# If we have an extra start symbol, remove the default one
if parsed_values[1] in op_symbols:
parsed_values = parsed_values[1:]
return parsed_values
return inner_parse(token_list, False)
class SpiresToInvenioSyntaxConverter:
"""Converts queries defined with SPIRES search syntax into queries
that use Invenio search syntax.
"""
# Constants defining fields
_DATE_ADDED_FIELD = 'datecreated:'
_DATE_UPDATED_FIELD = 'datemodified:'
_DATE_FIELD = 'year:'
_A_TAG = 'author:'
_EA_TAG = 'exactauthor:'
# Dictionary containing the matches between SPIRES keywords
# and their corresponding Invenio keywords or fields
# SPIRES keyword : Invenio keyword or field
_SPIRES_TO_INVENIO_KEYWORDS_MATCHINGS = {
# address
'address' : 'address:',
# affiliation
'affiliation' : 'affiliation:',
'affil' : 'affiliation:',
'aff' : 'affiliation:',
'af' : 'affiliation:',
'institution' : 'affiliation:',
'inst' : 'affiliation:',
# any field
'any' : 'anyfield:',
# author count
'ac' : 'authorcount:',
# bulletin
'bb' : 'reportnumber:',
'bbn' : 'reportnumber:',
'bull' : 'reportnumber:',
'bulletin-bd' : 'reportnumber:',
'bulletin-bd-no' : 'reportnumber:',
'eprint' : 'reportnumber:',
# citation / reference
'c' : 'reference:',
'citation' : 'reference:',
'cited' : 'reference:',
'jour-vol-page' : 'reference:',
'jvp' : 'reference:',
# collaboration
'collaboration' : 'collaboration:',
'collab-name' : 'collaboration:',
'cn' : 'collaboration:',
# conference number
'conf-number' : '111__g:',
'cnum' : '773__w:',
# country
'cc' : '044__a:',
'country' : '044__a:',
# date
'date': _DATE_FIELD,
'd': _DATE_FIELD,
# date added
'date-added': _DATE_ADDED_FIELD,
'dadd': _DATE_ADDED_FIELD,
'da': _DATE_ADDED_FIELD,
# date updated
'date-updated': _DATE_UPDATED_FIELD,
'dupd': _DATE_UPDATED_FIELD,
'du': _DATE_UPDATED_FIELD,
# first author
'fa' : 'firstauthor:',
'first-author' : 'firstauthor:',
# author
'a' : 'author:',
'au' : 'author:',
'author' : 'author:',
'name' : 'author:',
# exact author
# this is not a real keyword match. It is pseudo keyword that
# will be replaced later with author search
'ea' : 'exactauthor:',
'exact-author' : 'exactauthor:',
# experiment
'exp' : 'experiment:',
'experiment' : 'experiment:',
'expno' : 'experiment:',
'sd' : 'experiment:',
'se' : 'experiment:',
# journal
'journal' : 'journal:',
'j' : 'journal:',
'published_in' : 'journal:',
'spicite' : 'journal:',
'vol' : 'journal:',
# journal page
'journal-page' : '773__c:',
'jp' : '773__c:',
# journal year
'journal-year' : '773__y:',
'jy' : '773__y:',
# key
'key' : '970__a:',
'irn' : '970__a:',
'record' : '970__a:',
'document' : '970__a:',
'documents' : '970__a:',
# keywords
'k' : 'keyword:',
'keywords' : 'keyword:',
'kw' : 'keyword:',
# note
'note' : '500__a:',
# old title
'old-title' : '246__a:',
'old-t' : '246__a:',
'ex-ti' : '246__a:',
'et' : '246__a:',
#postal code
'postalcode' : 'postalcode:',
'zip' : 'postalcode:',
'cc' : 'postalcode:',
# ppf subject
'ppf-subject' : '650__a:',
'status' : '650__a:',
# recid
'recid' : 'recid:',
# report number
'r' : 'reportnumber:',
'rn' : 'reportnumber:',
'rept' : 'reportnumber:',
'report' : 'reportnumber:',
'report-num' : 'reportnumber:',
# title
't' : 'title:',
'ti' : 'title:',
'title' : 'title:',
'with-language' : 'title:',
# fulltext
'fulltext' : 'fulltext:',
'ft' : 'fulltext:',
# topic
'topic' : '695__a:',
'tp' : '695__a:',
'hep-topic' : '695__a:',
'desy-keyword' : '695__a:',
'dk' : '695__a:',
# topcite
'topcit' : 'cited:',
'topcite' : 'cited:',
# captions
'caption' : 'caption:',
# category
'arx' : '037__c:',
'category' : '037__c:',
# primarch
'parx' : '037__c:',
'primarch' : '037__c:',
# texkey
'texkey' : '035__z:',
# type code
'tc' : 'collection:',
'ty' : 'collection:',
'type' : 'collection:',
'type-code' : 'collection:',
'scl': 'collection:',
'ps': 'collection:',
# field code
'f' : 'subject:',
'fc' : 'subject:',
'field' : 'subject:',
'field-code' : 'subject:',
'subject' : 'subject:',
# coden
'bc' : 'journal:',
'browse-only-indx' : 'journal:',
'coden' : 'journal:',
'journal-coden' : 'journal:',
# jobs specific codes
'job' : 'title:',
'position' : 'title:',
'region' : 'region:',
'continent' : 'region:',
'deadline' : '046__a:',
'rank' : 'rank:',
# replace all the keywords without match with empty string
# this will remove the noise from the unknown keywrds in the search
# and will in all fields for the words following the keywords
# energy
'e' : '',
'energy' : '',
'energyrange-code' : '',
# exact experiment number
'ee' : '',
'exact-exp' : '',
'exact-expno' : '',
# hidden note
'hidden-note' : '',
'hn' : '',
# ppf
'ppf' : '',
'ppflist' : '',
# slac topics
'ppfa' : '',
'slac-topics' : '',
'special-topics' : '',
'stp' : '',
# test index
'test' : '',
'testindex' : '',
}
_SECOND_ORDER_KEYWORD_MATCHINGS = {
'refersto' : 'refersto:',
'refs': 'refersto:',
'citedby' : 'citedby:'
}
_INVENIO_KEYWORDS_FOR_SPIRES_PHRASE_SEARCHES = [
'affiliation:',
#'cited:', # topcite is technically a phrase index - this isn't necessary
'773__y:', # journal-year
'773__c:', # journal-page
'773__w:', # cnum
'044__a:', # country code
'subject:', # field code
'collection:', # type code
'035__z:', # texkey
# also exact expno, corp-auth, url, abstract, doi, mycite, citing
# but we have no invenio equivalents for these ATM
]
def __init__(self):
"""Initialize the state of the converter"""
self._months = {}
self._month_name_to_month_number = {}
self._init_months()
self._compile_regular_expressions()
def _compile_regular_expressions(self):
"""Compiles some of the regular expressions that are used in the class
for higher performance."""
# regular expression that matches the contents in single and double quotes
# taking in mind if they are escaped.
self._re_quotes_match = re.compile(r'(?![\\])(".*?[^\\]")' + r"|(?![\\])('.*?[^\\]')")
# match cases where a keyword distributes across a conjunction
self._re_distribute_keywords = re.compile(r'''(?ix) # verbose, ignorecase on
\b(?P<keyword>\S*:) # a keyword is anything that's not whitespace with a colon
(?P<content>[^:]+?)\s* # content is the part that comes after the keyword; it should NOT
# have colons in it! that implies that we might be distributing
# a keyword OVER another keyword. see ticket #701
(?P<combination>\ and\ not\ |\ and\ |\ or\ |\ not\ )\s*
(?P<last_content>[^:]*?) # oh look, content without a keyword!
(?=\ and\ |\ or\ |\ not\ |$)''')
# massaging SPIRES quirks
self._re_pattern_IRN_search = re.compile(r'970__a:(?P<irn>\d+)')
self._re_topcite_match = re.compile(r'(?P<x>cited:\d+)\+')
# regular expression that matches author patterns
# and author patterns with second-order-ops on top
# does not match names with " or ' around them, since
# those should not be touched
self._re_author_match = re.compile(r'''(?ix) # verbose, ignorecase
\b((?P<secondorderop>[^\s]+:)?) # do we have a second-order-op on top?
((?P<first>first)?)author:(?P<name>
[^\'\"] # first character not a quotemark
[^()]*? # some stuff that isn't parentheses (that is dealt with in pp)
[^\'\"]) # last character not a quotemark
(?=\ and\ not\ |\ and\ |\ or\ |\ not\ |$)''')
# regular expression that matches exact author patterns
# the group defined in this regular expression is used in method
# _convert_spires_exact_author_search_to_invenio_author_search(...)
# in case of changes correct also the code in this method
self._re_exact_author_match = re.compile(r'\b((?P<secondorderop>[^\s]+:)?)exactauthor:(?P<author_name>[^\'\"].*?[^\'\"]\b)(?= and not | and | or | not |$)', re.IGNORECASE)
# match a second-order operator with no operator following it
self._re_second_order_op_no_index_match = re.compile(r'''(?ix) # ignorecase, verbose
(^|\b|:)(?P<second_order_op>(refersto|citedby):)
(?P<search_terms>[^\"\'][^:]+?) # anything without an index should be absorbed here
\s*
(?P<conjunction_or_next_keyword>(\ and\ |\ not\ |\ or\ |\ \w+:\w+|$))
''')
# match search term, its content (words that are searched) and
# the operator preceding the term.
self._re_search_term_pattern_match = re.compile(r'\b(?P<combine_operator>find|and|or|not)\s+(?P<search_term>\S+:)(?P<search_content>.+?)(?= and not | and | or | not |$)', re.IGNORECASE)
# match journal searches
self._re_search_term_is_journal = re.compile(r'''(?ix) # verbose, ignorecase
\b(?P<leading>(find|and|or|not)\s+journal:) # first combining operator and index
(?P<search_content>.+?) # what we are searching
(?=\ and\ not\ |\ and\ |\ or\ |\ not\ |$)''')
# regular expression matching date after pattern
self._re_date_after_match = re.compile(r'\b(?P<searchop>d|date|dupd|dadd|da|date-added|du|date-updated)\b\s*(after|>)\s*(?P<search_content>.+?)(?= and not | and | or | not |$)', re.IGNORECASE)
# regular expression matching date after pattern
self._re_date_before_match = re.compile(r'\b(?P<searchop>d|date|dupd|dadd|da|date-added|du|date-updated)\b\s*(before|<)\s*(?P<search_content>.+?)(?= and not | and | or | not |$)', re.IGNORECASE)
# match date searches which have been keyword-substituted
self._re_keysubbed_date_expr = re.compile(r'\b(?P<term>(' + self._DATE_ADDED_FIELD + ')|(' + self._DATE_UPDATED_FIELD + ')|(' + self._DATE_FIELD + '))(?P<content>.+?)(?= and not | and | or | not |$)', re.IGNORECASE)
# for finding (and changing) a variety of different SPIRES search keywords
self._re_spires_find_keyword = re.compile('^(f|fin|find)\s+', re.IGNORECASE)
# for finding boolean expressions
self._re_boolean_expression = re.compile(r' and | or | not | and not ')
# patterns for subbing out spaces within quotes temporarily
self._re_pattern_single_quotes = re.compile("'(.*?)'")
self._re_pattern_double_quotes = re.compile("\"(.*?)\"")
self._re_pattern_regexp_quotes = re.compile("\/(.*?)\/")
self._re_pattern_space = re.compile("__SPACE__")
self._re_pattern_equals = re.compile("__EQUALS__")
# for date math:
self._re_datemath = re.compile(r'(?P<datestamp>.+)\s+(?P<operator>[-+])\s+(?P<units>\d+)')
def is_applicable(self, query):
"""Is this converter applicable to this query?
Return true if query begins with find, fin, or f, or if it contains
a SPIRES-specific keyword (a, t, etc.), or if it contains the invenio
author: field search. """
if not CFG_WEBSEARCH_SPIRES_SYNTAX:
#SPIRES syntax is switched off
return False
query = query.lower()
if self._re_spires_find_keyword.match(query):
#leading 'find' is present and SPIRES syntax is switched on
return True
if CFG_WEBSEARCH_SPIRES_SYNTAX > 1:
for word in query.split(' '):
if self._SPIRES_TO_INVENIO_KEYWORDS_MATCHINGS.has_key(word):
return True
return False
def convert_query(self, query):
"""Convert SPIRES syntax queries to Invenio syntax.
Do nothing to queries not in SPIRES syntax."""
# SPIRES syntax allows searches with 'find' or 'fin'.
if self.is_applicable(query):
query = re.sub(self._re_spires_find_keyword, 'find ', query)
if not query.startswith('find'):
query = 'find ' + query
# a holdover from SPIRES syntax is e.g. date = 2000 rather than just date 2000
query = self._remove_extraneous_equals_signs(query)
# these calls are before keywords replacement because when keywords
# are replaced, date keyword is replaced by specific field search
# and the DATE keyword is not match in DATE BEFORE or DATE AFTER
query = self._convert_spires_date_before_to_invenio_span_query(query)
query = self._convert_spires_date_after_to_invenio_span_query(query)
# call to _replace_spires_keywords_with_invenio_keywords should be at the
# beginning because the next methods use the result of the replacement
query = self._standardize_already_invenio_keywords(query)
query = self._replace_spires_keywords_with_invenio_keywords(query)
query = self._normalise_journal_page_format(query)
query = self._distribute_keywords_across_combinations(query)
query = self._distribute_and_quote_second_order_ops(query)
query = self._convert_dates(query)
query = self._convert_irns_to_spires_irns(query)
query = self._convert_topcite_to_cited(query)
query = self._convert_spires_author_search_to_invenio_author_search(query)
query = self._convert_spires_exact_author_search_to_invenio_author_search(query)
query = self._convert_spires_truncation_to_invenio_truncation(query)
query = self._expand_search_patterns(query)
# remove FIND in the beginning of the query as it is not necessary in Invenio
query = query[4:]
query = query.strip()
return query
def _init_months(self):
"""Defines a dictionary matching the name
of the month with its corresponding number"""
# this dictionary is used when generating match patterns for months
self._months = {'jan':'01', 'january':'01',
'feb':'02', 'february':'02',
'mar':'03', 'march':'03',
'apr':'04', 'april':'04',
'may':'05', 'may':'05',
'jun':'06', 'june':'06',
'jul':'07', 'july':'07',
'aug':'08', 'august':'08',
'sep':'09', 'september':'09',
'oct':'10', 'october':'10',
'nov':'11', 'november':'11',
'dec':'12', 'december':'12'}
# this dictionary is used to transform name of the month
# to a number used in the date format. By this reason it
# contains also the numbers itself to simplify the conversion
self._month_name_to_month_number = {'1':'01', '01':'01',
'2':'02', '02':'02',
'3':'03', '03':'03',
'4':'04', '04':'04',
'5':'05', '05':'05',
'6':'06', '06':'06',
'7':'07', '07':'07',
'8':'08', '08':'08',
'9':'09', '09':'09',
'10':'10',
'11':'11',
'12':'12',}
# combine it with months in order to cover all the cases
self._month_name_to_month_number.update(self._months)
def _get_month_names_match(self):
"""Retruns part of a patter that matches month in a date"""
months_match = ''
for month_name in self._months.keys():
months_match = months_match + month_name + '|'
months_match = r'\b(' + months_match[0:-1] + r')\b'
return months_match
def _convert_dates(self, query):
"""Tries to find dates in query and make them look like ISO-8601."""
def parse_relative_unit(date_str):
units = 0
datemath = self._re_datemath.match(date_str)
if datemath:
date_str = datemath.group('datestamp')
units = int(datemath.group('operator') + datemath.group('units'))
return date_str, units
def parse_date_unit(date_str):
begin = date_str
end = None
# First split, relative time directive
# e.g. "2012-01-01 - 3" to ("2012-01-01", -3)
date_str, relative_units = parse_relative_unit(date_str)
try:
d = datetime.strptime(date_str, '%Y-%m-%d')
d += du_delta(days=relative_units)
return datetime.strftime(d, '%Y-%m-%d'), end
except ValueError:
pass
try:
d = datetime.strptime(date_str, '%y-%m-%d')
d += du_delta(days=relative_units)
if d.year > datetime.today().year:
d -= du_delta(years=100)
return datetime.strftime(d, '%Y-%m-%d'), end
except ValueError:
pass
try:
d = datetime.strptime(date_str, '%Y-%m')
d += du_delta(months=relative_units)
return datetime.strftime(d, '%Y-%m'), end
except ValueError:
pass
try:
d = datetime.strptime(date_str, '%Y')
d += du_delta(years=relative_units)
return datetime.strftime(d, '%Y'), end
except ValueError:
pass
try:
d = datetime.strptime(date_str, '%y')
d += du_delta(days=relative_units)
if d.year > datetime.today().year:
d -= du_delta(years=100)
return datetime.strftime(d, '%Y'), end
except ValueError:
pass
if 'this week' in date_str:
# Past monday to today
# This week is iffy, not sure if we should
# start with sunday or monday
begin = datetime.today()
begin += du_delta(weekday=relativedelta.SU(-1))
end = datetime.today()
begin = datetime.strftime(begin, '%Y-%m-%d')
end = datetime.strftime(end, '%Y-%m-%d')
elif 'last week' in date_str:
# Past monday to today
# Same problem as last week
begin = datetime.today()
begin += du_delta(weekday=relativedelta.SU(-2))
end = datetime.today()
end += du_delta(weekday=relativedelta.SA(-1))
begin = datetime.strftime(begin, '%Y-%m-%d')
end = datetime.strftime(end, '%Y-%m-%d')
elif 'this month' in date_str:
d = datetime.today()
begin = datetime.strftime(d, '%Y-%m')
elif 'last month' in date_str:
d = datetime.today() - du_delta(months=1)
begin = datetime.strftime(d, '%Y-%m')
elif 'yesterday' in date_str:
d = datetime.today() - du_delta(days=1)
begin = datetime.strftime(d, '%Y-%m-%d')
elif 'today' in date_str:
start = datetime.today()
start += du_delta(days=relative_units)
begin = datetime.strftime(start, '%Y-%m-%d')
elif date_str.strip() == '0':
begin = '0'
else:
default = datetime(datetime.today().year, 1, 1)
- d = du_parser.parse(date_str, default=default)
- begin = datetime.strftime(d, '%Y-%m-%d')
+ try:
+ d = du_parser.parse(date_str, default=default)
+ except ValueError:
+ begin = date_str
+ else:
+ begin = datetime.strftime(d, '%Y-%m-%d')
return begin, end
def mangle_with_dateutils(query):
result = ''
position = 0
for match in self._re_keysubbed_date_expr.finditer(query):
result += query[position : match.start()]
datestamp = match.group('content')
if '->' in datestamp:
begin_unit, end_unit = datestamp.split('->', 1)
begin, dummy = parse_date_unit(begin_unit)
end, dummy = parse_date_unit(end_unit)
else:
begin, end = parse_date_unit(datestamp)
if end:
daterange = '%s->%s' % (begin, end)
else:
daterange = begin
result += match.group('term') + daterange
position = match.end()
result += query[position : ]
return result
if GOT_DATEUTIL:
query = mangle_with_dateutils(query)
# else do nothing with the dates
return query
def _convert_irns_to_spires_irns(self, query):
"""Prefix IRN numbers with SPIRES- so they match the INSPIRE format."""
def create_replacement_pattern(match):
"""method used for replacement with regular expression"""
return '970__a:SPIRES-' + match.group('irn')
query = self._re_pattern_IRN_search.sub(create_replacement_pattern, query)
return query
def _convert_topcite_to_cited(self, query):
"""Replace SPIRES topcite x+ with cited:x->999999999"""
def create_replacement_pattern(match):
"""method used for replacement with regular expression"""
return match.group('x') + '->999999999'
query = self._re_topcite_match.sub(create_replacement_pattern, query)
return query
def _convert_spires_date_after_to_invenio_span_query(self, query):
"""Converts date after SPIRES search term into invenio span query"""
def create_replacement_pattern(match):
"""method used for replacement with regular expression"""
return match.group('searchop') + ' ' + match.group('search_content') + '->9999'
query = self._re_date_after_match.sub(create_replacement_pattern, query)
return query
def _convert_spires_date_before_to_invenio_span_query(self, query):
"""Converts date before SPIRES search term into invenio span query"""
# method used for replacement with regular expression
def create_replacement_pattern(match):
return match.group('searchop') + ' ' + '0->' + match.group('search_content')
query = self._re_date_before_match.sub(create_replacement_pattern, query)
return query
def _expand_search_patterns(self, query):
"""Expands search queries.
If a search term is followed by several words e.g.
author:ellis or title:THESE THREE WORDS it is expanded to
author:ellis or (title:THESE and title:THREE...)
All keywords are thus expanded. XXX: this may lead to surprising
results for any later parsing stages if we're not careful.
"""
def create_replacements(term, content):
result = ''
content = content.strip()
# replace spaces within quotes by __SPACE__ temporarily:
content = self._re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), ' ', '__SPACE__')+"'", content)
content = self._re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), ' ', '__SPACE__')+"\"", content)
content = self._re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), ' ', '__SPACE__')+"/", content)
if term in self._INVENIO_KEYWORDS_FOR_SPIRES_PHRASE_SEARCHES \
and not self._re_boolean_expression.search(content) and ' ' in content:
# the case of things which should be searched as phrases
result = term + '"' + content + '"'
else:
words = content.split()
if len(words) == 0:
# this should almost never happen, req user to say 'find a junk:'
result = term
elif len(words) == 1:
# this is more common but still occasional
result = term + words[0]
else:
# general case
result = '(' + term + words[0]
for word in words[1:]:
result += ' and ' + term + word
result += ')'
# replace back __SPACE__ by spaces:
result = self._re_pattern_space.sub(" ", result)
return result.strip()
result = ''
current_position = 0
for match in self._re_search_term_pattern_match.finditer(query):
result += query[current_position : match.start()]
result += ' ' + match.group('combine_operator') + ' '
result += create_replacements(match.group('search_term'), match.group('search_content'))
current_position = match.end()
result += query[current_position : len(query)]
return result.strip()
def _remove_extraneous_equals_signs(self, query):
"""In SPIRES, both date = 2000 and date 2000 are acceptable. Get rid of the ="""
query = self._re_pattern_single_quotes.sub(lambda x: "'"+string.replace(x.group(1), '=', '__EQUALS__')+"'", query)
query = self._re_pattern_double_quotes.sub(lambda x: "\""+string.replace(x.group(1), '=', '__EQUALS__')+'\"', query)
query = self._re_pattern_regexp_quotes.sub(lambda x: "/"+string.replace(x.group(1), '=', '__EQUALS__')+"/", query)
query = query.replace('=', '')
query = self._re_pattern_equals.sub("=", query)
return query
def _convert_spires_truncation_to_invenio_truncation(self, query):
"""Replace SPIRES truncation symbol # with invenio trancation symbol *"""
return query.replace('#', '*')
def _convert_spires_exact_author_search_to_invenio_author_search(self, query):
"""Converts SPIRES search patterns for exact author into search pattern
for invenio"""
# method used for replacement with regular expression
def create_replacement_pattern(match):
# the regular expression where this group name is defined is in
# the method _compile_regular_expressions()
return self._EA_TAG + '"' + match.group('author_name') + '"'
query = self._re_exact_author_match.sub(create_replacement_pattern, query)
return query
def _convert_spires_author_search_to_invenio_author_search(self, query):
"""Converts SPIRES search patterns for authors to search patterns in invenio
that give similar results to the spires search.
"""
# result of the replacement
result = ''
current_position = 0
for match in self._re_author_match.finditer(query):
result += query[current_position : match.start() ]
if match.group('secondorderop'):
result += match.group('secondorderop')
scanned_name = NameScanner.scan(match.group('name'))
author_atoms = self._create_author_search_pattern_from_fuzzy_name_dict(scanned_name)
if match.group('first'):
author_atoms = author_atoms.replace('author:', 'firstauthor:')
if author_atoms.find(' ') == -1:
result += author_atoms + ' '
else:
result += '(' + author_atoms + ') '
current_position = match.end()
result += query[current_position : len(query)]
return result
def _create_author_search_pattern_from_fuzzy_name_dict(self, fuzzy_name):
"""Creates an invenio search pattern for an author from a fuzzy name dict"""
author_name = ''
author_middle_name = ''
author_surname = ''
full_search = ''
if len(fuzzy_name['nonlastnames']) > 0:
author_name = fuzzy_name['nonlastnames'][0]
if len(fuzzy_name['nonlastnames']) == 2:
author_middle_name = fuzzy_name['nonlastnames'][1]
if len(fuzzy_name['nonlastnames']) > 2:
author_middle_name = ' '.join(fuzzy_name['nonlastnames'][1:])
if fuzzy_name['raw']:
full_search = fuzzy_name['raw']
author_surname = ' '.join(fuzzy_name['lastnames'])
NAME_IS_INITIAL = (len(author_name) == 1)
NAME_IS_NOT_INITIAL = not NAME_IS_INITIAL
# we expect to have at least surname
if author_surname == '' or author_surname == None:
return ''
# ellis ---> "author:ellis"
#if author_name == '' or author_name == None:
if not author_name:
return self._A_TAG + author_surname
# ellis, j ---> "ellis, j*"
if NAME_IS_INITIAL and not author_middle_name:
return self._A_TAG + '"' + author_surname + ', ' + author_name + '*"'
# if there is middle name we expect to have also name and surname
# ellis, j. r. ---> ellis, j* r*
# j r ellis ---> ellis, j* r*
# ellis, john r. ---> ellis, j* r* or ellis, j. r. or ellis, jo. r.
# ellis, john r. ---> author:ellis, j* r* or exactauthor:ellis, j r or exactauthor:ellis jo r
if author_middle_name:
search_pattern = self._A_TAG + '"' + author_surname + ', ' + author_name + '*' + ' ' + author_middle_name.replace(" ","* ") + '*"'
if NAME_IS_NOT_INITIAL:
for i in range(1, len(author_name)):
search_pattern += ' or ' + self._EA_TAG + "\"%s, %s %s\"" % (author_surname, author_name[0:i], author_middle_name)
return search_pattern
# ellis, jacqueline ---> "ellis, jacqueline" or "ellis, j.*" or "ellis, j" or "ellis, ja.*" or "ellis, ja" or "ellis, jacqueline *, ellis, j *"
# in case we don't use SPIRES data, the ending dot is ommited.
search_pattern = self._A_TAG + '"' + author_surname + ', ' + author_name + '*"'
search_pattern += " or " + self._EA_TAG + "\"%s, %s *\"" % (author_surname, author_name[0])
if NAME_IS_NOT_INITIAL:
for i in range(1,len(author_name)):
search_pattern += ' or ' + self._EA_TAG + "\"%s, %s\"" % (author_surname, author_name[0:i])
search_pattern += ' or %s"%s, *"' % (self._A_TAG, full_search)
return search_pattern
def _normalise_journal_page_format(self, query):
"""Phys.Lett, 0903, 024 -> Phys.Lett,0903,024"""
def _is_triple(search):
return (len(re.findall('\s+', search)) + len(re.findall(':', search))) == 2
def _normalise_spaces_and_colons_to_commas_in_triple(search):
if not _is_triple(search):
return search
search = re.sub(',\s+', ',', search)
search = re.sub('\s+', ',', search)
search = re.sub(':', ',', search)
return search
result = ""
current_position = 0
for match in self._re_search_term_is_journal.finditer(query):
result += query[current_position : match.start()]
result += match.group('leading')
search = match.group('search_content')
search = _normalise_spaces_and_colons_to_commas_in_triple(search)
result += search
current_position = match.end()
result += query[current_position : ]
return result
def _standardize_already_invenio_keywords(self, query):
"""Replaces invenio keywords kw with "and kw" in order to
parse them correctly further down the line."""
unique_invenio_keywords = set(self._SPIRES_TO_INVENIO_KEYWORDS_MATCHINGS.values()) |\
set(self._SECOND_ORDER_KEYWORD_MATCHINGS.values())
unique_invenio_keywords.remove('') # for the ones that don't have invenio equivalents
for invenio_keyword in unique_invenio_keywords:
query = re.sub("(?<!... \+|... -| and |. or | not |....:)"+invenio_keyword, "and "+invenio_keyword, query)
query = re.sub("\+"+invenio_keyword, "and "+invenio_keyword, query)
query = re.sub("-"+invenio_keyword, "and not "+invenio_keyword, query)
return query
def _replace_spires_keywords_with_invenio_keywords(self, query):
"""Replaces SPIRES keywords that have directly
corresponding Invenio keywords
Replacements are done only in content that is not in quotes."""
# result of the replacement
result = ""
current_position = 0
for match in self._re_quotes_match.finditer(query):
# clean the content after the previous quotes and before current quotes
cleanable_content = query[current_position : match.start()]
cleanable_content = self._replace_all_spires_keywords_in_string(cleanable_content)
# get the content in the quotes (group one matches double
# quotes, group 2 singles)
if match.group(1):
quoted_content = match.group(1)
elif match.group(2):
quoted_content = match.group(2)
# append the processed content to the result
result = result + cleanable_content + quoted_content
# move current position at the end of the processed content
current_position = match.end()
# clean the content from the last appearance of quotes till the end of the query
cleanable_content = query[current_position : len(query)]
cleanable_content = self._replace_all_spires_keywords_in_string(cleanable_content)
result = result + cleanable_content
return result
def _replace_all_spires_keywords_in_string(self, query):
"""Replaces all SPIRES keywords in the string with their
corresponding Invenio keywords"""
for spires_keyword, invenio_keyword in self._SPIRES_TO_INVENIO_KEYWORDS_MATCHINGS.iteritems():
query = self._replace_keyword(query, spires_keyword, invenio_keyword)
for spires_keyword, invenio_keyword in self._SECOND_ORDER_KEYWORD_MATCHINGS.iteritems():
query = self._replace_second_order_keyword(query, spires_keyword, invenio_keyword)
return query
def _replace_keyword(self, query, old_keyword, new_keyword):
"""Replaces old keyword in the query with a new keyword"""
regex_string = r'(?P<operator>(^find|\band|\bor|\bnot|\brefersto|\bcitedby|^)\b[:\s\(]*)' + \
old_keyword + r'(?P<end>[\s\(]+|$)'
regular_expression = re.compile(regex_string, re.IGNORECASE)
result = regular_expression.sub(r'\g<operator>' + new_keyword + r'\g<end>', query)
result = re.sub(':\s+', ':', result)
return result
def _replace_second_order_keyword(self, query, old_keyword, new_keyword):
"""Replaces old second-order keyword in the query with a new keyword"""
regular_expression =\
re.compile(r'''(?ix) # verbose, ignorecase
(?P<operator>
(^find|\band|\bor|\bnot|\brefersto|\bcitedby|^)\b # operator preceding our operator
[:\s\(]* # trailing colon, spaces, parens, etc. for that operator
)
%s # the keyword we're searching for
(?P<endorop>
\s*[a-z]+:| # either an operator (like author:)
[\s\(]+| # or a paren opening
$ # or the end of the string
)''' % old_keyword)
result = regular_expression.sub(r'\g<operator>' + new_keyword + r'\g<endorop>', query)
result = re.sub(':\s+', ':', result)
return result
def _distribute_keywords_across_combinations(self, query):
"""author:ellis and james -> author:ellis and author:james"""
# method used for replacement with regular expression
def create_replacement_pattern(match):
return match.group('keyword') + match.group('content') + \
match.group('combination') + match.group('keyword') + \
match.group('last_content')
still_matches = True
while still_matches:
query = self._re_distribute_keywords.sub(create_replacement_pattern, query)
still_matches = self._re_distribute_keywords.search(query)
query = re.sub(r'\s+', ' ', query)
return query
def _distribute_and_quote_second_order_ops(self, query):
"""refersto:s parke -> refersto:\"s parke\""""
def create_replacement_pattern(match):
return match.group('second_order_op') + '"' +\
match.group('search_terms') + '"' +\
match.group('conjunction_or_next_keyword')
for match in self._re_second_order_op_no_index_match.finditer(query):
query = self._re_second_order_op_no_index_match.sub(create_replacement_pattern, query)
query = re.sub(r'\s+', ' ', query)
return query
diff --git a/modules/websearch/lib/search_engine_query_parser_unit_tests.py b/modules/websearch/lib/search_engine_query_parser_unit_tests.py
index c3a1563af..424fe62c0 100644
--- a/modules/websearch/lib/search_engine_query_parser_unit_tests.py
+++ b/modules/websearch/lib/search_engine_query_parser_unit_tests.py
@@ -1,1046 +1,1052 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2008, 2010, 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Unit tests for the search engine query parsers."""
import unittest
import datetime
from invenio import search_engine_query_parser
from invenio.testutils import make_test_suite, run_test_suite
from invenio.search_engine import create_basic_search_units, perform_request_search
from invenio.config import CFG_WEBSEARCH_SPIRES_SYNTAX
if search_engine_query_parser.GOT_DATEUTIL:
import dateutil
from dateutil.relativedelta import relativedelta as du_delta
DATEUTIL_AVAILABLE = True
else:
DATEUTIL_AVAILABLE = False
class TestParserUtilityFunctions(unittest.TestCase):
"""Test utility functions for the parsing components"""
def setUp(self):
self.parser = search_engine_query_parser.SearchQueryParenthesisedParser()
self.converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
def test_ndb_simple(self):
"""SQPP.test_nesting_depth_and_balance: ['p0']"""
self.assertEqual((0, True, 0),
self.parser.nesting_depth_and_balance(['p0']))
def test_ndb_simple_useful(self):
"""SQPP.test_nesting_depth_and_balance: ['(', 'p0', ')']"""
self.assertEqual((1, True, 1),
self.parser.nesting_depth_and_balance(['(', 'p0', ')']))
def test_ndb_slightly_complicated(self):
"""SQPP.test_nesting_depth_and_balance: ['(', 'p0', ')', '|', '(', 'p2', '+', 'p3', ')']"""
self.assertEqual((1, True, 2),
self.parser.nesting_depth_and_balance(['(', 'p0', ')', '|', '(', 'p2', '+', 'p3', ')']))
def test_ndb_sorta_hairy(self):
"""SQPP.test_nesting_depth_and_balance: ['(', '(', ')', ')', '(', '(', '(', ')', ')', ')']"""
self.assertEqual((3, True, 2),
self.parser.nesting_depth_and_balance(['(', '(', ')', ')', '(', '(', '(', ')', ')', ')']))
def test_ndb_broken_rhs(self):
"""SQPP.test_nesting_depth_and_balance: ['(', '(', ')', ')', '(', '(', '(', ')', ')', ]"""
self.assertEqual((3, False, 2),
self.parser.nesting_depth_and_balance(['(', '(', ')', ')', '(', '(', '(', ')', ')', ]))
def test_ndb_broken_lhs(self):
"""SQPP.test_nesting_depth_and_balance: ['(', ')', ')', '(', '(', '(', ')', ')', ')']"""
self.assertEqual((3, False, 2),
self.parser.nesting_depth_and_balance(['(', ')', ')', '(', '(', '(', ')', ')', ]))
def test_stisc(self):
"""Test whole convert/parse stack: SQPP.parse_query(STISC.convert_query('find a richter, burton and t quark'))"""
self.assertEqual(self.parser.parse_query(self.converter.convert_query('find a richter, burton and t quark')),
['+',
'author:"richter, burton*" | exactauthor:"richter, b *" | exactauthor:"richter, b" | exactauthor:"richter, bu" | exactauthor:"richter, bur" | exactauthor:"richter, burt" | exactauthor:"richter, burto" | author:"richter, burton, *"',
'+', 'title:quark'])
def test_stisc_not_vs_and_not1(self):
"""Parse stack parses "find a ellis, j and not a enqvist" == "find a ellis, j not a enqvist" """
self.assertEqual(self.parser.parse_query(self.converter.convert_query('find a ellis, j and not a enqvist')),
self.parser.parse_query(self.converter.convert_query('find a ellis, j not a enqvist')))
def test_stisc_not_vs_and_not2(self):
"""Parse stack parses "find a mangano, m and not a ellis, j" == "find a mangano, m not a ellis, j" """
self.assertEqual(self.parser.parse_query(self.converter.convert_query('find a mangano, m and not a ellis, j')),
self.parser.parse_query(self.converter.convert_query('find a mangano, m not a ellis, j')))
class TestSearchQueryParenthesisedParser(unittest.TestCase):
"""Test parenthesis parsing."""
def setUp(self):
self.parser = search_engine_query_parser.SearchQueryParenthesisedParser()
def test_sqpp_atom(self):
"""SearchQueryParenthesisedParser - expr1"""
self.assertEqual(self.parser.parse_query('expr1'),
['+', 'expr1'])
def test_sqpp_parened_atom(self):
"""SearchQueryParenthesisedParser - (expr1)"""
self.assertEqual(self.parser.parse_query('(expr1)'),
['+', 'expr1'])
def test_sqpp_expr1_minus_expr2(self):
"""SearchQueryParenthesisedParser - expr1 - (expr2)"""
self.assertEqual(self.parser.parse_query("expr1 - (expr2)"),
['+', 'expr1', '-', 'expr2'])
def test_sqpp_plus_expr1_minus_paren_expr2(self):
"""SearchQueryParenthesisedParser - + expr1 - (expr2)"""
self.assertEqual(self.parser.parse_query("+ expr1 - (expr2)"),
['+', 'expr1', '-', 'expr2'])
def test_sqpp_expr1_paren_expr2(self):
"""SearchQueryParenthesisedParser - expr1 (expr2)"""
self.assertEqual(self.parser.parse_query("expr1 (expr2)"),
['+', 'expr1', '+', 'expr2'])
def test_sqpp_paren_expr1_minus_expr2(self):
"""SearchQueryParenthesisedParser - (expr1) - expr2"""
self.assertEqual(self.parser.parse_query("(expr1) - expr2"),
['+', 'expr1', '-', 'expr2'])
def test_sqpp_paren_expr1_minus_paren_expr2(self):
"""SearchQueryParenthesisedParser - (expr1)-(expr2)"""
self.assertEqual(self.parser.parse_query("(expr1)-(expr2)"),
['+', 'expr1', '-', 'expr2'])
def test_sqpp_minus_paren_expr1_minus_paren_expr2(self):
"""SearchQueryParenthesisedParser - -(expr1)-(expr2)"""
self.assertEqual(self.parser.parse_query("-(expr1)-(expr2)"),
['-', 'expr1', '-', 'expr2'])
def test_sqpp_paren_expr1_minus_expr2_and_paren_expr3(self):
"""SearchQueryParenthesisedParser - (expr1) - expr2 + (expr3)"""
self.assertEqual(self.parser.parse_query('(expr1) - expr2 + (expr3)'),
['+', 'expr1', '-', 'expr2', '+', 'expr3'])
def test_sqpp_paren_expr1_minus_expr2_and_paren_expr3_or_expr4(self):
"""SearchQueryParenthesisedParser - (expr1) - expr2 + (expr3) | expr4"""
self.assertEqual(self.parser.parse_query('(expr1) - expr2 + (expr3) | expr4'),
['+', 'expr1', '-', 'expr2', '+', 'expr3', '|', 'expr4'])
#['+', '+ expr1 | expr4', '+', '- expr2 | expr4', '+', '+ expr3 | expr4'])
def test_sqpp_paren_expr1_minus_expr2_and_paren_expr3_or_expr4_or_quoted_expr5_and_expr6(self):
"""SearchQueryParenthesisedParser - (expr1) - expr2 + (expr3) | expr4 | \"expr5 + expr6\""""
self.assertEqual(self.parser.parse_query('(expr1) - expr2 + (expr3 | expr4) | "expr5 + expr6"'),
['+', 'expr1', '-', 'expr2', '+', 'expr3 | expr4', '|', '"expr5 + expr6"']),
#['+', '+ expr1 | "expr5 + expr6"', '+', '- expr2 | "expr5 + expr6"',
# '+', '+ expr3 | expr4 | "expr5 + expr6"'])
def test_sqpp_quoted_expr1_and_paren_expr2_and_expr3(self):
"""SearchQueryParenthesisedParser - \"expr1\" (expr2) expr3"""
self.assertEqual(self.parser.parse_query('"expr1" (expr2) expr3'),
['+', '"expr1"', '+', 'expr2', '+', 'expr3'])
def test_sqpp_quoted_expr1_arrow_quoted_expr2(self):
"""SearchQueryParenthesisedParser = \"expr1\"->\"expr2\""""
self.assertEqual(self.parser.parse_query('"expr1"->"expr2"'),
['+', '"expr1"->"expr2"'])
def test_sqpp_paren_expr1_expr2_paren_expr3_or_expr4(self):
"""SearchQueryParenthesisedParser - (expr1) expr2 (expr3) | expr4"""
# test parsing of queries with missing operators.
# in this case default operator + should be included on place of the missing one
self.assertEqual(self.parser.parse_query('(expr1) expr2 (expr3) | expr4'),
['+', 'expr1', '+', 'expr2', '+', 'expr3', '|', 'expr4'])
#['+', '+ expr1 | expr4', '+', '+ expr2 | expr4', '+', '+ expr3 | expr4'])
def test_sqpp_nested_paren_success(self):
"""SearchQueryParenthesizedParser - Arbitrarily nested parentheses: ((expr1)) + (expr2 - expr3)"""
self.assertEqual(self.parser.parse_query('((expr1)) + (expr2 - expr3)'),
['+', 'expr1', '+', 'expr2', '-', 'expr3'])
#['+', 'expr1', '+', 'expr2', '-', 'expr3'])
def test_sqpp_nested_paren_really_nested(self):
"""SearchQueryParenthesisedParser - Nested parentheses where order matters: expr1 - (expr2 - (expr3 | expr4))"""
self.assertEqual(self.parser.parse_query('expr1 - (expr2 - (expr3 | expr4))'),
['+', 'expr1', '+', '- expr2 | expr3 | expr4'])
def test_sqpp_paren_open_only_failure(self):
"""SearchQueryParenthesizedParser - Parentheses that only open should raise an exception"""
self.failUnlessRaises(SyntaxError,
self.parser.parse_query,"(expr")
def test_sqpp_paren_close_only_failure(self):
"""SearchQueryParenthesizedParser - Parentheses that only close should raise an exception"""
self.failUnlessRaises(SyntaxError,
self.parser.parse_query,"expr)")
def test_sqpp_paren_expr1_not_expr2_and_paren_expr3_or_expr4_WORDS(self):
"""SearchQueryParenthesisedParser - (expr1) not expr2 and (expr3) or expr4"""
self.assertEqual(self.parser.parse_query('(expr1) not expr2 and (expr3) or expr4'),
['+', 'expr1', '-', 'expr2', '+', 'expr3', '|', 'expr4'])
#['+', '+ expr1 | expr4', '+', '- expr2 | expr4', '+', '+ expr3 | expr4'])
def test_sqpp_paren_expr1_not_expr2_or_quoted_string_not_expr3_or_expr4WORDS(self):
"""SearchQueryParenthesisedParser - (expr1) not expr2 | "expressions not in and quotes | (are) not - parsed " - (expr3) or expr4"""
self.assertEqual(self.parser.parse_query('(expr1) not expr2 | "expressions not in and quotes | (are) not - parsed " - (expr3) or expr4'),
['+', 'expr1', '-', 'expr2', '|', '"expressions not in and quotes | (are) not - parsed "', '-', 'expr3', '|', 'expr4'])
#['+', '+ "expressions not in and quotes | (are) not - parsed " | expr1 | expr4',
# '+', '- expr3 | expr1 | expr4',
# '+', '+ "expressions not in and quotes | (are) not - parsed " - expr2 | expr4',
# '+', '- expr3 - expr2 | expr4'])
def test_sqpp_expr1_escaped_quoted_expr2_and_paren_expr3_not_expr4_WORDS(self):
"""SearchQueryParenthesisedParser - expr1 \\" expr2 foo(expr3) not expr4 \\" and (expr5)"""
self.assertEqual(self.parser.parse_query('expr1 \\" expr2 foo(expr3) not expr4 \\" and (expr5)'),
['+', 'expr1', '+', '\\"', '+', 'expr2', '+', 'foo(expr3)', '-', 'expr4', '+', '\\"', '+', 'expr5'])
def test_sqpp_paren_expr1_and_expr2_or_expr3_WORDS(self):
"""SearchQueryParenthesisedParser - (expr1 and expr2) or expr3"""
self.assertEqual(self.parser.parse_query('(expr1 and expr2) or expr3'),
['+', 'expr1 + expr2', '|', 'expr3'])
#['+', '+ expr1 | expr3', '+', '+ expr2 | expr3'])
def test_sqpp_paren_expr1_and_expr2_or_expr3_WORDS_equiv(self):
"""SearchQueryParenthesisedParser - (expr1 and expr2) or expr3 == (expr1 + expr2) | expr3"""
self.assertEqual(self.parser.parse_query('(expr1 and expr2) or expr3'),
self.parser.parse_query('(expr1 + expr2) | expr3'))
def test_sqpp_paren_expr1_and_expr2_or_expr3_WORDS_equiv_SYMBOLS(self):
"""SearchQueryParenthesisedParser - (expr1 and expr2) or expr3 == (expr1 + expr2) or expr3"""
self.assertEqual(self.parser.parse_query('(expr1 and expr2) or expr3'),
self.parser.parse_query('(expr1 + expr2) or expr3'))
def test_sqpp_double_quotes(self):
"""SearchQueryParenthesisedParser - Test double quotes"""
self.assertEqual(self.parser.parse_query(
'(expr1) - expr2 | "expressions - in + quotes | (are) not - parsed " - (expr3) | expr4'),
['+', 'expr1', '-', 'expr2', '|', '"expressions - in + quotes | (are) not - parsed "', '-', 'expr3', '|', 'expr4'])
#['+', '+ "expressions - in + quotes | (are) not - parsed " | expr1 | expr4',
# '+', '- expr3 | expr1 | expr4',
# '+', '+ "expressions - in + quotes | (are) not - parsed " - expr2 | expr4',
# '+', '- expr3 - expr2 | expr4'])
def test_sqpp_single_quotes(self):
"""SearchQueryParenthesisedParser - Test single quotes"""
self.assertEqual(self.parser.parse_query("(expr1) - expr2 | 'expressions - in + quotes | (are) not - parsed ' - (expr3) | expr4"),
['+', 'expr1', '-', 'expr2', '|', "'expressions - in + quotes | (are) not - parsed '", '-', 'expr3', '|', 'expr4'])
#['+', '+ \'expressions - in + quotes | (are) not - parsed \' | expr1 | expr4',
# '+', '- expr3 | expr1 | expr4',
# '+', '+ \'expressions - in + quotes | (are) not - parsed \' - expr2 | expr4',
# '+', '- expr3 - expr2 | expr4'])
def test_sqpp_escape_single_quotes(self):
"""SearchQueryParenthesisedParser - Test escaping single quotes"""
self.assertEqual(self.parser.parse_query("expr1 \\' expr2 +(expr3) -expr4 \\' + (expr5)"),
['+', 'expr1', '+', "\\'", '+', 'expr2', '+', 'expr3', '-', 'expr4', '+', "\\'", '+', 'expr5'])
def test_sqpp_escape_double_quotes(self):
"""SearchQueryParenthesisedParser - Test escaping double quotes"""
self.assertEqual(self.parser.parse_query('expr1 \\" expr2 +(expr3) -expr4 \\" + (expr5)'),
['+', 'expr1', '+', '\\"', '+', 'expr2', '+', 'expr3', '-', 'expr4', '+', '\\"', '+', 'expr5'])
def test_sqpp_beginning_double_quotes(self):
"""SearchQueryParenthesisedParser - Test parsing double quotes at beginning"""
self.assertEqual(self.parser.parse_query('"expr1" - (expr2)'),
['+', '"expr1"', '-', 'expr2'])
def test_sqpp_beginning_double_quotes_negated(self):
"""SearchQueryParenthesisedParser - Test parsing negated double quotes at beginning"""
self.assertEqual(self.parser.parse_query('-"expr1" - (expr2)'),
['-', '"expr1"', '-', 'expr2'])
def test_sqpp_long_or_chain(self):
"""SearchQueryParenthesisedParser - Test long or chains being parsed flat"""
self.assertEqual(self.parser.parse_query('p0 or p1 or p2 or p3 or p4'),
['+', 'p0', '|', 'p1', '|', 'p2', '|', 'p3', '|', 'p4'])
def test_sqpp_not_after_recursion(self):
"""SearchQueryParenthesisedParser - Test operations after recursive calls"""
self.assertEqual(self.parser.parse_query('(p0 or p1) not p2'),
['+', 'p0 | p1', '-', 'p2'])
#['+', '+ p0 | p1', '-', 'p2'])
def test_sqpp_oddly_capped_operators(self):
"""SearchQueryParenthesisedParser - Test conjunctions in any case"""
self.assertEqual(self.parser.parse_query('foo oR bar'),
['+', 'foo', '|', 'bar'])
def test_space_before_last_paren(self):
"""SearchQueryParenthesisedParser - Test (ellis )"""
self.assertEqual(self.parser.parse_query('(ellis )'),
['+', 'ellis'])
def test_sqpp_nested_U1_or_SL2(self):
"""SearchQueryParenthesisedParser - Test (U(1) or SL(2,Z))"""
self.assertEqual(self.parser.parse_query('(U(1) or SL(2,Z))'),
['+', 'u(1) | sl(2,z)'])
def test_sqpp_alternation_of_quote_marks_double(self):
"""SearchQueryParenthesisedParser - Test refersto:(author:"s parke" or author:ellis)"""
self.assertEqual(self.parser.parse_query('refersto:(author:"s parke" or author:ellis)'),
['+', 'refersto:\'author:"s parke" | author:ellis\''])
def test_sqpp_alternation_of_quote_marks_single(self):
"""SearchQueryParenthesisedParser - Test refersto:(author:'s parke' or author:ellis)"""
self.assertEqual(self.parser.parse_query('refersto:(author:\'s parke\' or author:ellis)'),
['+', 'refersto:"author:\'s parke\' | author:ellis"'])
def test_sqpp_alternation_of_quote_marks(self):
"""SearchQueryParenthesisedParser - Test refersto:(author:"s parke")"""
self.assertEqual(self.parser.parse_query('refersto:(author:"s parke")'),
['+', 'refersto:author:"s parke"'])
def test_sqpp_distributed_ands_equivalent(self):
"""SearchQueryParenthesisedParser - ellis and (kaluza-klein or r-parity) == ellis and (r-parity or kaluza-klein)"""
self.assertEqual(sorted(perform_request_search(p='ellis and (kaluza-klein or r-parity)')),
sorted(perform_request_search(p='ellis and (r-parity or kaluza-klein)')))
def test_sqpp_e_plus_e_minus(self):
"""SearchQueryParenthesisedParser - e(+)e(-)"""
self.assertEqual(self.parser.parse_query('e(+)e(-)'), ['+', 'e(+)e(-)'])
def test_sqpp_fe_2_plus(self):
"""SearchQueryParenthesisedParser - Fe(2+)"""
self.assertEqual(self.parser.parse_query('Fe(2+)'), ['+', 'fe(2+)'])
def test_sqpp_giant_evil_title_string(self):
"""SearchQueryParenthesisedParser - Measurements of CP-conserving trilinear gauge boson couplings WWV (V gamma, Z) in e(+)e(-) collisions at LEP2"""
self.assertEqual(self.parser.parse_query('Measurements of CP-conserving trilinear gauge boson couplings WWV (V gamma, Z) in e(+)e(-) collisions at LEP2'),
['+', 'measurements', '+', 'of', '+', 'cp-conserving', '+', 'trilinear', '+', 'gauge', \
'+', 'boson', '+', 'couplings', '+', 'wwv', '+', 'v + gamma, + z', \
'+', 'in', '+', 'e(+)e(-)', '+', 'collisions', '+', 'at', '+', 'lep2'])
def test_sqpp_second_order_operator_operates_on_parentheses(self):
"""SearchQueryParenthesisedParser - refersto:(author:ellis or author:hawking)"""
self.assertEqual(self.parser.parse_query('refersto:(author:ellis or author:hawking)'),
['+', 'refersto:"author:ellis | author:hawking"'])
class TestSpiresToInvenioSyntaxConverter(unittest.TestCase):
"""Test SPIRES query parsing and translation to Invenio syntax."""
def _compare_searches(self, invenio_syntax, spires_syntax):
"""Determine if two queries parse to the same search command.
For comparison of actual search results (regression testing), see the
tests in the Inspire module.
"""
parser = search_engine_query_parser.SearchQueryParenthesisedParser()
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
parsed_query = parser.parse_query(converter.convert_query(spires_syntax))
#parse_query removes any parens that convert_query added, but then
#we have to rejoin the list it returns and create basic searches
result_obtained = create_basic_search_units(
None,
' '.join(parsed_query).replace('+ ',''),
'',
None
)
# incase the desired result has parens
parsed_wanted = parser.parse_query(invenio_syntax)
result_wanted = create_basic_search_units(
None,
' '.join(parsed_wanted).replace('+ ',''),
'',
None)
assert result_obtained == result_wanted, \
"""SPIRES parsed as %s instead of %s""" % \
(repr(result_obtained), repr(result_wanted))
return
if CFG_WEBSEARCH_SPIRES_SYNTAX > 0:
def test_operators(self):
"""SPIRES search syntax - find a ellis and t shapes"""
invenio_search = "author:ellis and title:shapes"
spires_search = "find a ellis and t shapes"
self._compare_searches(invenio_search, spires_search)
def test_nots(self):
"""SPIRES search syntax - find a ellis and not t hadronic and not t collisions"""
invenio_search = "author:ellis and not title:hadronic and not title:collisions"
spires_search = "find a ellis and not t hadronic and not t collisions"
self._compare_searches(invenio_search, spires_search)
def test_author_simplest(self):
"""SPIRES search syntax - find a ellis"""
invenio_search = 'author:ellis'
spires_search = 'find a ellis'
self._compare_searches(invenio_search, spires_search)
def test_author_simple(self):
"""SPIRES search syntax - find a ellis, j"""
invenio_search = 'author:"ellis, j*"'
spires_search = 'find a ellis, j'
self._compare_searches(invenio_search, spires_search)
def test_exactauthor_simple(self):
"""SPIRES search syntax - find ea ellis, j"""
invenio_search = 'exactauthor:"ellis, j"'
spires_search = 'find ea ellis, j'
self._compare_searches(invenio_search, spires_search)
def test_author_reverse(self):
"""SPIRES search syntax - find a j ellis"""
invenio_search = 'author:"ellis, j*"'
spires_search = 'find a j ellis'
self._compare_searches(invenio_search, spires_search)
def test_author_initials(self):
"""SPIRES search syntax - find a a m polyakov"""
inv_search = 'author:"polyakov, a* m*"'
spi_search = 'find a a m polyakov'
self._compare_searches(inv_search, spi_search)
def test_author_many_initials(self):
"""SPIRES search syntax - find a p d q bach"""
inv_search = 'author:"bach, p* d* q*"'
spi_search = 'find a p d q bach'
self._compare_searches(inv_search, spi_search)
def test_author_many_lastnames(self):
"""SPIRES search syntax - find a alvarez gaume, j r r"""
inv_search = 'author:"alvarez gaume, j* r* r*"'
spi_search = 'find a alvarez gaume, j r r'
self._compare_searches(inv_search, spi_search)
def test_author_full_initial(self):
"""SPIRES search syntax - find a klebanov, ig.r."""
inv_search = 'author:"klebanov, ig* r*" or exactauthor:"klebanov, i r"'
spi_search = "find a klebanov, ig.r."
self._compare_searches(inv_search, spi_search)
def test_author_full_first(self):
"""SPIRES search syntax - find a ellis, john"""
invenio_search = 'author:"ellis, john*" or exactauthor:"ellis, j *" or exactauthor:"ellis, j" or exactauthor:"ellis, jo" or exactauthor:"ellis, joh" or author:"ellis, john, *"'
spires_search = 'find a ellis, john'
self._compare_searches(invenio_search, spires_search)
def test_combine_multiple(self):
"""SPIRES search syntax - find a gattringer, c and k symmetry chiral and not title chiral"""
inv_search = 'author:"gattringer, c*" keyword:chiral keyword:symmetry -title:chiral'
spi_search = "find a c gattringer and k chiral symmetry and not title chiral"
self._compare_searches(inv_search, spi_search)
def test_combine_multiple_or(self):
"""SPIRES search syntax - find a j ellis and (t report or k \"cross section\")"""
inv_search = 'author:"ellis, j*" and (title:report or keyword:"cross section")'
spi_search = 'find a j ellis and (t report or k "cross section")'
self._compare_searches(inv_search, spi_search)
def test_find_first_author(self):
"""SPIRES search syntax - find fa ellis"""
inv_search = 'firstauthor:ellis'
spi_search = 'find fa ellis'
self._compare_searches(inv_search, spi_search)
def test_find_first_author_initial(self):
"""SPIRES search syntax - find fa j ellis"""
inv_search = 'firstauthor:"ellis, j*"'
spi_search = 'find fa j ellis'
self._compare_searches(inv_search, spi_search)
def test_first_author_full_initial(self):
"""SPIRES search syntax - find fa klebanov, ig.r."""
inv_search = 'firstauthor:"klebanov, ig* r*" or exactfirstauthor:"klebanov, i r"'
spi_search = "find fa klebanov, ig.r."
self._compare_searches(inv_search, spi_search)
def test_citedby_author(self):
"""SPIRES search syntax - find citedby author doggy"""
inv_search = 'citedby:author:doggy'
spi_search = 'find citedby author doggy'
self._compare_searches(inv_search, spi_search)
def test_refersto_author(self):
"""SPIRES search syntax - find refersto author kitty"""
inv_search = 'refersto:author:kitty'
spi_search = 'find refersto author kitty'
self._compare_searches(inv_search, spi_search)
def test_refersto_author_multi_name(self):
"""SPIRES search syntax - find a ellis and refersto author \"parke, sj\""""
inv_search = 'author:ellis refersto:author:"parke, s. j."'
spi_search = 'find a ellis and refersto author "parke, s. j."'
self._compare_searches(inv_search, spi_search)
def test_refersto_author_multi_name_no_quotes(self):
"""SPIRES search syntax - find a ellis and refersto author parke, sj"""
inv_search = 'author:ellis refersto:(author:"parke, sj*" or exactauthor:"parke, s *" or exactauthor:"parke, s" or author:"parke, sj, *")'
spi_search = "find a ellis and refersto author parke, sj"
self._compare_searches(inv_search, spi_search)
def test_refersto_multi_word_no_quotes_no_index(self):
"""SPIRES search syntax - find refersto s parke"""
inv_search = 'refersto:"s parke"'
spi_search = 'find refersto s parke'
self._compare_searches(inv_search, spi_search)
def test_citedby_refersto_author(self):
"""SPIRES search syntax - find citedby refersto author penguin"""
inv_search = 'refersto:citedby:author:penguin'
spi_search = 'find refersto citedby author penguin'
self._compare_searches(inv_search, spi_search)
def test_irn_processing(self):
"""SPIRES search syntax - find irn 1360337 == find irn SPIRES-1360337"""
# Added for trac-130
with_spires = "fin irn SPIRES-1360337"
with_result = perform_request_search(p=with_spires)
without_spires = "fin irn 1360337"
without_result = perform_request_search(p=without_spires)
# We don't care if results are [], as long as they're the same
# Uncovered corner case: parsing could be broken and also happen to
# return [] twice. Unlikely though.
self.assertEqual(with_result, without_result)
def test_topcite(self):
"""SPIRES search syntax - find topcite 50+"""
inv_search = "cited:50->999999999"
spi_search = "find topcite 50+"
self._compare_searches(inv_search, spi_search)
def test_topcit(self):
"""SPIRES search syntax - find topcit 50+"""
inv_search = "cited:50->999999999"
spi_search = "find topcit 50+"
self._compare_searches(inv_search, spi_search)
def test_caption(self):
"""SPIRES search syntax - find caption muon"""
inv_search = "caption:muon"
spi_search = "find caption muon"
self._compare_searches(inv_search, spi_search)
def test_caption_multi_word(self):
"""SPIRES search syntax - find caption quark mass"""
inv_search = "caption:quark and caption:mass"
spi_search = "find caption quark mass"
self._compare_searches(inv_search, spi_search)
def test_quotes(self):
"""SPIRES search syntax - find t 'compton scattering' and a mele"""
inv_search = "title:'compton scattering' and author:mele"
spi_search = "find t 'compton scattering' and a mele"
self._compare_searches(inv_search, spi_search)
def test_equals_sign(self):
"""SPIRES search syntax - find a beacom and date = 2000"""
inv_search = "author:beacom year:2000"
spi_search = "find a beacom and date = 2000"
self._compare_searches(inv_search, spi_search)
def test_type_code(self):
"""SPIRES search syntax - find tc/ps/scl review"""
inv_search = "collection:review"
spi_search = "find tc review"
self._compare_searches(inv_search, spi_search)
inv_search = "collection:review"
spi_search = "find ps review"
self._compare_searches(inv_search, spi_search)
inv_search = "collection:review"
spi_search = "find scl review"
self._compare_searches(inv_search, spi_search)
def test_field_code(self):
"""SPIRES search syntax - f f p"""
inv_search = "subject:p"
spi_search = "f f p"
self._compare_searches(inv_search, spi_search)
def test_coden(self):
"""SPIRES search syntax - find coden aphys"""
inv_search = "journal:aphys"
spi_search = "find coden aphys"
self._compare_searches(inv_search, spi_search)
def test_job_title(self):
"""SPIRES search syntax - find job engineer not position programmer"""
inv_search = 'title:engineer not title:programmer'
spi_search = 'find job engineer not position programmer'
self._compare_searches(inv_search, spi_search)
def test_job_rank(self):
"""SPIRES search syntax - find rank Postdoc"""
inv_search = 'rank:Postdoc'
spi_search = 'find rank Postdoc'
self._compare_searches(inv_search, spi_search)
def test_job_region(self):
"""SPIRES search syntax - find region EU not continent Europe"""
inv_search = 'region:EU not region:Europe'
spi_search = 'find region EU not continent Europe'
self._compare_searches(inv_search, spi_search)
def test_fin_to_find_trans(self):
"""SPIRES search syntax - fin a ellis, j == find a ellis, j"""
fin_search = "fin a ellis, j"
fin_result = perform_request_search(p=fin_search)
find_search = "find a ellis, j"
find_result = perform_request_search(p=find_search)
# We don't care if results are [], as long as they're the same
# Uncovered corner case: parsing could be broken and also happen to
# return [] twice. Unlikely though.
self.assertEqual(fin_result, find_result)
def test_distribution_of_notted_search_terms(self):
"""SPIRES search syntax - find t this and not that ->title:this and not title:that"""
spi_search = "find t this and not that"
inv_search = "title:this and not title:that"
self._compare_searches(inv_search, spi_search)
def test_distribution_without_spacing(self):
"""SPIRES search syntax - find aff SLAC and Stanford ->affiliation:SLAC and affiliation:Stanford"""
# motivated by trac-187
spi_search = "find aff SLAC and Stanford"
inv_search = "affiliation:SLAC and affiliation:Stanford"
self._compare_searches(inv_search, spi_search)
def test_distribution_with_phrases(self):
"""SPIRES search syntax - find aff Penn State U -> affiliation:"Penn State U"""
# motivated by trac-517
spi_search = "find aff Penn State U"
inv_search = "affiliation:\"Penn State U\""
self._compare_searches(inv_search, spi_search)
def test_distribution_with_many_clauses(self):
"""SPIRES search syntax - find a mele and brooks and holtkamp and o'connell"""
spi_search = "find a mele and brooks and holtkamp and o'connell"
inv_search = "author:mele author:brooks author:holtkamp author:o'connell"
self._compare_searches(inv_search, spi_search)
def test_keyword_as_kw(self):
"""SPIRES search syntax - find kw something ->keyword:something"""
spi_search = "find kw meson"
inv_search = "keyword:meson"
self._compare_searches(inv_search, spi_search)
def test_recid(self):
"""SPIRES search syntax - find recid 11111"""
spi_search = 'find recid 111111'
inv_search = 'recid:111111'
self._compare_searches(inv_search, spi_search)
def test_desy_keyword_translation(self):
"""SPIRES search syntax - find dk "B --> pi pi" """
spi_search = "find dk \"B --> pi pi\""
inv_search = "695__a:\"B --> pi pi\""
self._compare_searches(inv_search, spi_search)
def test_journal_section_joining(self):
"""SPIRES search syntax - journal Phys.Lett, 0903, 024 -> journal:Phys.Lett,0903,024"""
spi_search = "find j Phys.Lett, 0903, 024"
inv_search = "journal:Phys.Lett,0903,024"
self._compare_searches(inv_search, spi_search)
def test_journal_search_with_colon(self):
"""SPIRES search syntax - find j physics 1:195 -> journal:physics,1,195"""
spi_search = "find j physics 1:195"
inv_search = "journal:physics,1,195"
self._compare_searches(inv_search, spi_search)
def test_journal_non_triple_syntax(self):
"""SPIRES search syntax - find j physics jcap"""
spi_search = "find j physics jcap"
inv_search = "journal:physics and journal:jcap"
self._compare_searches(inv_search, spi_search)
def test_journal_triple_with_many_spaces(self):
"""SPIRES search syntax - find j physics 0903 024"""
spi_search = 'find j physics 0903 024'
inv_search = 'journal:physics,0903,024'
self._compare_searches(inv_search, spi_search)
def test_distribution_of_search_terms(self):
"""SPIRES search syntax - find t this and that ->title:this and title:that"""
spi_search = "find t this and that"
inv_search = "title:this and title:that"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_search_patterns_alone(self):
"""SPIRES search syntax - simplest expansion"""
spi_search = "find t bob sam"
inv_search = "title:bob and title:sam"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_fulltext(self):
"""SPIRES search syntax - fulltext support"""
spi_search = "find ft The holographic RG is based on"
inv_search = "fulltext:The and fulltext:holographic and fulltext:RG and fulltext:is and fulltext:based and fulltext:on"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_fulltext_within_larger(self):
"""SPIRES search syntax - fulltext subsearch support"""
spi_search = "find au taylor and ft The holographic RG is based on and t brane"
inv_search = "author:taylor fulltext:The and fulltext:holographic and fulltext:RG and fulltext:is and fulltext:based and fulltext:on title:brane"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_search_patterns_conjoined(self):
"""SPIRES search syntax - simplest distribution"""
spi_search = "find t bob and sam"
inv_search = "title:bob and title:sam"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_search_patterns_multiple(self):
"""SPIRES search syntax - expansion (no distribution)"""
spi_search = "find t bob sam and k couch"
inv_search = "title:bob and title:sam and keyword:couch"
self._compare_searches(inv_search, spi_search)
def test_syntax_converter_expand_search_patterns_multiple_conjoined(self):
"""SPIRES search syntax - distribution and expansion"""
spi_search = "find t bob sam and couch"
inv_search = "title:bob and title:sam and title:couch"
self._compare_searches(inv_search, spi_search)
+ def test_date_invalid(self):
+ """SPIRES search syntax - searching an invalid date"""
+ spi_search = "find date foo"
+ inv_search = "year:foo"
+ self._compare_searches(inv_search, spi_search)
+
def test_date_by_yr(self):
"""SPIRES search syntax - searching by date year"""
spi_search = "find date 2002"
inv_search = "year:2002"
self._compare_searches(inv_search, spi_search)
def test_date_by_lt_yr(self):
"""SPIRES search syntax - searching by date < year"""
spi_search = "find date < 2002"
inv_search = 'year:0->2002'
self._compare_searches(inv_search, spi_search)
def test_date_by_gt_yr(self):
"""SPIRES search syntax - searching by date > year"""
spi_search = "find date > 1980"
inv_search = 'year:1980->9999'
self._compare_searches(inv_search, spi_search)
def test_date_by_yr_mo(self):
"""SPIRES search syntax - searching by date 1976-04"""
spi_search = "find date 1976-04"
inv_search = 'year:1976-04'
self._compare_searches(inv_search, spi_search)
def test_date_by_yr_mo_day_wholemonth_and_suffix(self):
"""SPIRES search syntax - searching by date 1976-04-01 and t dog"""
spi_search = "find date 1976-04-01 and t dog"
inv_search = 'year:1976-04-01 and title:dog'
self._compare_searches(inv_search, spi_search)
def test_date_by_yr_mo_day_and_suffix(self):
"""SPIRES search syntax - searching by date 1976-04-05 and t dog"""
spi_search = "find date 1976-04-05 and t dog"
inv_search = 'year:1976-04-05 and title:dog'
self._compare_searches(inv_search, spi_search)
def test_date_by_eq_yr_mo(self):
"""SPIRES search syntax - searching by date 1976-04"""
spi_search = "find date 1976-04"
inv_search = 'year:1976-04'
self._compare_searches(inv_search, spi_search)
def test_date_by_lt_yr_mo(self):
"""SPIRES search syntax - searching by date < 1978-10-21"""
spi_search = "find date < 1978-10-21"
inv_search = 'year:0->1978-10-21'
self._compare_searches(inv_search, spi_search)
def test_date_by_gt_yr_mo(self):
"""SPIRES search syntax - searching by date > 1978-10-21"""
spi_search = "find date > 1978-10-21"
inv_search = 'year:1978-10-21->9999'
self._compare_searches(inv_search, spi_search)
if DATEUTIL_AVAILABLE:
def test_date_2_digits_year_month_day(self):
"""SPIRES search syntax - searching by date > 78-10-21"""
spi_search = "find date 78-10-21"
inv_search = 'year:1978-10-21'
self._compare_searches(inv_search, spi_search)
if DATEUTIL_AVAILABLE:
def test_date_2_digits_year(self):
"""SPIRES search syntax - searching by date 78"""
spi_search = "find date 78"
inv_search = 'year:1978'
self._compare_searches(inv_search, spi_search)
def test_spires_syntax_trailing_colon(self):
"""SPIRES search syntax - test for blowup with trailing colon"""
spi_search = "find a watanabe:"
invenio_search = "author:watanabe:"
self._compare_searches(invenio_search, spi_search)
if DATEUTIL_AVAILABLE:
def test_date_by_lt_d_MO_yr(self):
"""SPIRES search syntax - searching by date < 23 Sep 2010: will only work with dateutil installed"""
spi_search = "find date < 23 Sep 2010"
inv_search = 'year:0->2010-09-23'
self._compare_searches(inv_search, spi_search)
def test_date_by_gt_d_MO_yr(self):
"""SPIRES search syntax - searching by date > 12 Jun 1960: will only work with dateutil installed"""
spi_search = "find date > 12 Jun 1960"
inv_search = 'year:1960-06-12->9999'
self._compare_searches(inv_search, spi_search)
def test_date_accept_today(self):
"""SPIRES search syntax - searching by today"""
spi_search = "find date today"
inv_search = "year:" + datetime.datetime.strftime(datetime.datetime.today(), '%Y-%m-%d')
self._compare_searches(inv_search, spi_search)
def test_date_accept_yesterday(self):
"""SPIRES search syntax - searching by yesterday"""
import dateutil.relativedelta
spi_search = "find date yesterday"
inv_search = "year:" + datetime.datetime.strftime(datetime.datetime.today()+dateutil.relativedelta.relativedelta(days=-1), '%Y-%m-%d')
self._compare_searches(inv_search, spi_search)
def test_date_accept_this_month(self):
"""SPIRES search syntax - searching by this month"""
spi_search = "find date this month"
inv_search = "year:" + datetime.datetime.strftime(datetime.datetime.today(), '%Y-%m')
self._compare_searches(inv_search, spi_search)
def test_date_accept_last_month(self):
"""SPIRES search syntax - searching by last month"""
spi_search = "find date last month"
inv_search = "year:" + datetime.datetime.strftime(datetime.datetime.today()\
+dateutil.relativedelta.relativedelta(months=-1), '%Y-%m')
self._compare_searches(inv_search, spi_search)
def test_date_accept_this_week(self):
"""SPIRES search syntax - searching by this week"""
spi_search = "find date this week"
begin = datetime.datetime.today()
days_to_remove = datetime.datetime.today().isoweekday() % 7
begin += du_delta(days=-days_to_remove)
begin_str = datetime.datetime.strftime(begin, '%Y-%m-%d')
# Only 6 days cause the last day is included in the search
end = datetime.datetime.today()
end_str = datetime.datetime.strftime(end, '%Y-%m-%d')
inv_search = "year:%s->%s" % (begin_str, end_str)
self._compare_searches(inv_search, spi_search)
def test_date_accept_last_week(self):
"""SPIRES search syntax - searching by last week"""
spi_search = "find date last week"
begin = datetime.datetime.today()
days_to_remove = 7 + datetime.datetime.today().isoweekday() % 7
begin += du_delta(days=-days_to_remove)
begin_str = datetime.datetime.strftime(begin, '%Y-%m-%d')
# Only 6 days cause the last day is included in the search
end = begin + du_delta(days=6)
end_str = datetime.datetime.strftime(end, '%Y-%m-%d')
inv_search = "year:%s->%s" % (begin_str, end_str)
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_minus_days(self):
"""SPIRES search syntax - searching by 2011-01-03 - 2"""
spi_search = "find date 2011-01-03 - 2"
inv_search = "year:2011-01-01"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_minus_days_with_month_wrap(self):
"""SPIRES search syntax - searching by 2011-03-01 - 1"""
spi_search = "find date 2011-03-01 - 1"
inv_search = "year:2011-02-28"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_minus_days_with_year_wrap(self):
"""SPIRES search syntax - searching by 2011-01-01 - 1"""
spi_search = "find date 2011-01-01 - 1"
inv_search = "year:2010-12-31"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_minus_days_with_leapyear_february(self):
"""SPIRES search syntax - searching by 2008-03-01 - 1"""
spi_search = "find date 2008-03-01 - 1"
inv_search = "year:2008-02-29"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_minus_many_days(self):
"""SPIRES search syntax - searching by 2011-02-24 - 946"""
spi_search = "find date 2011-02-24 - 946"
inv_search = "year:2008-07-23"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_plus_days(self):
"""SPIRES search syntax - searching by 2011-01-03 + 2"""
spi_search = "find date 2011-01-01 + 2"
inv_search = "year:2011-01-03"
self._compare_searches(inv_search, spi_search)
def test_date_accept_plus_days_with_month_wrap(self):
"""SPIRES search syntax - searching by 2011-03-31 + 2"""
spi_search = "find date 2011-03-31 + 2"
inv_search = "year:2011-04-02"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_plus_days_with_year_wrap(self):
"""SPIRES search syntax - searching by 2011-12-31 + 1"""
spi_search = "find date 2011-12-31 + 1"
inv_search = "year:2012-01-01"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_plus_days_with_leapyear_february(self):
"""SPIRES search syntax - searching by 2008-02-29 + 2"""
spi_search = "find date 2008-02-28 + 2"
inv_search = "year:2008-03-01"
self._compare_searches(inv_search, spi_search)
def test_date_accept_date_plus_many_days(self):
"""SPIRES search syntax - searching by 2011-02-24 + 666"""
spi_search = "find date 2011-02-24 + 666"
inv_search = "year:2012-12-21"
self._compare_searches(inv_search, spi_search)
def test_spires_syntax_detected_f(self):
"""SPIRES search syntax - test detection f t p"""
# trac #261
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("f t p")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_fin(self):
"""SPIRES search syntax - test detection fin t p"""
# trac #261
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("fin t p")
self.assertEqual(spi_search, True)
def test_spires_keyword_distribution_before_conjunctions(self):
"""SPIRES search syntax - test find journal phys.lett. 0903 024"""
spi_search = 'find journal phys.lett. 0903 024'
inv_search = '(journal:phys.lett.,0903,024)'
self._compare_searches(inv_search, spi_search)
def test_spires_keyword_distribution_with_parens(self):
"""SPIRES search syntax - test find cn d0 and (a abachi or abbott or abazov)"""
spi_search = "find cn d0 and (a abachi or abbott or abazov)"
inv_search = "collaboration:d0 and (author:abachi or author:abbott or author:abazov)"
self._compare_searches(inv_search, spi_search)
def test_super_short_author_name(self):
"""SPIRES search syntax - test fin a er and cn cms"""
spi_search = "fin a er and cn cms"
inv_search = "author:er collaboration:cms"
self._compare_searches(inv_search, spi_search)
def test_simple_syntax_mixing(self):
"""SPIRES and invenio search syntax - find a ellis and citedby:hawking"""
combo_search = "find a ellis and citedby:hawking"
inv_search = "author:ellis citedby:hawking"
self._compare_searches(inv_search, combo_search)
def test_author_first_syntax_mixing(self):
"""SPIRES and invenio search syntax - find a dixon, l.j. cited:10->52"""
combo_search = 'find a dixon, l.j. cited:10->52'
inv_search = 'author:"dixon, l* j*" cited:10->52'
self._compare_searches(inv_search, combo_search)
def test_minus_boolean_syntax_mixing(self):
"""SPIRES and invenio search syntax - find a ellis -title:muon"""
combo_search = 'find a ellis -title:muon'
inv_search = 'author:ellis -title:muon'
self._compare_searches(inv_search, combo_search)
def test_plus_boolean_syntax_mixing(self):
"""SPIRES and invenio search syntax - find a ellis +title:muon"""
combo_search = 'find a ellis +title:muon'
inv_search = 'author:ellis title:muon'
self._compare_searches(inv_search, combo_search)
def test_second_level_syntax_mixing(self):
"""SPIRES and invenio search syntax - find a ellis refersto:author:hawking"""
combo_search = 'find a ellis refersto:author:hawking'
inv_search = 'author:ellis refersto:author:hawking'
self._compare_searches(inv_search, combo_search)
if CFG_WEBSEARCH_SPIRES_SYNTAX > 1:
def test_absorbs_naked_a_search(self):
"""SPIRES search syntax - a ellis"""
invenio_search = "author:ellis"
naked_search = "a ellis"
self._compare_searches(invenio_search, naked_search)
def test_absorbs_naked_author_search(self):
"""SPIRES search syntax - author ellis"""
invenio_search = "author:ellis"
spi_search = "author ellis"
self._compare_searches(invenio_search, spi_search)
def test_spires_syntax_detected_naked_a(self):
"""SPIRES search syntax - test detection a ellis"""
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("a ellis")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_naked_author(self):
"""SPIRES search syntax - test detection author ellis"""
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("author ellis")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_naked_author_leading_spaces(self):
"""SPIRES search syntax - test detection author ellis"""
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable(" author ellis")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_naked_title(self):
"""SPIRES search syntax - test detection t muon"""
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("t muon")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_second_keyword(self):
"""SPIRES search syntax - test detection author:ellis and t muon"""
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
spi_search = converter.is_applicable("author:ellis and t muon")
self.assertEqual(spi_search, True)
def test_spires_syntax_detected_invenio(self):
"""SPIRES search syntax - test detection Not SPIRES"""
# trac #261
converter = search_engine_query_parser.SpiresToInvenioSyntaxConverter()
inv_search = converter.is_applicable("t:p a:c")
self.assertEqual(inv_search, False)
def test_invenio_syntax_only_second_level(self):
"""invenio search syntax - citedby:reportnumber:hep-th/0205061"""
inv_search = 'citedby:reportnumber:hep-th/0205061'
self._compare_searches(inv_search, inv_search)
def test_invenio_syntax_only_boolean(self):
"""invenio search syntax - author:ellis and not title:hadronic and not title:collisions"""
inv_search = "author:ellis and not title:hadronic and not title:collisions"
self._compare_searches(inv_search, inv_search)
TEST_SUITE = make_test_suite(TestSearchQueryParenthesisedParser,
TestSpiresToInvenioSyntaxConverter,
TestParserUtilityFunctions)
if __name__ == "__main__":
run_test_suite(TEST_SUITE)
#run_test_suite(make_test_suite(TestParserUtilityFunctions, TestSearchQueryParenthesisedParser)) # DEBUG