diff --git a/modules/bibrank/lib/bibrank_citerank_indexer.py b/modules/bibrank/lib/bibrank_citerank_indexer.py
index 8d96e42d7..f80e7bf2e 100644
--- a/modules/bibrank/lib/bibrank_citerank_indexer.py
+++ b/modules/bibrank/lib/bibrank_citerank_indexer.py
@@ -1,785 +1,785 @@
# -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Implementation of different ranking methods based on
the citation graph:
- citation count/ time decayed citation count
- pagerank / pagerank with external citations
- time decayed pagerank
"""
# pylint: disable-msg=E0611
import marshal
import ConfigParser
from math import exp
import datetime
import time
import re
import sys
from numpy import array, ones, zeros, int32, float32, sqrt, dot
from zlib import decompress
if sys.hexversion < 0x2040000:
# pylint: disable-msg=W0622
from sets import Set as set
# pylint: enable-msg=W0622
from invenio.dbquery import run_sql, serialize_via_marshal
from invenio.bibtask import write_message
from invenio.config import CFG_ETCDIR
def get_citations_from_file(filename):
"""gets the citation data (who cites who) from a file and returns
- a dictionary of type x:{x1,x2..},
where x is cited by x1,x2..
- a dictionary of type a:{b},
where recid 'a' is asociated with an index 'b' """
cit = {}
dict_of_ids = {}
count = 0
try:
citation_file = open(filename,"r")
except StandardError:
write_message("Cannot find file: %s" % filename, sys.stderr)
raise StandardError
for line in citation_file:
tokens = line.strip().split()
recid_cites = int(tokens[0])
recid_cited = int(tokens[1])
if recid_cited not in cit:
cit[recid_cited] = []
#without this, duplicates might be introduced
if recid_cites not in cit[recid_cited] and recid_cites != recid_cited:
cit[recid_cited].append(recid_cites)
if recid_cites not in dict_of_ids:
dict_of_ids[recid_cites] = count
count += 1
if recid_cited not in dict_of_ids:
dict_of_ids[recid_cited] = count
count += 1
citation_file.close()
write_message("Citation data collected from file: %s" %filename, verbose=2)
write_message("Ids and recids corespondace: %s" \
%str(dict_of_ids), verbose=9)
write_message("Citations: %s" % str(cit), verbose=9)
return cit, dict_of_ids
def get_citations_from_db():
"""gets the citation data (who cites who) from the rnkCITATIONDATA table,
and returns:
-a dictionary of type x:{x1,x2..}, where x is cited by x1,x2..
-a dict of type a:{b} where recid 'a' is asociated with an index 'b'"""
dict_of_ids = {}
count = 0
query = "select object_value from rnkCITATIONDATA \
where object_name = 'citationdict'"
cit_compressed = run_sql(query)
cit = []
if cit_compressed and cit_compressed[0] and cit_compressed[0][0]:
cit = marshal.loads(decompress(cit_compressed[0][0]))
if cit:
for item in cit:
#check for duplicates in citation dictionary
cit[item] = set(cit[item])
if item in cit[item]:
cit[item].remove(item)
if item not in dict_of_ids:
dict_of_ids[item] = count
count += 1
for value in cit[item]:
if value not in dict_of_ids:
dict_of_ids[value] = count
count += 1
write_message ("Citation data collected\
from rnkCITATIONDATA", verbose=2)
write_message("Ids and recids corespondace: %s" \
% str(dict_of_ids), verbose=9)
write_message("Citations: %s" % str(cit), verbose=9)
return cit, dict_of_ids
else:
write_message("Error while extracting citation data \
from rnkCITATIONDATA table", verbose=1)
else:
write_message("Error while extracting citation data \
from rnkCITATIONDATA table", verbose=1)
return {}, {}
def construct_ref_array (cit, dict_of_ids, len_):
"""returns an array with the number of references that each recid has """
ref = array((), int32)
ref = zeros(len_, int32)
for key in cit:
for value in cit[key]:
ref[dict_of_ids[value]] += 1
write_message("Number of references: %s" %str(ref), verbose=9)
write_message("Finished computing total number \
of references for each paper.", verbose=5)
return ref
def get_external_links_from_file(filename, ref, dict_of_ids):
"""returns a dictionary containing the number of
external links for each recid
external link=citation that is not in our database """
ext_links = {}
#format: ext_links[dict_of_ids[recid]]=number of total external links
try:
external_file = open(filename,"r")
except StandardError:
write_message("Cannot find file: %s" % filename, sys.stderr)
raise StandardError
for line in external_file:
tokens = line.strip().split()
recid = int(tokens[0])
nr_of_external = int(tokens[1])
ext_links[dict_of_ids[recid]] = nr_of_external - ref[dict_of_ids[recid]]
if ext_links[dict_of_ids[recid]] < 0:
ext_links[dict_of_ids[recid]] = 0
external_file.close()
write_message("External link information extracted", verbose=2)
return ext_links
def get_external_links_from_db(ref, dict_of_ids, reference_indicator):
"""returns a dictionary containing the number of
external links for each recid
external link=citation that is not in our database """
ext_links = {}
reference_tag_regex = reference_indicator + "[a-z]"
for recid in dict_of_ids:
query = "select COUNT(DISTINCT field_number) from bibrec_bib99x \
where id_bibrec='%s' and id_bibxxx in \
(select id from bib99x where tag RLIKE '%s');" \
% (str(recid), reference_tag_regex)
result_set = run_sql(query)
if result_set:
total_links = int(result_set[0][0])
internal_links = ref[dict_of_ids[recid]]
ext_links[dict_of_ids[recid]] = total_links - internal_links
if ext_links[dict_of_ids[recid]] < 0:
ext_links[dict_of_ids[recid]] = 0
else:
ext_links[dict_of_ids[recid]] = 0
write_message ("External link information extracted", verbose=2)
write_message("External links: %s" % str(ext_links), verbose=9)
return ext_links
def avg_ext_links_with_0(ext_links):
"""returns the average number of external links per paper
including in the counting the papers with 0 external links"""
total = 0.0
for item in ext_links:
total += ext_links[item]
avg_ext = total/len(ext_links)
write_message("The average number of external links per paper (including \
papers with 0 external links) is: %s" % str(avg_ext), verbose=3)
return avg_ext
def avg_ext_links_without_0(ext_links):
"""returns the average number of external links per paper
excluding in the counting the papers with 0 external links"""
count = 0.0
total = 0.0
for item in ext_links:
if ext_links[item] != 0:
count += 1
total += ext_links[item]
avg_ext = total/count
write_message ("The average number of external links per paper (excluding \
papers with 0 external links) is: %s" % str(avg_ext), verbose=3)
return avg_ext
def leaves(ref):
"""returns the number of papers that do not cite any other paper"""
nr_of_leaves = 0
for i in ref:
if i == 0:
nr_of_leaves += 1
write_message ("The number of papers that do not cite \
any other papers: %s" % str(leaves), verbose=3)
return nr_of_leaves
def get_dates_from_file(filename, dict_of_ids):
"""Returns the year of the publication for each paper.
In case the year is not in the db, the year of the submission is taken"""
dates = {}
# the format is: dates[dict_of_ids[recid]] = year
try:
dates_file = open(filename,"r")
except StandardError:
write_message("Cannot find file: %s" % filename, sys.stderr)
raise StandardError
for line in dates_file:
tokens = line.strip().split()
recid = int(tokens[0])
year = int(tokens[1])
dates[dict_of_ids[recid]] = year
dates_file.close()
write_message("Dates extracted", verbose=2)
write_message("Dates dictionary %s" % str(dates), verbose=9)
return dates
def get_dates_from_db(dict_of_ids):
"""Returns the year of the publication for each paper.
In case the year is not in the db, the year of the submission is taken"""
current_year = int(datetime.datetime.now().strftime("%Y"))
total = 0
count = 0
dict_of_dates = {}
for recid in dict_of_ids:
dict_of_dates[recid] = 0
- query1 = "select * from bib26x where tag='260__c';"
+ query1 = "select id,tag,value from bib26x where tag='260__c';"
date_list = run_sql(query1)
date_dict = {}
for item in date_list:
date_dict[int(item[0])] = item[2]
pattern = re.compile('.*(\d{4}).*')
- query2 = "select * from bibrec_bib26x;"
+ query2 = "select id_bibrec,id_bibxxx,field_number from bibrec_bib26x;"
date_list = run_sql(query2)
for item in date_list:
recid = int(item[0])
id_ = int(item[1])
if id_ in date_dict and recid in dict_of_dates:
reg = pattern.match(date_dict[id_])
if reg:
date = int(reg.group(1))
if date > 1000 and date <= current_year:
dict_of_dates[recid] = date
total += date
count += 1
not_covered = []
for recid in dict_of_dates:
if dict_of_dates[recid] == 0:
not_covered.append(recid)
- query3 = "select * from bib96x where tag='961__x';"
+ query3 = "select id,tag,value from bib96x where tag='961__x';"
date_list = run_sql(query3)
date_dict = {}
for item in date_list:
date_dict[int(item[0])] = item[2]
- query4 = "select * from bibrec_bib96x;"
+ query4 = "select id_bibrec,id_bibxxx,field_number from bibrec_bib96x;"
date_list = run_sql(query4)
for item in date_list:
recid = int(item[0])
id_ = int(item[1])
if id_ in date_dict and recid in not_covered:
date = int(str(date_dict[id_])[0:4])
if date > 1000 and date <= current_year:
dict_of_dates[recid] = date
total += date
count += 1
dates = {}
med = total/count
for recid in dict_of_dates:
if dict_of_dates[recid] == 0:
dates[dict_of_ids[recid]] = med
else:
dates[dict_of_ids[recid]] = dict_of_dates[recid]
write_message("Dates extracted", verbose=2)
write_message("Dates dictionary %s" % str(dates), verbose=9)
return dates
def construct_sparse_matrix(cit, ref, dict_of_ids, len_, damping_factor):
"""returns several structures needed in the calculation
of the PAGERANK method using this structures, we don't need
to keep the full matrix in the memory"""
sparse = {}
for item in cit:
for value in cit[item]:
sparse[(dict_of_ids[item], dict_of_ids[value])] = \
damping_factor * 1.0/ref[dict_of_ids[value]]
semi_sparse = []
for j in range(len_):
if ref[j] == 0:
semi_sparse.append(j)
semi_sparse_coeficient = damping_factor/len_
#zero_coeficient = (1-damping_factor)/len_
write_message("Sparse information calculated", verbose=3)
return sparse, semi_sparse, semi_sparse_coeficient
def construct_sparse_matrix_ext(cit, ref, ext_links, dict_of_ids, alpha, beta):
"""if x doesn't cite anyone: cites everyone : 1/len_ -- should be used!
returns several structures needed in the calculation
of the PAGERANK_EXT method"""
len_ = len(dict_of_ids)
sparse = {}
semi_sparse = {}
for i in range(len_):
sparse[i+1, 0] = alpha/(len_)
sparse[0, 0] = 1.0 - alpha
for j in range(len_):
if j not in ext_links:
sparse[0, j+1] = beta/(len_ + beta)
else:
if ext_links[j] == 0:
sparse[0, j+1] = beta/(len_ + beta)
else:
aux = beta * ext_links[j]
if ref[j] == 0:
sparse[0, j+1] = aux/(aux + len_)
else:
sparse[0, j+1] = aux/(aux + ref[j])
if ref[j] == 0:
semi_sparse[j+1] = (1.0 - sparse[0, j + 1])/len_
for item in cit:
for value in cit[item]:
sparse[(dict_of_ids[item] + 1, dict_of_ids[value] + 1)] = \
(1.0 - sparse[0, dict_of_ids[value] + 1])/ref[dict_of_ids[value]]
#for i in range(len_ + 1):
# a = ""
# for j in range (len_ + 1):
# if (i,j) in sparse:
# a += str(sparse[(i,j)]) + "\t"
# else:
# a += "0\t"
# print a
#print semi_sparse
write_message("Sparse information calculated", verbose=3)
return sparse, semi_sparse
def construct_sparse_matrix_time(cit, ref, dict_of_ids, \
damping_factor, date_coef):
"""returns several structures needed in the calculation of the PAGERANK_time
method using this structures,
we don't need to keep the full matrix in the memory"""
len_ = len(dict_of_ids)
sparse = {}
for item in cit:
for value in cit[item]:
sparse[(dict_of_ids[item], dict_of_ids[value])] = damping_factor * \
date_coef[dict_of_ids[value]]/ref[dict_of_ids[value]]
semi_sparse = []
for j in range(len_):
if ref[j] == 0:
semi_sparse.append(j)
semi_sparse_coeficient = damping_factor/len_
#zero_coeficient = (1-damping_factor)/len_
write_message("Sparse information calculated", verbose=3)
return sparse, semi_sparse, semi_sparse_coeficient
def statistics_on_sparse(sparse):
"""returns the number of papers that cite themselves"""
count_diag = 0
for (i, j) in sparse.keys():
if i == j:
count_diag += 1
write_message("The number of papers that cite themselves: %s" % \
str(count_diag), verbose=3)
return count_diag
def pagerank(conv_threshold, check_point, len_, sparse, \
semi_sparse, semi_sparse_coef):
"""the core function of the PAGERANK method
returns an array with the ranks coresponding to each recid"""
weights_old = array((), float32)
weights_old = ones((len_), float32) # initial weights
weights_new = array((), float32)
converged = False
nr_of_check_points = 0
difference = len_
while not converged:
nr_of_check_points += 1
for step in (range(check_point)):
weights_new = zeros((len_), float32)
for (i, j) in sparse.keys():
weights_new[i] += sparse[(i, j)]*weights_old[j]
semi_total = 0.0
for j in semi_sparse:
semi_total += weights_old[j]
weights_new = weights_new + semi_sparse_coef * semi_total + \
(1.0/len_ - semi_sparse_coef) * sum(weights_old)
if step == check_point - 1:
diff = weights_new - weights_old
difference = sqrt(dot(diff, diff))/len_
write_message( "Finished step: %s, %s " \
%(str(check_point*(nr_of_check_points-1) + step), \
str(difference)), verbose=5)
weights_old = weights_new.copy()
converged = (difference < conv_threshold)
write_message("PageRank calculated for all recids finnished in %s steps. \
The threshold was %s" % (str(nr_of_check_points), str(difference)),\
verbose=2)
return weights_old
def pagerank_ext( conv_threshold, check_point, len_, sparse, semi_sparse):
"""the core function of the PAGERANK_EXT method
returns an array with the ranks coresponding to each recid"""
weights_old = array((), float32)
weights_old = ones((len_), float32)
weights_new = array((), float32)
converged = False
nr_of_check_points = 0
difference = len_
while not converged:
nr_of_check_points += 1
for step in (range(check_point)):
weights_new = zeros((len_), float32)
for (i, j) in sparse.keys():
weights_new[i] += sparse[(i, j)]*weights_old[j]
total_sum = 0.0
for j in semi_sparse:
total_sum += semi_sparse[j]*weights_old[j]
weights_new[1:len_] = weights_new[1:len_] + total_sum
if step == check_point - 1:
diff = weights_new - weights_old
difference = sqrt(dot(diff, diff))/len_
write_message( "Finished step: %s, %s " \
% (str(check_point*(nr_of_check_points-1) + step), \
str(difference)), verbose=5)
weights_old = weights_new.copy()
converged = (difference < conv_threshold)
write_message("PageRank calculated for all recids finnished in %s steps. \
The threshold was %s" % (str(nr_of_check_points), \
str(difference)), verbose=2)
#return weights_old[1:len_]/(len_ - weights_old[0])
return weights_old[1:len_]
def pagerank_time(conv_threshold, check_point, len_, \
sparse, semi_sparse, semi_sparse_coeficient, date_coef):
"""the core function of the PAGERANK_TIME method: pageRank + time decay
returns an array with the ranks coresponding to each recid"""
weights_old = array((), float32)
weights_old = ones((len_), float32) # initial weights
weights_new = array((), float32)
converged = False
nr_of_check_points = 0
difference = len_
while not converged:
nr_of_check_points += 1
for step in (range(check_point)):
weights_new = zeros((len_), float32)
for (i, j) in sparse.keys():
weights_new[i] += sparse[(i, j)]*weights_old[j]
semi_total = 0.0
for j in semi_sparse:
semi_total += weights_old[j]*date_coef[j]
zero_total = 0.0
for i in range(len_):
zero_total += weights_old[i]*date_coef[i]
#dates = array(date_coef.keys())
#zero_total = dot(weights_old, dates)
weights_new = weights_new + semi_sparse_coeficient * semi_total + \
(1.0/len_ - semi_sparse_coeficient) * zero_total
if step == check_point - 1:
diff = weights_new - weights_old
difference = sqrt(dot(diff, diff))/len_
write_message( "Finished step: %s, %s " \
% (str(check_point*(nr_of_check_points-1) + step), \
str(difference)), verbose=5)
weights_old = weights_new.copy()
converged = (difference < conv_threshold)
write_message("PageRank calculated for all recids finnished in %s steps.\
The threshold was %s" % (str(nr_of_check_points), \
str(difference)), verbose=2)
return weights_old
def citation_rank_time(cit, dict_of_ids, date_coef, dates, decimals):
"""returns a dictionary recid:weight based on the total number of
citations as function of time"""
dict_of_ranks = {}
for key in dict_of_ids:
if key in cit:
dict_of_ranks[key] = 0
for recid in cit[key]:
dict_of_ranks[key] += date_coef[dict_of_ids[recid]]
dict_of_ranks[key] = round(dict_of_ranks[key], decimals) \
+ dates[dict_of_ids[key]]* pow(10, 0-4-decimals)
else:
dict_of_ranks[key] = dates[dict_of_ids[key]]* pow(10, 0-4-decimals)
write_message("Citation rank calculated", verbose=2)
return dict_of_ranks
def get_ranks(weights, dict_of_ids, mult, dates, decimals):
"""returns a dictionary recid:value, where value is the weight of the
recid paper; the second order is the reverse time order,
from recent to past"""
dict_of_ranks = {}
for item in dict_of_ids:
dict_of_ranks[item] = round(weights[dict_of_ids[item]]* mult, decimals) \
+ dates[dict_of_ids[item]]* pow(10, 0-4-decimals)
#dict_of_ranks[item] = weights[dict_of_ids[item]]
return dict_of_ranks
def sort_weights(dict_of_ranks):
"""sorts the recids based on weights(first order)
and on dates(second order)"""
ranks_by_citations = sorted(dict_of_ranks.keys(), lambda x, y: \
cmp(dict_of_ranks[y], dict_of_ranks[x]))
return ranks_by_citations
def write_first_ranks_to_file(ranks_by_citations, dict_of_ranks, \
nr_of_ranks, filename):
"""Writes the first n results of the ranking method into a file"""
try:
ranks_file = open(filename,"w")
except StandardError:
write_message("Problems with file: %s" % filename, sys.stderr)
raise StandardError
for i in range(nr_of_ranks):
ranks_file.write(str(i+1) + "\t" + str(ranks_by_citations[i]) + \
"\t" + str(dict_of_ranks[ranks_by_citations[i]]) + "\n")
ranks_file.close()
write_message("The first %s pairs recid:rank in the ranking order \
are written into this file: %s" % (nr_of_ranks, filename), verbose=2)
def del_rank_method_data(rank_method_code):
"""Delete the data for a rank method from rnkMETHODDATA table"""
id_ = run_sql("SELECT id from rnkMETHOD where name=%s", (rank_method_code,))
run_sql("DELETE FROM rnkMETHODDATA WHERE id_rnkMETHOD=%s", (id_[0][0], ))
def into_db(dict_of_ranks, rank_method_code):
"""Writes into the rnkMETHODDATA table the ranking results"""
method_id = run_sql("SELECT id from rnkMETHOD where name=%s", \
(rank_method_code, ))
del_rank_method_data(rank_method_code)
serialized_data = serialize_via_marshal(dict_of_ranks)
method_id_str = str(method_id[0][0])
run_sql("INSERT INTO rnkMETHODDATA(id_rnkMETHOD, relevance_data) \
VALUES (%s,%s)",(method_id_str, serialized_data,))
date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
run_sql("UPDATE rnkMETHOD SET last_updated=%s WHERE name=%s", \
(date, rank_method_code))
write_message("Finished writing the ranks into rnkMETHOD table", verbose=5)
def run_pagerank(cit, dict_of_ids, len_, ref, damping_factor, \
conv_threshold, check_point, dates):
"""returns the final form of the ranks when using pagerank method"""
write_message("Running the PageRank method", verbose=5)
sparse, semi_sparse, semi_sparse_coeficient = \
construct_sparse_matrix(cit, ref, dict_of_ids, len_, damping_factor)
weights = pagerank(conv_threshold, check_point, len_, \
sparse, semi_sparse, semi_sparse_coeficient)
dict_of_ranks = get_ranks(weights, dict_of_ids, 1, dates, 2)
return dict_of_ranks
def run_pagerank_ext(cit, dict_of_ids, ref, ext_links, \
conv_threshold, check_point, alpha, beta, dates):
"""returns the final form of the ranks when using pagerank_ext method"""
write_message( "Running the PageRank with external links method", verbose=5)
len_ = len(dict_of_ids)
sparse, semi_sparse = construct_sparse_matrix_ext(cit, ref, \
ext_links, dict_of_ids, alpha, beta)
weights = pagerank_ext(conv_threshold, check_point, \
len_ + 1, sparse, semi_sparse)
dict_of_ranks = get_ranks(weights, dict_of_ids, 1, dates, 2)
return dict_of_ranks
def run_pagerank_time(cit, dict_of_ids, len_, ref, damping_factor, \
conv_threshold, check_point, date_coef, dates):
"""returns the final form of the ranks when using
pagerank + time decay method"""
write_message("Running the PageRank_time method", verbose=5)
sparse, semi_sparse, semi_sparse_coeficient = \
construct_sparse_matrix_time(cit, ref, dict_of_ids, \
damping_factor, date_coef)
weights = pagerank_time(conv_threshold, check_point, len_, \
sparse, semi_sparse, semi_sparse_coeficient, date_coef)
dict_of_ranks = get_ranks(weights, dict_of_ids, 100000, dates, 2)
return dict_of_ranks
def run_citation_rank_time(cit, dict_of_ids, date_coef, dates):
"""returns the final form of the ranks when using citation count
as function of time method"""
write_message("Running the citation rank with time decay method", verbose=5)
dict_of_ranks = citation_rank_time(cit, dict_of_ids, date_coef, dates, 2)
return dict_of_ranks
def spearman_rank_correlation_coef(rank1, rank2, len_):
"""rank1 and rank2 are arrays containing the recids in the ranking order
returns the corelation coeficient (-1 <= c <= 1) between 2 rankings
the closec c is to 1, the more correlated are the two ranking methods"""
total = 0
for i in range(len_):
rank_value = rank2.index(rank1[i])
total += ( i - rank_value)*( i - rank_value)
return 1 - (6.0 * total) / (len_*(len_*len_ - 1))
def remove_loops(cit, dates, dict_of_ids):
"""when using time decay, new papers that are part of a loop
are accumulating a lot of fake weight"""
new_cit = {}
for recid in cit:
new_cit[recid] = []
for cited_by in cit[recid]:
if dates[dict_of_ids[cited_by]] >= dates[dict_of_ids[recid]]:
if cited_by in cit:
if recid not in cit[cited_by]:
new_cit[recid].append(cited_by)
else:
write_message("Loop removed: %s <-> %s" \
%(cited_by, recid), verbose=9)
else:
new_cit[recid].append(cited_by)
else:
write_message("Loop removed: %s <-> %s" \
%(cited_by, recid), verbose=9)
write_message("Simple loops removed", verbose=5)
return new_cit
def calculate_time_weights(len_, time_decay, dates):
"""calculates the time coeficients for each paper"""
current_year = int(datetime.datetime.now().strftime("%Y"))
date_coef = {}
for j in range(len_):
date_coef[j] = exp(time_decay*(dates[j] - current_year))
write_message("Time weights calculated", verbose=5)
write_message("Time weights: %s" % str(date_coef), verbose=9)
return date_coef
def get_dates(function, config, dict_of_ids):
"""returns a dictionary containing the year of
publishing for each paper"""
try:
file_for_dates = config.get(function, "file_with_dates")
dates = get_dates_from_file(file_for_dates, dict_of_ids)
except (ConfigParser.NoOptionError, StandardError), err:
write_message("If you want to read the dates from file set up the \
'file_for_dates' variable in the config file [%s]" %err, verbose=3)
dates = get_dates_from_db(dict_of_ids)
return dates
def citerank(rank_method_code):
"""new ranking method based on the citation graph"""
write_message("Running rank method: %s" % rank_method_code, verbose=0)
try:
file_ = CFG_ETCDIR + "/bibrank/" + rank_method_code + ".cfg"
config = ConfigParser.ConfigParser()
config.readfp(open(file_))
except StandardError:
write_message("Cannot find configuration file: %s" % file_, sys.stderr)
raise StandardError
# the file for citations needs to have the following format:
#each line needs to be x[tab]y, where x cites y; x,y are recids
function = config.get("rank_method", "function")
try:
file_for_citations = config.get(function, "file_with_citations")
cit, dict_of_ids = get_citations_from_file(file_for_citations)
except (ConfigParser.NoOptionError, StandardError), err:
write_message("If you want to read the citation data from file set up \
the file_for_citations parameter in the config file [%s]" %err, verbose=2)
cit, dict_of_ids = get_citations_from_db()
len_ = len(dict_of_ids.keys())
write_message("Number of nodes(papers) to rank : %s" % str(len_), verbose=3)
if len_ == 0:
write_message("Error: No citations to read!", sys.stderr)
raise Exception
try:
method = config.get(function, "citerank_method")
except ConfigParser.NoOptionError, err:
write_message("Exception: %s " %err, sys.stderr)
raise Exception
write_message("Running %s method." % method, verbose=2)
dates = get_dates(function, config, dict_of_ids)
if method == "citation_time":
try:
time_decay = float(config.get(function, "time_decay"))
except (ConfigParser.NoOptionError, ValueError), err:
write_message("Exception: %s" % err, sys.stderr)
raise Exception
date_coef = calculate_time_weights(len_, time_decay, dates)
#cit = remove_loops(cit, dates, dict_of_ids)
dict_of_ranks = \
run_citation_rank_time(cit, dict_of_ids, date_coef, dates)
else:
try:
conv_threshold = float(config.get(function, "conv_threshold"))
check_point = int(config.get(function, "check_point"))
damping_factor = float(config.get(function, "damping_factor"))
write_message("Parameters: d = %s, conv_threshold = %s, \
check_point = %s" %(str(damping_factor), \
str(conv_threshold), str(check_point)), verbose=5)
except (ConfigParser.NoOptionError, StandardError), err:
write_message("Exception: %s" % err, sys.stderr)
raise Exception
if method == "pagerank_classic":
ref = construct_ref_array(cit, dict_of_ids, len_)
use_ext_cit = ""
try:
use_ext_cit = config.get(function, "use_external_citations")
write_message("Pagerank will use external citations: %s" \
%str(use_ext_cit), verbose=5)
except (ConfigParser.NoOptionError, StandardError), err:
write_message("%s" % err, verbose=2)
if use_ext_cit == "yes":
try:
ext_citation_file = config.get(function, "ext_citation_file")
ext_links = get_external_links_from_file\
(ext_citation_file, ref, dict_of_ids)
except (ConfigParser.NoOptionError, StandardError):
write_message("If you want to read the external citation data \
from file set up the ext_citation_file parameter in the config. file", verbose=3)
try:
reference_tag = config.get(function, "ext_reference_tag")
dummy = int(reference_tag[0:3])
except (ConfigParser.NoOptionError, StandardError):
write_message("You need to set up correctly the \
reference_tag in the cfg file", sys.stderr)
raise Exception
ext_links = get_external_links_from_db(ref, \
dict_of_ids, reference_tag)
avg = avg_ext_links_with_0(ext_links)
if avg < 1:
write_message("This method can't be ran. There is not enough \
information about the external citation. Hint: check the reference tag", sys.stderr)
raise Exception
avg_ext_links_without_0(ext_links)
try:
alpha = float(config.get(function, "ext_alpha"))
beta = float(config.get(function, "ext_beta"))
except (ConfigParser.NoOptionError, StandardError), err:
write_message("Exception: %s" % err, sys.stderr)
raise Exception
dict_of_ranks = run_pagerank_ext(cit, dict_of_ids, ref, \
ext_links, conv_threshold, check_point, alpha, beta, dates)
else:
dict_of_ranks = run_pagerank(cit, dict_of_ids, len_, ref, \
damping_factor, conv_threshold, check_point, dates)
elif method == "pagerank_time":
try:
time_decay = float(config.get(function, "time_decay"))
write_message("Parameter: time_decay = %s" %str(time_decay), verbose=5)
except (ConfigParser.NoOptionError, StandardError), err:
write_message("Exception: %s" % err, sys.stderr)
raise Exception
date_coef = calculate_time_weights(len_, time_decay, dates)
cit = remove_loops(cit, dates, dict_of_ids)
ref = construct_ref_array(cit, dict_of_ids, len_)
dict_of_ranks = run_pagerank_time(cit, dict_of_ids, len_, ref, \
damping_factor, conv_threshold, check_point, date_coef, dates)
else:
write_message("Error: Unknown ranking method. \
Please check the ranking_method parameter in the config. file.", sys.stderr)
raise Exception
try:
filename_ranks = config.get(function, "output_ranks_to_filename")
max_ranks = config.get(function, "output_rank_limit")
if not max_ranks.isdigit():
max_ranks = len_
else:
max_ranks = int(max_ranks)
if max_ranks > len_:
max_ranks = len_
ranks = sort_weights(dict_of_ranks)
write_message("Ranks: %s" % str(ranks), verbose=9)
write_first_ranks_to_file(ranks, dict_of_ranks, \
max_ranks, filename_ranks)
except (ConfigParser.NoOptionError, StandardError):
write_message("If you want the ranks to be printed in a file you have \
to set output_ranks_to_filename and output_rank_limit \
parameters in the configuration file", verbose=3)
into_db(dict_of_ranks, rank_method_code)
diff --git a/modules/bibupload/lib/bibupload_regression_tests.py b/modules/bibupload/lib/bibupload_regression_tests.py
index ea604c941..cec249906 100644
--- a/modules/bibupload/lib/bibupload_regression_tests.py
+++ b/modules/bibupload/lib/bibupload_regression_tests.py
@@ -1,3481 +1,3481 @@
# -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable-msg=C0301
"""Regression tests for the BibUpload."""
__revision__ = "$Id$"
import re
import unittest
import datetime
import os
import time
import sys
from urllib2 import urlopen
import pprint
if sys.hexversion < 0x2060000:
from md5 import md5
else:
from hashlib import md5
from invenio.config import CFG_OAI_ID_FIELD, CFG_PREFIX, CFG_SITE_URL, CFG_TMPDIR, \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG, \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG, \
CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG, \
CFG_WEBDIR
from invenio import bibupload
from invenio.search_engine import print_record
from invenio.dbquery import run_sql, get_table_status_info
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.testutils import make_test_suite, run_test_suite
from invenio.bibdocfile import BibRecDocs
from invenio.bibtask import task_set_task_param, setup_loggers
# helper functions:
def remove_tag_001_from_xmbuffer(xmbuffer):
"""Remove tag 001 from MARCXML buffer. Useful for testing two
MARCXML buffers without paying attention to recIDs attributed
during the bibupload.
"""
return re.sub(r'.*', '', xmbuffer)
def compare_xmbuffers(xmbuffer1, xmbuffer2):
"""Compare two XM (XML MARC) buffers by removing whitespaces
before testing.
"""
def remove_blanks_from_xmbuffer(xmbuffer):
"""Remove \n and blanks from XMBUFFER."""
out = xmbuffer.replace("\n", "")
out = out.replace(" ", "")
return out
# remove whitespace:
xmbuffer1 = remove_blanks_from_xmbuffer(xmbuffer1)
xmbuffer2 = remove_blanks_from_xmbuffer(xmbuffer2)
if xmbuffer1 != xmbuffer2:
return "\n=" + xmbuffer1 + "=\n" + '!=' + "\n=" + xmbuffer2 + "=\n"
return ''
def remove_tag_001_from_hmbuffer(hmbuffer):
"""Remove tag 001 from HTML MARC buffer. Useful for testing two
HTML MARC buffers without paying attention to recIDs attributed
during the bibupload.
"""
return re.sub(r'(^|\n)(
)?[0-9]{9}\s001__\s\d+($|\n)', '', hmbuffer)
def compare_hmbuffers(hmbuffer1, hmbuffer2):
"""Compare two HM (HTML MARC) buffers by removing whitespaces
before testing.
"""
hmbuffer1 = hmbuffer1.strip()
hmbuffer2 = hmbuffer2.strip()
# remove eventual
...
formatting:
hmbuffer1 = re.sub(r'^
', '', hmbuffer1)
hmbuffer2 = re.sub(r'^
', '', hmbuffer2)
hmbuffer1 = re.sub(r'
$', '', hmbuffer1)
hmbuffer2 = re.sub(r'
$', '', hmbuffer2)
# remove leading recid, leaving only field values:
hmbuffer1 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer2)
# remove leading whitespace:
hmbuffer1 = re.sub(r'(^|\n)\s+', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)\s+', '', hmbuffer2)
compare_hmbuffers = hmbuffer1 == hmbuffer2
if not compare_hmbuffers:
return "\n=" + hmbuffer1 + "=\n" + '!=' + "\n=" + hmbuffer2 + "=\n"
return ''
def wipe_out_record_from_all_tables(recid):
"""
Wipe out completely the record and all its traces of RECID from
the database (bibrec, bibrec_bibxxx, bibxxx, bibfmt). Useful for
the time being for test cases.
"""
# delete all the linked bibdocs
for bibdoc in BibRecDocs(recid).list_bibdocs():
bibdoc.expunge()
# delete from bibrec:
run_sql("DELETE FROM bibrec WHERE id=%s", (recid,))
# delete from bibrec_bibxxx:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE FROM %(bibrec_bibxxx)s WHERE id_bibrec=%%s" % \
{'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)},
(recid,))
# delete all unused bibxxx values:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE %(bibxxx)s FROM %(bibxxx)s " \
" LEFT JOIN %(bibrec_bibxxx)s " \
" ON %(bibxxx)s.id=%(bibrec_bibxxx)s.id_bibxxx " \
" WHERE %(bibrec_bibxxx)s.id_bibrec IS NULL" % \
{'bibxxx': "bib%i%ix" % (i, j),
'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)})
# delete from bibfmt:
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s", (recid,))
# delete from bibrec_bibdoc:
run_sql("DELETE FROM bibrec_bibdoc WHERE id_bibrec=%s", (recid,))
def try_url_download(url):
"""Try to download a given URL"""
try:
open_url = urlopen(url)
open_url.read()
except Exception, e:
raise StandardError("Downloading %s is impossible because of %s"
% (url, str(e)))
return True
class GenericBibUploadTest(unittest.TestCase):
"""Generic BibUpload testing class with predefined
setUp and tearDown methods.
"""
def setUp(self):
self.verbose = 0
setup_loggers()
task_set_task_param('verbose', self.verbose)
self.last_recid = run_sql("SELECT MAX(id) FROM bibrec")[0][0]
def tearDown(self):
for recid in run_sql("SELECT id FROM bibrec WHERE id>%s", (self.last_recid,)):
wipe_out_record_from_all_tables(recid[0])
class BibUploadInsertModeTest(GenericBibUploadTest):
"""Testing insert mode."""
def setUp(self):
# pylint: disable-msg=C0103
"""Initialise the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test = """somethingTester, J YMITTester, K JCERN2Tester, GCERN3test11test31test12test32test13test33test21test41test22test42test14test51test52Tester, TCERN"""
self.test_hm = """
100__ $$aTester, T$$uCERN
111__ $$atest11$$ctest31
111__ $$atest12$$ctest32
111__ $$atest13$$ctest33
111__ $$btest21$$dtest41
111__ $$btest22$$dtest42
111__ $$atest14
111__ $$etest51
111__ $$etest52
245__ $$asomething
700__ $$aTester, J Y$$uMIT
700__ $$aTester, K J$$uCERN2
700__ $$aTester, G$$uCERN3
"""
def test_create_record_id(self):
"""bibupload - insert mode, trying to create a new record ID in the database"""
rec_id = bibupload.create_new_record()
self.assertNotEqual(-1, rec_id)
def test_no_retrieve_record_id(self):
"""bibupload - insert mode, detection of record ID in the input file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'insert')
# We compare the value found with None
self.assertEqual(None, rec_id)
def test_insert_complete_xmlmarc(self):
"""bibupload - insert mode, trying to insert complete MARCXML file"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the main function with the record as a parameter
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# We retrieve the inserted xml
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.test), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.test_hm), '')
class BibUploadAppendModeTest(GenericBibUploadTest):
"""Testing append mode."""
def setUp(self):
# pylint: disable-msg=C0103
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_existing = """123456789Tester, TDESY0003719PHOPHO"""
self.test_to_append = """123456789Tester, UCERN0003719PHOPHO"""
self.test_expected_xm = """123456789Tester, TDESYTester, UCERN0003719PHOPHO"""
self.test_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uDESY
100__ $$aTester, U$$uCERN
970__ $$a0003719PHOPHO
"""
# insert test record:
test_to_upload = self.test_existing.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
self.test_recid = recid
# replace test buffers with real recid of inserted test record:
self.test_existing = self.test_existing.replace('123456789',
str(self.test_recid))
self.test_to_append = self.test_to_append.replace('123456789',
str(self.test_recid))
self.test_expected_xm = self.test_expected_xm.replace('123456789',
str(self.test_recid))
self.test_expected_hm = self.test_expected_hm.replace('123456789',
str(self.test_recid))
def test_retrieve_record_id(self):
"""bibupload - append mode, the input file should contain a record ID"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'append')
# We compare the value found with None
self.assertEqual(self.test_recid, rec_id)
# clean up after ourselves:
def test_update_modification_record_date(self):
"""bibupload - append mode, checking the update of the modification date"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_existing)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], opt_mode='append')
# Retrieve current localtime
now = time.localtime()
# We update the modification date
bibupload.update_bibrec_modif_date(convert_datestruct_to_datetext(now), rec_id)
# We retrieve the modification date from the database
query = """SELECT DATE_FORMAT(modification_date,'%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec where id = %s"""
res = run_sql(query % rec_id)
# We compare the two results
self.assertEqual(res[0][0], convert_datestruct_to_datetext(now))
# clean up after ourselves:
def test_append_complete_xml_marc(self):
"""bibupload - append mode, appending complete MARCXML file"""
# Now we append a datafield
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the main function with the record as a parameter
err, recid = bibupload.bibupload(recs[0], opt_mode='append')
# We retrieve the inserted xm
after_append_xm = print_record(recid, 'xm')
after_append_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(after_append_xm, self.test_expected_xm), '')
self.assertEqual(compare_hmbuffers(after_append_hm, self.test_expected_hm), '')
# clean up after ourselves:
class BibUploadCorrectModeTest(GenericBibUploadTest):
"""
Testing correcting a record containing similar tags (identical
tag, different indicators). Currently CDS Invenio replaces only
those tags that have matching indicators too, unlike ALEPH500 that
does not pay attention to indicators, it corrects all fields with
the same tag, regardless of the indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest, JohnTest UniversityCoolTest, JimTest Laboratory
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
self.testrec1_xm_to_correct = """
123456789Test, JosephTest AcademyTest2, JosephTest2 Academy
"""
self.testrec1_corrected_xm = """
123456789SzGeCERNTest, JaneTest InstituteCoolTest, JosephTest AcademyTest2, JosephTest2 Academy
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10048 $$aCool
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_record_correction(self):
"""bibupload - correct mode, similar MARCXML tags/indicators"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
err, self.recid = bibupload.bibupload(recs[0], opt_mode='correct')
corrected_xm = print_record(self.recid, 'xm')
corrected_hm = print_record(self.recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# clean up after ourselves:
return
class BibUploadDeleteModeTest(GenericBibUploadTest):
"""
Testing deleting specific tags from a record while keeping anything else
untouched. Currently CDS Invenio deletes only those tags that have
matching indicators too, unlike ALEPH500 that does not pay attention to
indicators, it corrects all fields with the same tag, regardless of the
indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest, JohnTest UniversityCoolTest, JimTest Laboratorydumb text
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
888__ $$adumb text
"""
self.testrec1_xm_to_delete = """
123456789Test, JaneTest InstituteTest, JohnsonTest UniversityCooldumb text
"""
self.testrec1_corrected_xm = """
123456789SzGeCERNTest, JohnTest UniversityTest, JimTest Laboratory
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
10047 $$aTest, John$$uTest University
10047 $$aTest, Jim$$uTest Laboratory
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_delete = self.testrec1_xm_to_delete.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
# Checking dumb text is in bibxxx
- self.failUnless(run_sql("SELECT * from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
+ self.failUnless(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
def test_record_tags_deletion(self):
"""bibupload - delete mode, deleting specific tags"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_delete)
err, recid = bibupload.bibupload(recs[0], opt_mode='delete')
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# Checking dumb text is no more in bibxxx
- self.failIf(run_sql("SELECT * from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
+ self.failIf(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
# clean up after ourselves:
class BibUploadReplaceModeTest(GenericBibUploadTest):
"""Testing replace mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest, JohnTest UniversityCoolTest, JimTest Laboratory
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
self.testrec1_xm_to_replace = """
123456789Test, JosephTest AcademyTest2, JosephTest2 Academy
"""
self.testrec1_replaced_xm = """
123456789Test, JosephTest AcademyTest2, JosephTest2 Academy
"""
self.testrec1_replaced_hm = """
001__ 123456789
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_replace = self.testrec1_xm_to_replace.replace('123456789', str(recid))
self.testrec1_replaced_xm = self.testrec1_replaced_xm.replace('123456789', str(recid))
self.testrec1_replaced_hm = self.testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_record_replace(self):
"""bibupload - replace mode, similar MARCXML tags/indicators"""
# replace some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_replace)
err, self.recid = bibupload.bibupload(recs[0], opt_mode='replace')
replaced_xm = print_record(self.recid, 'xm')
replaced_hm = print_record(self.recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, self.testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, self.testrec1_replaced_hm), '')
# clean up after ourselves:
class BibUploadReferencesModeTest(GenericBibUploadTest):
"""Testing references mode."""
def setUp(self):
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_insert = """123456789Tester, TCERN"""
self.test_reference = """123456789M. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,J. High Energy Phys. 07 (2004) 014"""
self.test_reference_expected_xm = """123456789Tester, TCERNM. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,J. High Energy Phys. 07 (2004) 014"""
self.test_insert_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
"""
self.test_reference_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
%(reference_tag)sC5 $$mM. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,$$sJ. High Energy Phys. 07 (2004) 014
""" % {'reference_tag': bibupload.CFG_BIBUPLOAD_REFERENCE_TAG}
# insert test record:
test_insert = self.test_insert.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_insert)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.test_insert = self.test_insert.replace('123456789', str(recid))
self.test_insert_hm = self.test_insert_hm.replace('123456789', str(recid))
self.test_reference = self.test_reference.replace('123456789', str(recid))
self.test_reference_expected_xm = self.test_reference_expected_xm.replace('123456789', str(recid))
self.test_reference_expected_hm = self.test_reference_expected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.test_insert), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.test_insert_hm), '')
self.test_recid = recid
def test_reference_complete_xml_marc(self):
"""bibupload - reference mode, inserting references MARCXML file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_reference)
# We call the main function with the record as a parameter
err, recid = bibupload.bibupload(recs[0], opt_mode='reference')
# We retrieve the inserted xml
reference_xm = print_record(recid, 'xm')
reference_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(reference_xm, self.test_reference_expected_xm), '')
self.assertEqual(compare_hmbuffers(reference_hm, self.test_reference_expected_hm), '')
class BibUploadFMTModeTest(GenericBibUploadTest):
"""Testing FMT mode."""
def setUp(self):
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.new_xm_with_fmt = """
SzGeCERNHBTest. Okay.2008-03-14 15:14:00Bar, BazFooOn the quux and huux
"""
self.expected_xm_after_inserting_new_xm_with_fmt = """
123456789SzGeCERNBar, BazFooOn the quux and huux
"""
self.expected_hm_after_inserting_new_xm_with_fmt = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux
"""
self.recid76_xm_before_all_the_tests = print_record(76, 'xm')
self.recid76_hm_before_all_the_tests = print_record(76, 'hm')
self.recid76_fmts = run_sql("""SELECT last_updated, value, format FROM bibfmt WHERE id_bibrec=76""")
self.recid76_xm_with_fmt = """
76SzGeCERNHBTest. Here is some format value.Doe, JohnCERNOn the foos and bars
"""
self.recid76_xm_with_fmt_only_first = """
76HBTest. Let us see if this gets inserted well.
"""
self.recid76_xm_with_fmt_only_second = """
76HBTest. Yet another test, to be run after the first one.HDTest. Let's see what will be stored in the detailed format field.
"""
def tearDown(self):
"""Helper function that restores recID 76 MARCXML, using the
value saved before all the tests started to execute.
(see self.recid76_xm_before_all_the_tests).
Does not restore HB and HD formats.
"""
recs = bibupload.xml_marc_to_records(self.recid76_xm_before_all_the_tests)
err, recid = bibupload.bibupload(recs[0], opt_mode='replace')
for (last_updated, value, format) in self.recid76_fmts:
run_sql("""UPDATE bibfmt SET last_updated=%s, value=%s WHERE id_bibrec=76 AND format=%s""", (last_updated, value, format))
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.recid76_xm_before_all_the_tests), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.recid76_hm_before_all_the_tests), '')
def test_inserting_new_record_containing_fmt_tag(self):
"""bibupload - FMT tag, inserting new record containing FMT tag"""
recs = bibupload.xml_marc_to_records(self.new_xm_with_fmt)
(dummy, new_recid) = bibupload.bibupload(recs[0], opt_mode='insert')
xm_after = print_record(new_recid, 'xm')
hm_after = print_record(new_recid, 'hm')
hb_after = print_record(new_recid, 'hb')
self.assertEqual(compare_xmbuffers(xm_after,
self.expected_xm_after_inserting_new_xm_with_fmt.replace('123456789', str(new_recid))), '')
self.assertEqual(compare_hmbuffers(hm_after,
self.expected_hm_after_inserting_new_xm_with_fmt.replace('123456789', str(new_recid))), '')
self.assertEqual(run_sql('SELECT last_updated from bibfmt WHERE id_bibrec=%s', (new_recid, ))[0][0], datetime.datetime(2008, 3, 14, 15, 14))
self.failUnless(hb_after.startswith("Test. Okay."))
def test_updating_existing_record_formats_in_format_mode(self):
"""bibupload - FMT tag, updating existing record via format mode"""
xm_before = print_record(76, 'xm')
hm_before = print_record(76, 'hm')
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='format')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Let us see if this gets inserted well."))
# now insert another format value and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_second)
bibupload.bibupload(recs[0], opt_mode='format')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
def test_updating_existing_record_formats_in_correct_mode(self):
"""bibupload - FMT tag, updating existing record via correct mode"""
xm_before = print_record(76, 'xm')
hm_before = print_record(76, 'hm')
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='correct')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Let us see if this gets inserted well."))
# now insert another format value and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_second)
bibupload.bibupload(recs[0], opt_mode='correct')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
def test_updating_existing_record_formats_in_replace_mode(self):
"""bibupload - FMT tag, updating existing record via replace mode"""
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='replace')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(compare_xmbuffers(xm_after,
'76'), '')
self.assertEqual(compare_hmbuffers(hm_after,
'000000076 001__ 76'), '')
self.failUnless(hb_after.startswith("Test. Let us see if this gets inserted well."))
# now insert another format value and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_second)
bibupload.bibupload(recs[0], opt_mode='replace')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(compare_xmbuffers(xm_after, """
76"""), '')
self.assertEqual(compare_hmbuffers(hm_after, '000000076 001__ 76'), '')
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
# final insertion and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt)
bibupload.bibupload(recs[0], opt_mode='replace')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(compare_xmbuffers(xm_after, """
76SzGeCERNDoe, JohnCERNOn the foos and bars
"""), '')
self.assertEqual(compare_hmbuffers(hm_after, """
001__ 76
003__ SzGeCERN
100__ $$aDoe, John$$uCERN
245__ $$aOn the foos and bars
"""), '')
self.failUnless(hb_after.startswith("Test. Here is some format value."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
class BibUploadRecordsWithSYSNOTest(GenericBibUploadTest):
"""Testing uploading of records that have external SYSNO present."""
def setUp(self):
# pylint: disable-msg=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that SYSNO fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789SzGeCERNBar, BazFooOn the quux and huux 1sysno1sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERNBar, BazFooOn the quux and huux 1 Updatedsysno1sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789SzGeCERNBar, BazFooOn the quux and huux 1 Updatedsysno1sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec2 = """
987654321SzGeCERNBar, BazFooOn the quux and huux 2sysno2sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
def test_insert_the_same_sysno_record(self):
"""bibupload - SYSNO tag, refuse to insert the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_sysno_record() finished"
def test_insert_or_replace_the_same_sysno_record(self):
"""bibupload - SYSNO tag, allow to insert or replace the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0],
opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() finished"
def test_replace_nonexisting_sysno_record(self):
"""bibupload - SYSNO tag, refuse to replace non-existing SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_sysno_record() finished"
class BibUploadRecordsWithEXTOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have external EXTOAIID present."""
def setUp(self):
# pylint: disable-msg=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that EXTOAIID fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789SzGeCERNextoaiid1extoaisrc1extoaiid2Bar, BazFooOn the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERNextoaiid1extoaisrc1extoaiid2Bar, BazFooOn the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789SzGeCERNextoaiid1extoaisrc1extoaiid2Bar, BazFooOn the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec2 = """
987654321SzGeCERNextoaiid2extoaisrc1extoaiid1Bar, BazFooOn the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid2
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid1
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
def test_insert_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to insert the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_extoaiid_record() finished"
def test_insert_or_replace_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, allow to insert or replace the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() finished"
def test_replace_nonexisting_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to replace non-existing EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() finished"
class BibUploadRecordsWithOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have OAI ID present."""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that OAI fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
GenericBibUploadTest.setUp(self)
self.xm_testrec1 = """
123456789SzGeCERNBar, BazFooOn the quux and huux 1oai:foo:1oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERNBar, BazFooOn the quux and huux 1 Updatedoai:foo:1oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_updated = """
123456789SzGeCERNBar, BazFooOn the quux and huux 1 Updatedoai:foo:1oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec2 = """
987654321SzGeCERNBar, BazFooOn the quux and huux 2oai:foo:2oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
def test_insert_the_same_oai_record(self):
"""bibupload - OAIID tag, refuse to insert the same OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
def test_insert_or_replace_the_same_oai_record(self):
"""bibupload - OAIID tag, allow to insert or replace the same OAI record"""
# initialize bibupload mode:
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
def test_replace_nonexisting_oai_record(self):
"""bibupload - OAIID tag, refuse to replace non-existing OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
class BibUploadIndicatorsTest(GenericBibUploadTest):
"""
Testing uploading of a MARCXML record with indicators having
either blank space (as per MARC schema) or empty string value (old
behaviour).
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERNTest, JohnTest University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERNTest, JohnTest University
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
def test_record_with_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec1_hm), '')
def test_record_with_no_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with no spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec2_hm), '')
class BibUploadUpperLowerCaseTest(GenericBibUploadTest):
"""
Testing treatment of similar records with only upper and lower
case value differences in the bibxxx table.
"""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERNTest, JohnTest University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERNTeSt, JoHnTest UniVeRsity
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTeSt, JoHn$$uTest UniVeRsity
"""
def test_record_with_upper_lower_case_letters(self):
"""bibupload - inserting similar MARCXML records with upper/lower case"""
# insert test record #1:
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
err1, recid1 = bibupload.bibupload(recs[0], opt_mode='insert')
recid1_inserted_xm = print_record(recid1, 'xm')
recid1_inserted_hm = print_record(recid1, 'hm')
# insert test record #2:
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
err1, recid2 = bibupload.bibupload(recs[0], opt_mode='insert')
recid2_inserted_xm = print_record(recid2, 'xm')
recid2_inserted_hm = print_record(recid2, 'hm')
# let us compare stuff now:
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid1_inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid1_inserted_hm),
self.testrec1_hm), '')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid2_inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid2_inserted_hm),
self.testrec2_hm), '')
class BibUploadControlledProvenanceTest(GenericBibUploadTest):
"""Testing treatment of tags under controlled provenance in the correct mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest titleblablasamblublusimhuman
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$9sim$$ablublu
6531_ $$ahuman
"""
self.testrec1_xm_to_correct = """
123456789bleblesimbloblosom
"""
self.testrec1_corrected_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest titleblablasamhumanbleblesimbloblosom
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$ahuman
6531_ $$9sim$$ableble
6531_ $$9som$$abloblo
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_controlled_provenance_persistence(self):
"""bibupload - correct mode, tags with controlled provenance"""
# correct metadata tags; will the protected tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
err, recid = bibupload.bibupload(recs[0], opt_mode='correct')
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
class BibUploadStrongTagsTest(GenericBibUploadTest):
"""Testing treatment of strong tags and the replace mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789SzGeCERNTest, JaneTest InstituteTest titleA valueAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_xm_to_replace = """
123456789Test, JosephTest Academy
"""
self.testrec1_replaced_xm = """
123456789Test, JosephTest AcademyA valueAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_replaced_hm = """
001__ 123456789
100__ $$aTest, Joseph$$uTest Academy
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_replace = self.testrec1_xm_to_replace.replace('123456789', str(recid))
self.testrec1_replaced_xm = self.testrec1_replaced_xm.replace('123456789', str(recid))
self.testrec1_replaced_hm = self.testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_strong_tags_persistence(self):
"""bibupload - strong tags, persistence in replace mode"""
# replace all metadata tags; will the strong tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_replace)
err, recid = bibupload.bibupload(recs[0], opt_mode='replace')
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, self.testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, self.testrec1_replaced_hm), '')
class BibUploadPretendTest(GenericBibUploadTest):
"""
Testing bibupload --pretend correctness.
"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.demo_data = bibupload.xml_marc_to_records(open(os.path.join(CFG_TMPDIR, 'demobibdata.xml')).read())[0]
self.before = self._get_tables_fingerprint()
task_set_task_param('pretend', True)
def tearDown(self):
task_set_task_param('pretend', False)
def _get_tables_fingerprint():
"""
Take lenght and last modification time of all the tables that
might be touched by bibupload and return them in a nice structure.
"""
fingerprint = {}
tables = ['bibrec', 'bibdoc', 'bibrec_bibdoc', 'bibdoc_bibdoc', 'bibfmt', 'hstDOCUMENT', 'hstRECORD']
for i in xrange(100):
tables.append('bib%02dx' % i)
tables.append('bibrec_bib%02dx' % i)
for table in tables:
fingerprint[table] = get_table_status_info(table)
return fingerprint
_get_tables_fingerprint = staticmethod(_get_tables_fingerprint)
def _checks_tables_fingerprints(before, after):
"""
Checks differences in table_fingerprints.
"""
err = True
for table in before.keys():
if before[table] != after[table]:
print >> sys.stderr, "Table %s has been modified: before was [%s], after was [%s]" % (table, pprint.pformat(before[table]), pprint.pformat(after[table]))
err = False
return err
_checks_tables_fingerprints = staticmethod(_checks_tables_fingerprints)
def test_pretend_insert(self):
"""bibupload - pretend insert"""
bibupload.bibupload(self.demo_data, opt_mode='insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_correct(self):
"""bibupload - pretend correct"""
bibupload.bibupload(self.demo_data, opt_mode='correct', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace(self):
"""bibupload - pretend replace"""
bibupload.bibupload(self.demo_data, opt_mode='replace', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_append(self):
"""bibupload - pretend append"""
bibupload.bibupload(self.demo_data, opt_mode='append', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace_or_insert(self):
"""bibupload - pretend replace or insert"""
bibupload.bibupload(self.demo_data, opt_mode='replace_or_insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_holdingpen(self):
"""bibupload - pretend holdingpen"""
bibupload.bibupload(self.demo_data, opt_mode='holdingpen', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_delete(self):
"""bibupload - pretend delete"""
bibupload.bibupload(self.demo_data, opt_mode='delete', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_reference(self):
"""bibupload - pretend reference"""
bibupload.bibupload(self.demo_data, opt_mode='reference', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
class BibUploadFFTModeTest(GenericBibUploadTest):
"""
Testing treatment of fulltext file transfer import mode.
"""
def _test_bibdoc_status(self, recid, docname, status):
res = run_sql('SELECT bd.status FROM bibrec_bibdoc as bb JOIN bibdoc as bd ON bb.id_bibdoc = bd.id WHERE bb.id_bibrec = %s AND bd.docname = %s', (recid, docname))
self.failUnless(res)
self.assertEqual(status, res[0][0])
def test_writing_rights(self):
"""bibupload - FFT has writing rights"""
self.failUnless(bibupload.writing_rights_p())
def test_simple_fft_insert(self):
"""bibupload - simple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gif
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url))
def test_exotic_format_fft_append(self):
"""bibupload - exotic format FFT append"""
# define the test case:
testfile = os.path.join(CFG_TMPDIR, 'test.ps.Z')
open(testfile, 'w').write('TEST')
test_to_upload = """
SzGeCERNTest, JohnTest University
"""
testrec_to_append = """
123456789%s
""" % testfile
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/test.ps.Z
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/test.ps.Z
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/test.ps.Z" \
% {'siteurl': CFG_SITE_URL}
testrec_expected_url2 = "%(siteurl)s/record/123456789/files/test?format=ps.Z" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_to_append = testrec_to_append.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url.replace('123456789',
str(recid))
recs = bibupload.xml_marc_to_records(testrec_to_append)
err, recid = bibupload.bibupload(recs[0], opt_mode='append')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.assertEqual(urlopen(testrec_expected_url).read(), 'TEST')
self.assertEqual(urlopen(testrec_expected_url2).read(), 'TEST')
def test_fft_check_md5_through_bibrecdoc_str(self):
"""bibupload - simple FFT insert, check md5 through BibRecDocs.str()"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest University%s/img/head.gif
""" % CFG_SITE_URL
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
original_md5 = md5(urlopen('%s/img/head.gif' % CFG_SITE_URL).read()).hexdigest()
bibrec_str = str(BibRecDocs(int(recid)))
md5_found = False
for row in bibrec_str.split('\n'):
if 'checksum' in row:
if original_md5 in row:
md5_found = True
self.failUnless(md5_found)
def test_detailed_fft_insert(self):
"""bibupload - detailed FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifSuperMainThis is a descriptionThis is a commentCIDIESSEhttp://cds.cern.ch/img/cds.gifSuperMain.jpegThis is a descriptionThis is a second commentCIDIESSE
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/CIDIESSE.gifThis is a descriptionThis is a comment%(siteurl)s/record/123456789/files/CIDIESSE.jpegThis is a descriptionThis is a second comment
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/CIDIESSE.gif$$yThis is a description$$zThis is a comment
8564_ $$u%(siteurl)s/record/123456789/files/CIDIESSE.jpeg$$yThis is a description$$zThis is a second comment
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_url1 = "%(siteurl)s/record/123456789/files/CIDIESSE.gif" % {'siteurl': CFG_SITE_URL}
testrec_expected_url2 = "%(siteurl)s/record/123456789/files/CIDIESSE.jpeg" % {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url1 = testrec_expected_url1.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url1.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url1))
self.failUnless(try_url_download(testrec_expected_url2))
def test_simple_fft_insert_with_restriction(self):
"""bibupload - simple FFT insert with restriction"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifthesishttp://cds.cern.ch/img/cds.gif
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif%(siteurl)s/record/123456789/files/cds.gif?subformat=iconicon
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif?subformat=icon$$xicon
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" \
% {'siteurl': CFG_SITE_URL}
testrec_expected_icon = "%(siteurl)s/record/123456789/files/cds.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.assertEqual(urlopen(testrec_expected_icon).read(), open('%s/img/restricted.gif' % CFG_WEBDIR).read())
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
def test_simple_fft_insert_with_icon(self):
"""bibupload - simple FFT insert with icon"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifhttp://cds.cern.ch/img/cds.gif
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif%(siteurl)s/record/123456789/files/cds.gif?subformat=iconicon
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif?subformat=icon$$xicon
""" % {'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" \
% {'siteurl': CFG_SITE_URL}
testrec_expected_icon = "%(siteurl)s/record/123456789/files/cds.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_icon))
def test_multiple_fft_insert(self):
"""bibupload - multiple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifhttp://cdsweb.cern.ch/img/head.gifhttp://doc.cern.ch/archive/electronic/hep-th/0101/0101001.pdf%(prefix)s/var/tmp/demobibdata.xml
""" % { 'prefix': CFG_PREFIX }
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/0101001.pdf%(siteurl)s/record/123456789/files/cds.gif%(siteurl)s/record/123456789/files/demobibdata.xml%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/0101001.pdf
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
8564_ $$u%(siteurl)s/record/123456789/files/demobibdata.xml
8564_ $$u%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
# insert test record:
testrec_expected_urls = []
for files in ('cds.gif', 'head.gif', '0101001.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/record/123456789/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'files' : files})
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_urls = []
for files in ('cds.gif', 'head.gif', '0101001.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/record/%(recid)s/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'files' : files, 'recid' : recid})
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
for url in testrec_expected_urls:
self.failUnless(try_url_download(url))
self._test_bibdoc_status(recid, 'head', '')
self._test_bibdoc_status(recid, '0101001', '')
self._test_bibdoc_status(recid, 'cds', '')
self._test_bibdoc_status(recid, 'demobibdata', '')
def test_simple_fft_correct(self):
"""bibupload - simple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gif
"""
test_to_correct = """
123456789http://cds.cern.ch/img/cds.gif
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'cds', '')
def test_fft_implicit_fix_marc(self):
"""bibupload - FFT implicit FIX-MARC"""
test_to_upload = """
SzGeCERNTest, JohnTest Universityfoo@bar.comhttp://cds.cern.ch/img/cds.gif
"""
test_to_correct = """
123456789foo@bar.comhttp://cds.cern.ch/img/cds.gif%(siteurl)s/record/123456789/files/cds.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest Universityfoo@bar.comhttp://cds.cern.ch/img/cds.gif
"""
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8560_ $$ffoo@bar.com
8564_ $$uhttp://cds.cern.ch/img/cds.gif
"""
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
test_to_correct = test_to_correct.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
# correct test record with implicit FIX-MARC:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
def test_fft_vs_bibedit(self):
"""bibupload - FFT Vs. BibEdit compatibility"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gif
"""
test_to_replace = """
123456789SzGeCERNTest, JohnTest Universityhttp://www.google.com/BibEdit Comment%(siteurl)s/record/123456789/files/cds.gifBibEdit Description01http://cern.ch/
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_xm = str(test_to_replace)
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$uhttp://www.google.com/
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif$$x01$$yBibEdit Description$$zBibEdit Comment
8564_ $$uhttp://cern.ch/
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload(recs[0], opt_mode='replace')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'cds', '')
bibrecdocs = BibRecDocs(recid)
bibdoc = bibrecdocs.get_bibdoc('cds')
self.assertEqual(bibdoc.get_description('.gif'), 'BibEdit Description')
def test_detailed_fft_correct(self):
"""bibupload - detailed FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifTryComment
"""
test_to_correct = """
123456789http://cdsweb.cern.ch/img/head.gifcdspatataNext TryKEEP-OLD-VALUE
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/patata.gifNext TryComment
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/patata.gif$$yNext Try$$zComment
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', '')
def test_no_url_fft_correct(self):
"""bibupload - no_url FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifTryComment
"""
test_to_correct = """
123456789cdspatata.gifKEEP-OLD-VALUENext Comment
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/patata.gifTryNext Comment
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/patata.gif$$yTry$$zNext Comment
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', '')
def test_new_icon_fft_append(self):
"""bibupload - new icon FFT append"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest University
"""
test_to_correct = """
123456789cdshttp://cds.cern.ch/img/cds.gif
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif?subformat=iconicon
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif?subformat=icon$$xicon
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='append')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'cds', '')
def test_multiple_fft_correct(self):
"""bibupload - multiple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifTryCommentRestrictedhttp://cds.cern.ch/img/cds.gif.jpegTry jpegComment jpegRestricted
"""
test_to_correct = """
123456789http://cds.cern.ch/img/cds.gifpatata.gifNew restricted
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/patata.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/patata.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', 'New restricted')
def test_purge_fft_correct(self):
"""bibupload - purge FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest Universityhttp://cds.cern.ch/img/cds.gifhttp://cdsweb.cern.ch/img/head.gif
"""
test_to_correct = """
123456789http://cds.cern.ch/img/cds.gif
"""
test_to_purge = """
123456789http://cds.cern.ch/img/cds.gifPURGE
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
8564_ $$u%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" % { 'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_purge = test_to_purge.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# purge test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_purge)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'cds', '')
self._test_bibdoc_status(recid, 'head', '')
def test_revert_fft_correct(self):
"""bibupload - revert FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest University%s/img/iconpen.gifcds
""" % CFG_SITE_URL
test_to_correct = """
123456789%s/img/head.gifcds
""" % CFG_SITE_URL
test_to_revert = """
123456789cdsREVERT1
"""
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/cds.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/cds.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/cds.gif" % { 'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_revert = test_to_revert.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# revert test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_revert)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'cds', '')
expected_content_version1 = urlopen('%s/img/iconpen.gif' % CFG_SITE_URL).read()
expected_content_version2 = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
expected_content_version3 = expected_content_version1
content_version1 = urlopen('%s/record/%s/files/cds.gif?version=1' % (CFG_SITE_URL, recid)).read()
content_version2 = urlopen('%s/record/%s/files/cds.gif?version=2' % (CFG_SITE_URL, recid)).read()
content_version3 = urlopen('%s/record/%s/files/cds.gif?version=3' % (CFG_SITE_URL, recid)).read()
self.assertEqual(expected_content_version1, content_version1)
self.assertEqual(expected_content_version2, content_version2)
self.assertEqual(expected_content_version3, content_version3)
def test_simple_fft_replace(self):
"""bibupload - simple FFT replace"""
# define the test case:
test_to_upload = """
SzGeCERNTest, JohnTest University%s/img/iconpen.gifcds
""" % CFG_SITE_URL
test_to_replace = """
123456789SzGeCERNTest, JohnTest University%s/img/head.gif
""" % CFG_SITE_URL
testrec_expected_xm = """
123456789SzGeCERNTest, JohnTest University%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/record/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL}
testrec_expected_url = "%(siteurl)s/record/123456789/files/head.gif" % { 'siteurl': CFG_SITE_URL}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# replace test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload(recs[0], opt_mode='replace')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
expected_content_version = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
content_version = urlopen('%s/record/%s/files/head.gif' % (CFG_SITE_URL, recid)).read()
self.assertEqual(expected_content_version, content_version)
TEST_SUITE = make_test_suite(BibUploadInsertModeTest,
BibUploadAppendModeTest,
BibUploadCorrectModeTest,
BibUploadDeleteModeTest,
BibUploadReplaceModeTest,
BibUploadReferencesModeTest,
BibUploadRecordsWithSYSNOTest,
BibUploadRecordsWithEXTOAIIDTest,
BibUploadRecordsWithOAIIDTest,
BibUploadFMTModeTest,
BibUploadIndicatorsTest,
BibUploadUpperLowerCaseTest,
BibUploadControlledProvenanceTest,
BibUploadStrongTagsTest,
BibUploadFFTModeTest,
BibUploadPretendTest,
)
if __name__ == "__main__":
run_test_suite(TEST_SUITE, warn_user=True)
diff --git a/modules/webbasket/lib/webbasket_dblayer.py b/modules/webbasket/lib/webbasket_dblayer.py
index ec2b85d58..7b64f84d5 100644
--- a/modules/webbasket/lib/webbasket_dblayer.py
+++ b/modules/webbasket/lib/webbasket_dblayer.py
@@ -1,2252 +1,2249 @@
# -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
""" Database related functions for webbasket module """
__revision__ = "$Id$"
from zlib import decompress
from zlib import compress
from time import localtime
from invenio.textutils import encode_for_xml
from invenio.dbquery import run_sql
from invenio.webbasket_config import CFG_WEBBASKET_SHARE_LEVELS, \
CFG_WEBBASKET_ACTIONS, \
CFG_WEBBASKET_SHARE_LEVELS_ORDERED
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.websession_config import CFG_WEBSESSION_USERGROUP_STATUS
########################### Table of contents ################################
#
# NB. functions preceeded by a star use usergroup table
#
# 1. General functions
# - count_baskets
# - check_user_owns_basket
# - get_max_user_rights_on_basket
#
# 2. Personal baskets
# - get_personal_baskets_info_for_topic
# - get_all_personal_basket_ids_and_names_by_topic
# - get_all_personal_baskets_names
# - get_basket_name
# - is_personal_basket_valid
# - is_topic_valid
# - get_basket_topic
# - get_personal_topics_infos
# - rename_basket
# - rename_topic
# - move_baskets_to_topic
# - delete_basket
# - create_basket
#
# 3. Actions on baskets
# - get_basket_record
# - get_basket_content
# - get_basket_item
# - share_basket_with_group
# - update_rights
# - move_item
# - delete_item
# - add_to_basket
# - get_external_records_by_collection
# - store_external_records
# - store_external_urls
# - store_external_source
# - get_external_colid_and_url
#
# 4. Group baskets
# - get_group_basket_infos
# - get_group_name
# - get_all_group_basket_ids_and_names_by_group
# - (*) get_all_group_baskets_names
# - is_shared_to
#
# 5. External baskets (baskets user has subscribed to)
# - get_external_baskets_infos
# - get_external_basket_info
# - get_all_external_basket_ids_and_names
# - count_external_baskets
# - get_all_external_baskets_names
#
# 6. Public baskets (interface to subscribe to baskets)
# - get_public_basket_infos
# - get_public_basket_info
# - get_basket_general_infos
# - get_basket_owner_id
# - count_public_baskets
# - get_public_baskets_list
# - is_basket_public
# - subscribe
# - unsubscribe
# - is_user_subscribed_to_basket
# - count_subscribers
# - (*) get_groups_subscribing_to_basket
# - get_rights_on_public_basket
#
# 7. Annotating
# - get_notes
# - get_note
# - save_note
# - delete_note
# - note_belongs_to_item_in_basket_p
#
# 8. Usergroup functions
# - (*) get_group_infos
# - count_groups_user_member_of
# - (*) get_groups_user_member_of
#
# 9. auxilliary functions
# - __wash_sql_count
# - __decompress_last
# - create_pseudo_record
# - prettify_url
########################## General functions ##################################
def count_baskets(uid):
"""Return (nb personal baskets, nb group baskets, nb external
baskets) tuple for given user"""
query1 = "SELECT COUNT(id) FROM bskBASKET WHERE id_owner=%s"
res1 = run_sql(query1, (int(uid),))
personal = __wash_sql_count(res1)
query2 = """SELECT count(ugbsk.id_bskbasket)
FROM usergroup_bskBASKET ugbsk LEFT JOIN user_usergroup uug
ON ugbsk.id_usergroup=uug.id_usergroup
WHERE uug.id_user=%s AND uug.user_status!=%s
GROUP BY ugbsk.id_usergroup"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res2 = run_sql(query2, params)
if len(res2):
groups = reduce(lambda x, y: x + y, map(lambda x: x[0], res2))
else:
groups = 0
external = count_external_baskets(uid)
return (personal, groups, external)
def check_user_owns_baskets(uid, bskids):
""" Return 1 if user is owner of every basket in list bskids"""
if not((type(bskids) is list) or (type(bskids) is tuple)):
bskids = [bskids]
query = """SELECT id_owner FROM bskBASKET WHERE %s GROUP BY id_owner"""
sep = ' OR '
query %= sep.join(['id=%s'] * len(bskids))
res = run_sql(query, tuple(bskids))
if len(res)==1 and int(res[0][0])==uid:
return 1
else:
return 0
def get_max_user_rights_on_basket(uid, bskid):
"""Return the max rights a user has on this basket"""
query_owner = "SELECT count(id_owner) FROM bskBASKET WHERE id_owner=%s and id=%s"
params_owner = (int(uid), int(bskid))
res = run_sql(query_owner, params_owner)
if res and res[0][0]:
# if this user is owner of this baskets he can do anything he wants.
return CFG_WEBBASKET_SHARE_LEVELS['MANAGE']
# not owner => group member ?
query_group_baskets = """
SELECT share_level
FROM user_usergroup AS ug LEFT JOIN usergroup_bskBASKET AS ub
ON ug.id_usergroup=ub.id_usergroup
WHERE ug.id_user=%s AND ub.id_bskBASKET=%s AND NOT(ub.share_level='NO') AND ug.user_status!=%s
"""
params_group_baskets = (int(uid), int(bskid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query_group_baskets, params_group_baskets)
group_index = None
if res:
try:
group_index = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(res[0][0])
except:
return None
# public basket ?
query_public_baskets = """
SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0 AND id_bskBASKET=%s
"""
public_index = None
res = run_sql(query_public_baskets, (int(bskid),))
if res:
try:
public_index = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(res[0][0])
except:
return None
if group_index or public_index:
if group_index > public_index:
return CFG_WEBBASKET_SHARE_LEVELS_ORDERED[group_index]
else:
return CFG_WEBBASKET_SHARE_LEVELS_ORDERED[public_index]
return None
########################### Personal baskets ##################################
def get_personal_baskets_info_for_topic(uid, topic):
"""Return information about every basket that belongs to the given user and topic."""
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s')
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner=%s
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
WHERE ubsk.id_user=%s
AND ubsk.topic=%s
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (uid, uid, topic)
res = run_sql(query, params)
return res
def get_all_personal_basket_ids_and_names_by_topic(uid):
"""For a given user return all their personal baskets
(in tuples: (id, name)) grouped by topic. Note that the
basket tuples have to evaluated to be converted to actual
tuples."""
query = """ SELECT ubsk.topic,
count(bsk.id),
GROUP_CONCAT('(', bsk.id, ', \"', bsk.name, '\")'
ORDER BY bsk.name)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
GROUP BY ubsk.topic
ORDER BY ubsk.topic"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_personal_basket_ids_and_names_by_topic_for_add_to_list(uid):
"""For a given user return all their personal baskets
(in tuples: (id, name)) grouped by topic. Note that the
basket tuples have to evaluated to be converted to actual
tuples."""
query = """ SELECT ubsk.topic,
GROUP_CONCAT('(', bsk.id, ', \"', bsk.name, '\")'
ORDER BY bsk.name)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
GROUP BY ubsk.topic
ORDER BY ubsk.topic"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_personal_baskets_names(uid):
""" for a given user, returns every basket he is owner of
returns list of tuples: (bskid, bsk_name, topic)
"""
query = """
SELECT bsk.id,
bsk.name,
ubsk.topic
FROM user_bskBASKET ubsk JOIN bskBASKET bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
ORDER BY ubsk.topic
"""
params = (int(uid),)
return run_sql(query, params)
def get_basket_name(bskid):
"""return the name of a given basket"""
query = 'SELECT name FROM bskBASKET where id=%s'
res = run_sql(query, (int(bskid), ))
if res:
return res[0][0]
else:
return ''
def is_personal_basket_valid(uid, bskid):
"""Check if the basked (bskid) belongs to user (uid) and is valid."""
query = """ SELECT id
FROM bskBASKET
WHERE id=%s
AND id_owner=%s"""
params = (bskid, uid)
res = run_sql(query, params)
return res
def is_topic_valid(uid, topic):
"""Check if the topic defined by user (uid) exists."""
query = """ SELECT distinct(topic)
FROM user_bskBASKET
WHERE topic=%s
AND id_user=%s"""
params = (topic, uid)
res = run_sql(query, params)
return res
def get_basket_topic(uid, bskid):
"""Return the name of the topic this basket (bskid) belongs to."""
query = """ SELECT topic
FROM user_bskBASKET
WHERE id_bskBASKET=%s
AND id_user=%s"""
params = (bskid,uid)
res = run_sql(query, params)
return res
def get_personal_topics_infos(uid):
"""
Get the list of every topic user has defined,
and the number of baskets in each topic
@param uid: user id (int)
@return: a list of tuples (topic name, nb of baskets)
"""
query = """SELECT topic, count(b.id)
FROM user_bskBASKET ub JOIN bskBASKET b
ON ub.id_bskBASKET=b.id AND
b.id_owner=ub.id_user
WHERE ub.id_user=%s
GROUP BY topic
ORDER BY topic"""
uid = int(uid)
res = run_sql(query, (uid,))
return res
def rename_basket(bskid, new_name):
"""Rename basket to new_name"""
run_sql("UPDATE bskBASKET SET name=%s WHERE id=%s", (new_name, bskid))
def rename_topic(uid, old_topic, new_topic):
"""Rename topic to new_topic """
res = run_sql("UPDATE user_bskBASKET SET topic=%s WHERE id_user=%s AND topic=%s",
(new_topic, uid, old_topic))
return res
def move_baskets_to_topic(uid, bskids, new_topic):
"""Move given baskets to another topic"""
if not((type(bskids) is list) or (type(bskids) is tuple)):
bskids = [bskids]
query = "UPDATE user_bskBASKET SET topic=%s WHERE id_user=%s AND ("
query += ' OR '.join(['id_bskBASKET=%s'] * len(bskids))
query += ")"
params = (new_topic, uid) + tuple(bskids)
res = run_sql(query, params)
return res
def delete_basket(bskid):
"""Delete given basket."""
# TODO: check if any alerts are automaticly adding items to the given basket.
bskid = int(bskid)
query1 = "DELETE FROM bskBASKET WHERE id=%s"
res = run_sql(query1, (bskid,))
query2A = "SELECT id_bibrec_or_bskEXTREC FROM bskREC WHERE id_bskBASKET=%s"
local_and_external_ids = run_sql(query2A, (bskid,))
external_ids = [local_and_external_id[0] for local_and_external_id in \
local_and_external_ids if local_and_external_id[0]<0]
for external_id in external_ids:
delete_item(bskid=bskid, recid=external_id, update_date_modification=False)
query2B = "DELETE FROM bskREC WHERE id_bskBASKET=%s"
run_sql(query2B, (bskid,))
query3 = "DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET=%s"
run_sql(query3, (bskid,))
query4 = "DELETE FROM user_bskBASKET WHERE id_bskBASKET=%s"
run_sql(query4, (bskid,))
query5 = "DELETE FROM usergroup_bskBASKET WHERE id_bskBASKET=%s"
run_sql(query5, (bskid,))
query6 = "DELETE FROM user_query_basket WHERE id_basket=%s"
run_sql(query6, (bskid,))
return int(res)
def create_basket(uid, basket_name, topic):
"""Create new basket for given user in given topic"""
now = convert_datestruct_to_datetext(localtime())
id_bsk = run_sql("""INSERT INTO bskBASKET (id_owner, name, date_modification)
VALUES (%s, %s, %s)""",
(uid, basket_name, now))
run_sql("""INSERT INTO user_bskBASKET (id_user, id_bskBASKET, topic)
VALUES (%s, %s, %s)""",
(uid, id_bsk, topic))
return id_bsk
def get_all_items_in_user_personal_baskets(uid,
topic="",
format='hb'):
"""For the specified user, return all the items in their personal baskets,
grouped by basket if local or as a list if external.
If topic is set, return only that topic's items."""
if topic:
topic_clause = """AND ubsk.topic=%s"""
params_local = (uid, uid, topic)
params_external = (uid, uid, topic, format)
else:
topic_clause = ""
params_local = (uid, uid)
params_external = (uid, uid, format)
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
ubsk.topic,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%%s
%s
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET""" % (topic_clause,)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
ubsk.topic,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%%s
%s
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET""" % (topic_clause,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_personal_baskets_by_matching_notes(uid,
topic="",
p=""):
"""For the specified user, return all the items in their personal baskets
matching their notes' titles and bodies, grouped by basket.
If topic is set, return only that topic's items."""
p = p and '%' + p + '%' or '%'
if topic:
topic_clause = """AND ubsk.topic=%s"""
params = (uid, uid, topic, p, p)
else:
topic_clause = ""
params = (uid, uid, p, p)
query = """ SELECT notes.id_bskBASKET,
bsk.name,
ubsk.topic,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=notes.id_bskBASKET
AND ubsk.id_user=%%s
%s
WHERE notes.title like %%s
OR notes.body like %%s
GROUP BY notes.id_bskBASKET""" % (topic_clause,)
res = run_sql(query, params)
return res
def get_all_user_topics(uid):
"""Return a list of the user's topics."""
query = """ SELECT ubsk.topic
FROM bskBASKET AS bsk
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
GROUP BY ubsk.topic"""
params = (uid,)
res = run_sql(query, params)
return res
########################## Actions on baskets #################################
def get_basket_record(bskid, recid, format='hb'):
"""get record recid in basket bskid
"""
if recid < 0:
rec_table = 'bskEXTREC'
format_table = 'bskEXTFMT'
id_field = 'id_bskEXTREC'
sign = '-'
else:
rec_table = 'bibrec'
format_table = 'bibfmt'
id_field = 'id_bibrec'
sign = ''
query = """
SELECT DATE_FORMAT(record.creation_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
DATE_FORMAT(record.modification_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
DATE_FORMAT(bskREC.date_added, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
user.nickname,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
fmt.value
FROM bskREC LEFT JOIN user
ON bskREC.id_user_who_added_item=user.id
LEFT JOIN bskRECORDCOMMENT cmt
ON bskREC.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
LEFT JOIN %(rec_table)s record
ON (%(sign)sbskREC.id_bibrec_or_bskEXTREC=record.id)
LEFT JOIN %(format_table)s fmt
ON (record.id=fmt.%(id_field)s)
WHERE bskREC.id_bskBASKET=%%s AND
bskREC.id_bibrec_or_bskEXTREC=%%s AND
fmt.format=%%s
GROUP BY bskREC.id_bibrec_or_bskEXTREC
""" % {'rec_table': rec_table,
'sign': sign,
'format_table': format_table,
'id_field':id_field}
params = (int(bskid), int(recid), format)
res = run_sql(query, params)
if res:
return __decompress_last(res[0])
return ()
def get_basket_content(bskid, format='hb'):
"""Get all records for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
extrec.collection_id,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%Y-%%m-%%d %%H:%%i:%%s'),
extern.value as ext_val,
intern.value as int_val,
rec.score
FROM bskREC AS rec
LEFT JOIN bskRECORDCOMMENT AS cmt
ON (rec.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
AND rec.id_bskBASKET=cmt.id_bskBASKET)
LEFT JOIN bskEXTFMT AS extern
ON (-rec.id_bibrec_or_bskEXTREC=extern.id_bskEXTREC
AND extern.format=%s)
LEFT JOIN bibfmt AS intern
ON (rec.id_bibrec_or_bskEXTREC=intern.id_bibrec
AND intern.format=%s)
LEFT JOIN bskEXTREC AS extrec
ON extrec.id=-rec.id_bibrec_or_bskEXTREC
WHERE rec.id_bskBASKET=%s
GROUP BY rec.id_bibrec_or_bskEXTREC
ORDER BY rec.score"""
params = (format, format, int(bskid))
res = run_sql(query, params)
if res:
query2 = "UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"
run_sql(query2, (int(bskid),))
return res
return ()
def get_basket_item(bskid, recid, format='hb'):
"""Get item (recid) for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
extrec.collection_id,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%Y-%%m-%%d %%H:%%i:%%s'),
extern.value as ext_val,
intern.value as int_val,
rec.score
FROM bskREC rec
LEFT JOIN bskRECORDCOMMENT cmt
ON (rec.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
AND
rec.id_bskBASKET=cmt.id_bskBASKET)
LEFT JOIN bskEXTFMT extern
ON (-rec.id_bibrec_or_bskEXTREC=extern.id_bskEXTREC
AND
extern.format=%s)
LEFT JOIN bibfmt intern
ON (rec.id_bibrec_or_bskEXTREC=intern.id_bibrec
AND
intern.format=%s)
LEFT JOIN bskEXTREC AS extrec
ON extrec.id=-rec.id_bibrec_or_bskEXTREC
WHERE rec.id_bskBASKET=%s
AND rec.id_bibrec_or_bskEXTREC=%s
GROUP BY rec.id_bibrec_or_bskEXTREC
ORDER BY rec.score"""
params = (format, format, bskid, recid)
res = run_sql(query, params)
if res:
queryU = """UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"""
paramsU = (bskid,)
run_sql(queryU, paramsU)
score = res[0][6]
query_previous = """SELECT id_bibrec_or_bskEXTREC
FROM bskREC
WHERE id_bskBASKET=%s
AND score<%s
ORDER BY score DESC
LIMIT 1"""
params_previous = (bskid, score)
res_previous = run_sql(query_previous, params_previous)
query_next = """SELECT id_bibrec_or_bskEXTREC
FROM bskREC
WHERE id_bskBASKET=%s
AND score>%s
ORDER BY score ASC
LIMIT 1"""
params_next = (bskid, score)
res_next = run_sql(query_next, params_next)
query_index = """ SELECT COUNT(id_bibrec_or_bskEXTREC)
FROM bskREC
WHERE id_bskBASKET=%s
AND score<=%s
ORDER BY score"""
params_index = (bskid, score)
res_index = run_sql(query_index, params_index)
res_index = __wash_sql_count(res_index)
return (res[0], res_previous and res_previous[0][0] or 0, res_next and res_next[0][0] or 0, res_index)
else:
return ()
def share_basket_with_group(bskid, group_id,
share_level=CFG_WEBBASKET_SHARE_LEVELS['READITM']):
""" Share basket bskid with group group_id with given share_level
@param share_level: see CFG_WEBBASKET_SHARE_LEVELS in webbasket_config
"""
now = convert_datestruct_to_datetext(localtime())
run_sql("""REPLACE INTO usergroup_bskBASKET
(id_usergroup, id_bskBASKET, date_shared, share_level)
VALUES (%s,%s,%s,%s)""",
(group_id, bskid, now, str(share_level)))
def update_rights(bskid, group_rights):
"""update rights (permissions) for groups.
@param bskid: basket id
@param group_rights: dictionary of {group id: new rights}
"""
now = convert_datestruct_to_datetext(localtime())
query1 = """REPLACE INTO usergroup_bskBASKET
(id_usergroup, id_bskBASKET, date_shared, share_level)
VALUES """ + \
', '.join(["(%s, %s, %s, %s)"] * len(group_rights.items()))
params = ()
for (group_id, share_level) in group_rights.items():
params += (int(group_id), int(bskid), now, str(share_level))
run_sql(query1, params)
query2 = """DELETE FROM usergroup_bskBASKET WHERE share_level='NO'"""
run_sql(query2)
def move_item(bskid, recid, direction):
"""Change score of an item in a basket"""
bskid = int(bskid)
query1 = """SELECT id_bibrec_or_bskEXTREC,
score
FROM bskREC
WHERE id_bskBASKET=%s
ORDER BY score, date_added"""
items = run_sql(query1, (bskid,))
(recids, scores) = zip(*items)
(recids, scores) = (list(recids), list(scores))
if len(recids) and recid in recids:
current_index = recids.index(recid)
if direction == CFG_WEBBASKET_ACTIONS['UP']:
switch_index = 0
if current_index != 0:
switch_index = current_index -1
else:
switch_index = len(recids) - 1
if current_index != len(recids)-1:
switch_index = current_index + 1
query2 = """UPDATE bskREC
SET score=%s
WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"""
res1 = run_sql(query2, (scores[switch_index], bskid, recids[current_index]))
res2 = run_sql(query2, (scores[current_index], bskid, recids[switch_index]))
if res1 and res2:
now = convert_datestruct_to_datetext(localtime())
query3 = "UPDATE bskBASKET SET date_modification=%s WHERE id=%s"
params3 = (now, int(bskid))
run_sql(query3, params3)
def delete_item(bskid, recid, update_date_modification=True):
"""Remove item recid from basket bskid"""
if recid < 0:
query0A = "select count(id_bskBASKET) from bskREC where id_bibrec_or_bskEXTREC=%s" % (int(recid))
ncopies = run_sql(query0A)
if ncopies and ncopies[0][0]<=1:
# uncomment the following 5 lines and comment the following 2 to delete cached records
# only for external sources and not for external records
#query0B = "SELECT collection_id FROM bskEXTREC WHERE id=%s" % (-int(recid))
#colid = run_sql(query0B)
#if colid and colid[0][0]==0:
#query0C = "DELETE from bskEXTFMT WHERE id_bskEXTREC=%s" % (-int(recid))
#run_sql(query0C)
# the following two lines delete cached external records. We could keep them if we find
# a way to reuse them in case the external records are added again in the future.
query0D = "DELETE from bskEXTFMT WHERE id_bskEXTREC=%s" % (-int(recid))
run_sql(query0D)
query0E = "DELETE from bskEXTREC WHERE id=%s" % (-int(recid))
run_sql(query0E)
query_notes = "DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"
run_sql(query_notes, (bskid, recid,))
query1 = "DELETE from bskREC WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"
params1 = (int(bskid), int(recid))
res = run_sql(query1, params1)
if update_date_modification and res:
now = convert_datestruct_to_datetext(localtime())
query2 = "UPDATE bskBASKET SET date_modification=%s WHERE id=%s"
params2 = (now, int(bskid))
run_sql(query2, params2)
return res
def add_to_basket(uid,
recids=[],
colid=0,
bskid=0,
es_title="",
es_desc="",
es_url=""):
"""Add items (recids) basket (bskid)."""
if (recids or (colid == -1 and es_title and es_desc and es_url)) and bskid > 0:
query_max_score = """ SELECT MAX(score)
FROM bskREC
WHERE id_bskBASKET=%s"""
params_max_score = (bskid,)
res_max_score = run_sql(query_max_score, params_max_score)
max_score = __wash_sql_count(res_max_score)
if not max_score:
# max_score == None actually means that the basket doesn't exist.
# Maybe we should return 0 and inform the admin?
max_score = 1
if colid > 0:
query_existing = """ SELECT id,
external_id
FROM bskEXTREC
WHERE %s
AND collection_id=%s"""
sep_or = ' OR '
query_existing %= (sep_or.join(['external_id=%s'] * len(recids)), colid)
params_existing = tuple(recids)
res_existing = run_sql(query_existing, params_existing)
existing_recids = [int(external_ids_couple[1]) for external_ids_couple in res_existing]
existing_ids = [int(ids[0]) for ids in res_existing]
new_recids = [recid for recid in recids if int(recid) not in existing_recids]
# sets approach
#existing_recids = [ids[1] for ids in res_existing]
#new_recids = list(set(recids)-set(existing_recids))
if new_recids:
- query_new = """ INSERT INTO bskEXTREC
- (external_id,
- collection_id,
- creation_date,
- modification_date)
+ query_new = """ INSERT INTO bskEXTREC (external_id,
+ collection_id,
+ creation_date,
+ modification_date)
VALUES """
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s)"] * len(new_recids)
query_new += ', '.join(records)
params_new = ()
for new_recid in new_recids:
params_new += (int(new_recid), colid, now, now)
res_new = run_sql(query_new, params_new)
recids = [-int(recid) for recid in existing_ids]
recids.extend(range(-res_new,-(res_new+len(new_recids)),-1))
else:
recids = [-int(recid) for recid in existing_ids]
elif colid < 0:
- query_external = """INSERT INTO bskEXTREC
- (collection_id,
- original_url,
- creation_date,
- modification_date)
+ query_external = """INSERT INTO bskEXTREC (collection_id,
+ original_url,
+ creation_date,
+ modification_date)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
params_external = (colid, es_url, now, now)
res_external = run_sql(query_external, params_external)
recids = [-res_external]
store_external_source(res_external, es_title, es_desc, es_url, 'xm')
store_external_source(res_external, es_title, es_desc, es_url, 'hb')
query_insert = """ INSERT IGNORE INTO bskREC
(id_bibrec_or_bskEXTREC,
id_bskBASKET,
id_user_who_added_item,
date_added,
score)
VALUES """
if colid == 0 or (colid > 0 and not new_recids):
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s, %s)"] * len(recids)
query_insert += ', '.join(records)
params_insert = ()
i = 1
for recid in recids:
params_insert += (recid, bskid, uid, now, max_score + i)
i += 1
run_sql(query_insert, params_insert)
query_update = """ UPDATE bskBASKET
SET date_modification=%s
WHERE id=%s"""
params_update = (now, bskid)
run_sql(query_update, params_update)
return recids
return 0
def add_to_many_baskets(uid, recids=[], colid=0, bskids=[], es_title="", es_desc="", es_url=""):
"""Add items recids to every basket in bskids list."""
if (len(recids) or colid == -1) and len(bskids):
query1 = """SELECT id_bskBASKET,
max(score)
FROM bskREC
WHERE %s
GROUP BY id_bskBASKET"""
bskids = [bskid for bskid in bskids if int(bskid) >= 0]
sep_or = ' OR '
query1 %= sep_or.join(['id_bskBASKET=%s'] * len(bskids))
bsks = dict.fromkeys(bskids, 0)
params = tuple(bskids)
bsks.update(dict(run_sql(query1, params)))
if colid > 0:
query2A = """SELECT id,
external_id
FROM bskEXTREC
WHERE %s
AND collection_id=%s"""
query2A %= (sep_or.join(['external_id=%s'] * len(recids)), colid)
params2A = tuple(recids)
res2A = run_sql(query2A, params2A)
existing_recids = [int(external_ids_couple[1]) for external_ids_couple in res2A]
existing_ids = [int(ids[0]) for ids in res2A]
new_recids = [recid for recid in recids if int(recid) not in existing_recids]
# sets approach
#existing_recids = [ids[1] for ids in res2A]
#new_recids = list(set(recids)-set(existing_recids))
f = open("/tmp/bsk_db", "w")
f.write(str(recids) + "\n" + str(existing_recids) + "\n" + str(existing_ids) + "\n" + str(new_recids) + "\n")
f.close()
if new_recids:
query2B = """INSERT
INTO bskEXTREC
(external_id,
collection_id,
creation_date,
modification_date)
VALUES """
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s)"] * len(new_recids)
query2B += ', '.join(records)
params2B = ()
for new_recid in new_recids:
params2B += (int(new_recid), colid, now, now)
res = run_sql(query2B, params2B)
recids = [-int(recid) for recid in existing_ids]
recids.extend(range(-res,-(res+len(new_recids)),-1))
else:
recids = [-int(recid) for recid in existing_ids]
elif colid < 0:
query2C = """INSERT
INTO bskEXTREC
(collection_id,
original_url,
creation_date,
modification_date)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
params = (colid, es_url, now, now)
res = run_sql(query2C, params)
recids = [-res]
store_external_source(res, es_title, es_desc, es_url, 'xm')
store_external_source(res, es_title, es_desc, es_url, 'hb')
query2 = """INSERT IGNORE
INTO bskREC
(id_bibrec_or_bskEXTREC,
id_bskBASKET,
id_user_who_added_item,
date_added,
score)
VALUES """
if colid == 0 or (colid > 0 and not new_recids):
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s, %s)"] * (len(recids) * len(bsks.items()))
query2 += ', '.join(records)
params = ()
for (bskid, max_score) in bsks.items():
i = 1
for recid in recids:
params += (int(recid), int(bskid), int(uid), now, int(max_score) + i)
i += 1
run_sql(query2, params)
query3 = """UPDATE bskBASKET
SET date_modification=%s
WHERE """
query3 += sep_or.join(["id=%s"] * len(bskids))
params = (now,) + tuple(bskids)
run_sql(query3, params)
return len(bskids)
return 0
def get_external_records_by_collection(recids):
"""Get the selected recids, both local and external, grouped by collection."""
if recids:
query = """ SELECT GROUP_CONCAT(id),
GROUP_CONCAT(external_id),
collection_id
FROM bskEXTREC
WHERE %s
GROUP BY collection_id"""
recids = [-recid for recid in recids]
sep_or = ' OR '
query %= sep_or.join(['id=%s'] * len(recids))
params = tuple(recids)
res = run_sql(query,params)
return res
return 0
def get_external_records(recids, of="hb"):
"""Get formatted external records from the database."""
if recids:
query = """ SELECT rec.collection_id,
fmt.id_bskEXTREC,
fmt.value
FROM bskEXTFMT AS fmt
JOIN bskEXTREC AS rec
ON rec.id=fmt.id_bskEXTREC
WHERE format=%%s
AND ( %s )"""
recids = [-recid for recid in recids]
sep_or = ' OR '
query %= sep_or.join(['id_bskEXTREC=%s'] * len(recids))
params = [of]
params.extend(recids)
params = tuple(params)
res = run_sql(query,params)
return res
return ()
def store_external_records(records, of="hb"):
"""Store formatted external records to the database."""
if records:
query = """INSERT
INTO bskEXTFMT
(id_bskEXTREC,
format,
last_updated,
value)
VALUES """
now = convert_datestruct_to_datetext(localtime())
formatted_records = ["(%s, %s, %s, %s)"] * len(records)
query += ', '.join(formatted_records)
params = ()
for record in records:
params += (record[0], of, now, compress(record[1]))
run_sql(query,params)
def store_external_urls(ids_urls):
"""Store original urls for external records to the database."""
#for id_url in ids_urls.iteritems():
for id_url in ids_urls:
query = """UPDATE
bskEXTREC
SET original_url=%s
WHERE id=%s"""
params = (id_url[1], id_url[0])
run_sql(query,params)
def store_external_source(es_id, es_title, es_desc, es_url, of="hb"):
"""Store formatted external sources to the database."""
if es_id and es_title and es_desc:
query = """INSERT INTO bskEXTFMT
(id_bskEXTREC,
format,
last_updated,
value)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
value = create_pseudo_record(es_title, es_desc, es_url, of)
params = (es_id, of, now, compress(value))
run_sql(query,params)
def get_external_colid_and_url(recid):
"""Get the collection id and original url for an external record."""
if recid:
query = """SELECT
collection_id,
original_url
FROM bskEXTREC
WHERE id=%s"""
params = (-recid,)
res = run_sql(query,params)
if res:
return res
else:
return 0
############################ Group baskets ####################################
def get_group_baskets_info_for_group(grpid):
"""Return information about every basket that belongs to the given group,
provided the user is its manager or a member of it."""
if not grpid:
return ()
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
COUNT(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level,
bsk.id_owner
FROM usergroup_bskBASKET AS ugbsk
JOIN bskBASKET AS bsk
ON bsk.id=ugbsk.id_bskBASKET
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
WHERE ugbsk.id_usergroup=%s
AND ugbsk.share_level!='NO'
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (grpid,)
res = run_sql(query, params)
return res
def get_group_name(gid):
"""Given its id return the group's name."""
query = """ SELECT name
FROM usergroup
WHERE id=%s"""
params = (gid,)
res = run_sql(query, params)
return res
def get_all_group_basket_ids_and_names_by_group(uid):
"""For a given user return all their group baskets
(in tuples: (id, name)) grouped by group. Note that the
basket tuples have to evaluated to be converted to actual
tuples."""
query = """ SELECT ug.id,
ug.name,
count(bsk.id),
GROUP_CONCAT('(', ugbsk.id_bskBASKET, ', \"', bsk.name, '\")'
ORDER BY bsk.name)
FROM usergroup AS ug
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=ug.id
JOIN bskBASKET AS bsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON ug.id=uug.id_usergroup
AND uug.id_user=%s
GROUP BY ug.name
ORDER BY ug.name"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_group_basket_ids_and_names_by_group_for_add_to_list(uid):
"""For a given user return all their group baskets
(in tuples: (id, name)) grouped by group. Note that the
basket tuples have to evaluated to be converted to actual
tuples."""
query = """ SELECT ug.name,
GROUP_CONCAT('(', ugbsk.id_bskBASKET, ', \"', bsk.name, '\")'
ORDER BY bsk.name)
FROM usergroup AS ug
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=ug.id
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
AND ugbsk.share_level!='RC'
AND ugbsk.share_level!='AC'
JOIN bskBASKET AS bsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON ug.id=uug.id_usergroup
AND uug.id_user=%s
GROUP BY ug.name
ORDER BY ug.name"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_group_baskets_names(uid,
min_rights=CFG_WEBBASKET_SHARE_LEVELS['ADDCMT']):
"""For a given user returns every group baskets in which he can
return a list of tuples: (bskid, bsk_name, group_name)."""
# TODO: This function is no longer used. Delete if necessary.
uid = int(uid)
try:
min_rights_num = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(min_rights)
except ValueError:
return ()
groups = get_groups_user_member_of(uid)
if groups:
where_clause = '('
where_clause += " OR ".join(["ugbsk.id_usergroup=%s"] * len(groups))
where_clause += ') AND ('
where_clause += " OR ".join(["ugbsk.share_level=%s"] * len(CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:]))
where_clause += ")"
query = """
SELECT bsk.id,
bsk.name,
ug.name
FROM usergroup ug JOIN usergroup_bskBASKET ugbsk
ON ug.id=ugbsk.id_usergroup
JOIN bskBASKET bsk
ON bsk.id=ugbsk.id_bskBASKET
WHERE %s AND NOT(ugbsk.share_level='NO')
ORDER BY ug.name""" % where_clause
params = tuple([group_id for (group_id, dummy) in groups])
params += tuple(CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:])
return run_sql(query, params)
return ()
def is_shared_to(bskids):
"""For each bskid in bskids get id of one of its group. Used to
make distinction between private basket (no group), 'world' basket
(0) or group basket (any int > 0)
"""
if not((type(bskids) == list) or (type(bskids) == tuple)):
bskids = [bskids]
query = """SELECT b.id,
min(u.id_usergroup)
FROM
bskBASKET b LEFT JOIN usergroup_bskBASKET u
ON (b.id=u.id_bskBASKET) """
if len(bskids) != 0:
query += " WHERE "
query += " OR ".join(['b.id=%s'] * len(bskids))
query += " GROUP BY b.id"
params = tuple(bskids)
res = run_sql(query, params)
if res:
return res
return ()
def get_basket_share_level(bskid):
"""Get the minimum share level of the basket (bskid).
Returns:
None for personal baskets
positive integet for group baskets
0 for public baskets
Will return 0 if the basket is both group and publicly shared."""
query = """ SELECT MIN(ugbsk.id_usergroup)
FROM bskBASKET AS bsk
LEFT JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=bsk.id
WHERE bsk.id=%s
GROUP BY bsk.id"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_all_items_in_user_group_baskets(uid,
group=0,
format='hb'):
"""For the specified user, return all the items in their group baskets,
grouped by basket if local or as a list if external.
If group is set, return only that group's items."""
if group:
group_clause = """AND ubsk.id_usergroup=%s"""
params_local = (group, uid)
params_external = (group, uid, format)
else:
group_clause = ""
params_local = (uid,)
params_external = (uid, format)
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ubsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET""" % (group_clause,)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ubsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET""" % (group_clause,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_group_baskets_by_matching_notes(uid,
group=0,
p=""):
"""For the specified user, return all the items in group personal baskets
matching their notes' titles and bodies, grouped by basket.
If topic is set, return only that topic's items."""
p = p and '%' + p + '%' or '%'
if group:
group_clause = """AND ugbsk.id_usergroup=%s"""
params = (group, uid, p, p)
else:
group_clause = ""
params = (uid, p, p)
query = """ SELECT notes.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.share_level IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
WHERE notes.title like %%s
OR notes.body like %%s
GROUP BY notes.id_bskBASKET""" % (group_clause,)
res = run_sql(query, params)
return res
def is_group_basket_valid(uid, bskid):
"""Check if the basked (bskid) belongs to one of the groups the user (uid)
is a member of and is valid."""
query = """ SELECT id
FROM bskBASKET AS bsk
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%s
WHERE id=%s"""
params = (uid, bskid)
res = run_sql(query, params)
return res
def is_group_valid(uid, group):
"""Check if the group exists and the user is a member or manager."""
query = """ SELECT id_usergroup
FROM user_usergroup
WHERE id_usergroup=%s
AND id_user=%s"""
params = (group, uid)
res = run_sql(query, params)
return res
def get_all_user_groups(uid):
"""Return a list of the groups the user is a member of or manages."""
query = """ SELECT ug.id,
ug.name
FROM user_usergroup AS uug
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=uug.id_usergroup
WHERE uug.id_user=%s
GROUP BY uug.id_usergroup"""
params = (uid,)
res = run_sql(query, params)
return res
########################## External baskets ###################################
def get_external_baskets_infos(uid):
"""Get general informations about every external basket user uid has subscribed to."""
query = """
SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level
FROM bskBASKET bsk JOIN user_bskBASKET ubsk
ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s)
LEFT JOIN bskREC rec
ON (bsk.id=rec.id_bskBASKET)
LEFT JOIN usergroup_bskBASKET ugbsk
ON (ugbsk.id_bskBASKET=bsk.id AND ugbsk.id_usergroup=0)
WHERE bsk.id_owner!=%s
GROUP BY bsk.id
"""
uid = int(uid)
params = (uid, uid)
res = run_sql(query, params)
if res:
return res
return ()
def get_external_basket_info(bskid):
""""""
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level
FROM bskBASKET AS bsk
LEFT JOIN bskREC AS rec
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON bsk.id=ugbsk.id_bskBASKET
AND ugbsk.id_usergroup=0
JOIN user_bskBASKET AS ubsk
ON bsk.id_owner!=ubsk.id_user
AND bsk.id=ubsk.id_bskBASKET
WHERE id=%s
GROUP BY bsk.id"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_all_external_basket_ids_and_names(uid):
"""For a given user return all their external baskets
(in tuples: (id, name, number_of_records))."""
query = """ SELECT bsk.id,
bsk.name,
count(rec.id_bibrec_or_bskEXTREC),
ugbsk.id_usergroup
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user!=bsk.id_owner
LEFT JOIN bskREC AS rec
ON ubsk.id_bskBASKET=rec.id_bskBASKET
LEFT JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=0
AND ugbsk.id_bskBASKET=bsk.id
WHERE ubsk.id_user=%s
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (uid,)
res = run_sql(query, params)
return res
def count_external_baskets(uid):
"""Returns the number of external baskets the user is subscribed to."""
query = """ SELECT COUNT(ubsk.id_bskBASKET)
FROM user_bskBASKET ubsk
LEFT JOIN bskBASKET bsk
ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s)
WHERE bsk.id_owner!=%s"""
params = (int(uid), int(uid))
res = run_sql(query, params)
return __wash_sql_count(res)
def get_all_external_baskets_names(uid,
min_rights=CFG_WEBBASKET_SHARE_LEVELS['ADDCMT']):
""" for a given user returns every basket which he has subscribed to and in which
he can
return a list of tuples: (bskid, bsk_name)
"""
uid = int(uid)
try:
min_rights_num = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(min_rights)
except ValueError:
return ()
where_clause = ' AND ('
for right in CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:-1]:
where_clause += "ugbsk.share_level = '%s' OR " % right
where_clause += "ugbsk.share_level = '%s')" % CFG_WEBBASKET_SHARE_LEVELS_ORDERED[-1]
query = """
SELECT bsk.id,
bsk.name
FROM bskBASKET bsk JOIN usergroup_bskBASKET ugbsk
ON bsk.id=ugbsk.id_bskBASKET
JOIN user_bskBASKET ubsk
ON ubsk.id_bskBASKET=bsk.id
WHERE ugbsk.id_usergroup=0 AND
ubsk.id_user=%s AND
NOT(bsk.id_owner=%s) AND
NOT(ugbsk.share_level='NO')
""" + where_clause
params = (uid, uid)
return run_sql(query, params)
def get_all_items_in_user_public_baskets(uid,
format='hb'):
"""For the specified user, return all the items in the public baskets they
are subscribed to, grouped by basket if local or as a list if external."""
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET"""
params_local = (uid, uid)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
params_external = (uid, uid, format)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_public_baskets_by_matching_notes(uid,
p=""):
"""For the specified user, return all the items in the public baskets they
are subscribed to, matching their notes' titles and bodies,
grouped by basket"""
p = p and '%' + p + '%' or '%'
query = """ SELECT notes.id_bskBASKET,
bsk.name,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=notes.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.id_usergroup=0
AND ugbsk.share_level IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
WHERE notes.title like %s
OR notes.body like %s
GROUP BY notes.id_bskBASKET"""
params = (uid, uid, p, p)
res = run_sql(query, params)
return res
def get_all_items_in_all_public_baskets(format='hb'):
"""Return all the items in all the public baskets,
grouped by basket if local or as a list if external."""
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET"""
res_local = run_sql(query_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
params_external = (format,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_all_public_baskets_by_matching_notes(p=""):
"""For the specified user, return all the items in the public baskets they
are subscribed to, matching their notes' titles and bodies,
grouped by basket"""
p = p and '%' + p + '%' or '%'
query = """ SELECT notes.id_bskBASKET,
bsk.name,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.id_usergroup=0
AND ugbsk.share_lelel IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
WHERE notes.title like %s
OR notes.body like %s
GROUP BY notes.id_bskBASKET"""
params = (p, p)
res = run_sql(query, params)
return res
############################ Public access ####################################
def get_public_basket_infos(bskid):
"""return (id, name, date modification, nb of views, id of owner, nickname of owner, rights for public access)
for a given basket"""
basket = []
query1 = """SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
bsk.id_owner,
user.nickname
FROM bskBASKET bsk LEFT JOIN user
ON bsk.id_owner=user.id
WHERE bsk.id=%s"""
res1 = run_sql(query1, (int(bskid),))
if len(res1):
basket = list(res1[0])
query2 = """SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0 and id_bskBASKET=%s"""
res2 = run_sql(query2, (int(bskid),))
if res2:
basket.append(res2[0][0])
else:
basket.append(None)
return basket
def get_public_basket_info(bskid):
"""Return information about a given public basket."""
query = """ SELECT bsk.id,
bsk.name,
bsk.id_owner,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
COUNT(rec.id_bibrec_or_bskEXTREC),
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC),
ubsk.share_level
FROM bskBASKET AS bsk
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
JOIN usergroup_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_usergroup=0
WHERE bsk.id=%s
GROUP BY bsk.id;"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_basket_general_infos(bskid):
"""return information about a basket, suited for public access.
@return: a (id, name, date of modification, nb of views, nb of records, id of owner) tuple
"""
query = """SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
bsk.id_owner
FROM bskBASKET bsk LEFT JOIN bskREC rec
ON bsk.id=rec.id_bskBASKET
WHERE bsk.id=%s
GROUP BY bsk.id"""
res = run_sql(query, (int(bskid),))
if res:
query2 = "UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"
run_sql(query2, (int(bskid),))
return res[0]
return ()
def get_basket_owner_id(bskid):
"""Return the uid of the owner."""
query = """SELECT id_owner
FROM bskBASKET
WHERE id=%s"""
res = run_sql(query, (bskid, ))
if res:
return res[0][0]
return -1
def count_public_baskets():
"""Returns the number of public baskets."""
query = """ SELECT COUNT(id_bskBASKET)
FROM usergroup_bskBASKET
WHERE id_usergroup=0"""
res = run_sql(query)
return __wash_sql_count(res)
def get_public_baskets_list(inf_limit, max_number, order=1, asc=1):
"""Return list of public baskets
@param inf_limit: limit to baskets from number x
@param max_number: number of baskets to return
@order: 1: order by name of basket, 2: number of views, 3: owner
@return:
[(basket id, basket name, nb of views, uid of owner, nickname of owner)]"""
query = """SELECT bsk.id,
bsk.name,
bsk.nb_views,
u.id,
u.nickname
FROM bskBASKET bsk LEFT JOIN usergroup_bskBASKET ugbsk
on bsk.id=ugbsk.id_bskBASKET
LEFT JOIN user u
on bsk.id_owner=u.id
WHERE ugbsk.id_usergroup=0
"""
if order == 2:
query += 'ORDER BY bsk.nb_views'
elif order == 3:
query += 'ORDER BY u.nickname'
if asc:
query += ' ASC'
else:
query += ' DESC'
query += ', u.id'
else:
query += 'ORDER BY bsk.name'
if asc:
query += ' ASC '
else:
query += ' DESC '
query += "LIMIT %s,%s"
return run_sql(query, (inf_limit, max_number))
def count_all_public_baskets():
"""Return the number of all the public baskets."""
query = """ SELECT count(id_bskBASKET)
FROM usergroup_bskBASKET
WHERE id_usergroup=0"""
res = run_sql(query)
return __wash_sql_count(res)
def get_list_public_baskets(page, max_number, sort='name', asc=1):
"""Return list of public baskets
@param page: limit to baskets from number x
@param max_number: maximum number of baskets to return
@sort: 1: order by name of basket, 2: number of views, 3: owner
@return:
[(basket id, basket name, nb of views, uid of owner, nickname of owner)]"""
query = """ SELECT bsk.id,
bsk.name,
bsk.id_owner,
u.nickname,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
COUNT(rec.id_bibrec_or_bskEXTREC) AS items,
bsk.nb_views
FROM usergroup_bskBASKET AS ugbsk
JOIN bskBASKET AS bsk
ON bsk.id=ugbsk.id_bskBASKET
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
LEFT JOIN user AS u
ON u.id=bsk.id_owner
WHERE ugbsk.id_usergroup=0
GROUP BY bsk.id"""
if sort == 'name':
query += """
ORDER BY bsk.name"""
elif sort == 'owner':
query += """
ORDER BY u.nickname"""
elif sort == 'views':
query += """
ORDER BY bsk.nb_views"""
elif sort == 'date':
query += """
ORDER BY bsk.date_modification"""
elif sort == 'items':
query += """
ORDER BY items"""
else:
query += """
ORDER BY bsk.name"""
if asc:
query += """ ASC"""
if sort == """owner""":
query += """, u.id"""
else:
query += """ DESC"""
if sort == """owner""":
query += """, u.id"""
query += """
LIMIT %s, %s"""
page = max(0, page)
res = run_sql(query, (page, max_number))
return res
def is_basket_public(bskid):
"""Check if the given basket is public.
Returns ((0,),) if False, ((1,),) if True."""
query = """ SELECT COUNT(*)
FROM usergroup_bskBASKET
WHERE id_usergroup=0
AND id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return __wash_sql_count(res)
def subscribe(uid, bskid):
"""Subscribe the given user to the given basket."""
query1 = """SELECT COUNT(*)
FROM user_bskBASKET
WHERE id_user=%s
AND id_bskBASKET=%s"""
params1 = (uid, bskid)
res1 = run_sql(query1, params1)
if res1[0][0]:
# The user is either the owner of the basket or is already subscribed.
return False
else:
- query2 = """INSERT INTO user_bskBASKET
- (id_user, id_bskBASKET)
- VALUES (%s, %s)"""
+ query2 = """INSERT INTO user_bskBASKET (id_user, id_bskBASKET)
+ VALUES (%s, %s)"""
params2 = (uid, bskid)
run_sql(query2, params2)
return True
def unsubscribe(uid, bskid):
"""Unsubscribe the given user from the given basket."""
query1 = """SELECT COUNT(*)
FROM bskBASKET
WHERE id_owner=%s
AND id=%s"""
params1 = (uid, bskid)
res1 = run_sql(query1, params1)
if res1[0][0]:
# The user is the owner of the basket.
return False
else:
query2 = """DELETE FROM user_bskBASKET
WHERE id_user=%s
AND id_bskBASKET=%s"""
params2 = (uid, bskid)
res2 = run_sql(query2, params2)
if res2:
return True
else:
return False
def is_user_subscribed_to_basket(uid, bskid):
"""Return ((1,),) if the user is subscribed to the given basket
or ((0,),) if the user is not subscribed or is the owner of the basket."""
query = """ SELECT COUNT(ubsk.id_bskBASKET)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner!=ubsk.id_user
WHERE ubsk.id_user=%s
AND ubsk.id_bskBASKET=%s"""
params = (uid, bskid)
res = run_sql(query, params)
return __wash_sql_count(res)
def count_subscribers(uid, bskid):
"""Returns a (number of users, number of groups, number of alerts) tuple
for the given user (uid) and basket (bskid)."""
uid = int(uid)
bskid = int(bskid)
query_groups = """ SELECT count(id_usergroup)
FROM usergroup_bskBASKET
WHERE id_bskBASKET=%s
AND NOT(share_level='NO')
GROUP BY id_bskBASKET"""
params_groups = (bskid,)
res_groups = run_sql(query_groups, params_groups)
nb_groups = __wash_sql_count(res_groups)
query_users = """ SELECT count(id_user)
FROM user_bskBASKET
WHERE id_bskBASKET=%s
AND id_user!=%s
GROUP BY id_bskBASKET"""
params_users = (bskid, uid)
res_users = run_sql(query_users, params_users)
nb_users = __wash_sql_count(res_users)
query_alerts = """ SELECT count(id_query)
FROM user_query_basket
WHERE id_basket=%s
GROUP BY id_basket"""
params_alerts = (bskid,)
res_alerts = run_sql(query_alerts, params_alerts)
nb_alerts = __wash_sql_count(res_alerts)
return (nb_users, nb_groups, nb_alerts)
def get_groups_subscribing_to_basket(bskid):
""" get list of (group id, group name, rights) tuples for a given basket
Please note that group 0 is used to mean everybody.
"""
query = """SELECT ugb.id_usergroup,
ug.name,
ugb.share_level
FROM usergroup_bskBASKET ugb LEFT JOIN usergroup ug
ON ugb.id_usergroup=ug.id
WHERE ugb.id_bskBASKET=%s
ORDER BY ugb.id_usergroup"""
return run_sql(query, (int(bskid),))
def get_rights_on_public_basket(bskid):
""""""
query = """ SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0
AND id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return res
def count_public_basket_subscribers(bskid):
"""Return the number of users subscribed to the given public basket."""
query = """ SELECT COUNT(ubsk.id_user)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner!=ubsk.id_user
WHERE ubsk.id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return __wash_sql_count(res)
################################ Notes ########################################
def get_notes(bskid, recid):
"""Return all comments for record recid in basket bskid."""
query = """
SELECT user.id,
user.nickname,
bskcmt.title,
bskcmt.body,
DATE_FORMAT(bskcmt.date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
bskcmt.priority,
bskcmt.id
FROM bskRECORDCOMMENT bskcmt LEFT JOIN user
ON (bskcmt.id_user=user.id)
WHERE bskcmt.id_bskBASKET=%s AND
bskcmt.id_bibrec_or_bskEXTREC=%s
ORDER BY bskcmt.date_creation
"""
bskid = int(bskid)
recid = int(recid)
res = run_sql(query, (bskid, recid))
if res:
return res
else:
return ()
def get_note(cmtid):
"""Return comment cmtid as a (author's nickname, author's uid, title, body, date of creation, priority) tuple"""
out = ()
query = """
SELECT user.nickname,
user.id,
bskcmt.title,
bskcmt.body,
DATE_FORMAT(bskcmt.date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
bskcmt.priority
FROM bskRECORDCOMMENT bskcmt LEFT JOIN user
ON (bskcmt.id_user=user.id)
WHERE bskcmt.id=%s
"""
cmtid = int(cmtid)
res = run_sql(query, (cmtid,))
if res:
return res[0]
return out
def save_note(uid, bskid, recid, title, body):
"""Save then given note (title, body) on the given item in the given basket."""
date = convert_datestruct_to_datetext(localtime())
res = run_sql("""INSERT INTO bskRECORDCOMMENT (id_user, id_bskBASKET,
id_bibrec_or_bskEXTREC, title, body, date_creation)
VALUES (%s, %s, %s, %s, %s, %s)""",
(int(uid), int(bskid), int(recid), title, body, date))
if res:
return int(res)
return 0
def delete_note(bskid, recid, cmtid):
"""Delete a comment on an item of a basket"""
query = """ DELETE
FROM bskRECORDCOMMENT
WHERE id_bskBASKET=%s
AND id_bibrec_or_bskEXTREC=%s
AND id=%s"""
params = (int(bskid), int(recid), int(cmtid))
run_sql(query, params)
def note_belongs_to_item_in_basket_p(cmtid, recid, bskid):
"""Returns 1 (True) if the given note (cmtid) belongs to the given item
(recid) and the given basket (bskid) or 0 (False)."""
query = """ SELECT COUNT(*)
FROM bskRECORDCOMMENT
WHERE id=%s
AND id_bibrec_or_bskEXTREC=%s
AND id_bskBASKET=%s"""
params = (cmtid, recid, bskid)
res = run_sql(query, params)
return __wash_sql_count(res)
########################## Usergroup functions ################################
def get_group_infos(uid):
"""For each group the user with uid is a member of return the id, name and number of baskets."""
query = """SELECT g.id,
g.name,
count(ugb.id_bskBASKET)
FROM usergroup g LEFT JOIN (user_usergroup ug,
usergroup_bskBASKET ugb)
ON (g.id=ug.id_usergroup
AND
g.id=ugb.id_usergroup)
WHERE ug.id_user=%s AND NOT(ugb.share_level='NO') AND ug.user_status!=%s
GROUP BY g.id
ORDER BY g.name"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
return res
def count_groups_user_member_of(uid):
"""Returns the number of groups the user has joined."""
query = """ SELECT COUNT(id_usergroup)
FROM user_usergroup
WHERE id_user=%s
AND user_status!=%s"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
return __wash_sql_count(res)
def get_groups_user_member_of(uid):
"""
Get uids and names of groups user is member of.
@param uid: user id (int)
@return: a tuple of (group_id, group_name) tuples
"""
query = """
SELECT g.id,
g.name
FROM usergroup g JOIN user_usergroup ug
ON (g.id=ug.id_usergroup)
WHERE ug.id_user=%s and ug.user_status!=%s
ORDER BY g.name
"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
if res:
return res
return ()
########################## auxilliary functions ###############################
def __wash_sql_count(res):
"""Wash the result of SQL COUNT function and return only an integer."""
if res:
return res[0][0]
return 0
def __decompress_last(item):
"""private function, used to shorten code"""
item = list(item)
item[-1] = decompress(item[-1])
return item
def create_pseudo_record(es_title, es_desc, es_url, of="hb"):
"""Return a pseudo record representation given a title and a description."""
if of == 'hb':
record = '\n'.join([es_title, es_desc, es_url])
if of == 'xm':
# In case we want to use the controlfield,
# the -es_id must be used.
#%s
record = """%s%s%s""" % (encode_for_xml(es_title), encode_for_xml(es_desc), es_url)
return record
def prettify_url(url, char_limit=50, nb_dots=3):
"""If the url has more characters than char_limit return a shortened version of it
keeping the beginning and ending and replacing the rest with dots."""
if len(url) > char_limit:
# let's set a minimum character limit
if char_limit < 5:
char_limit = 5
# let's set a maximum number of dots in relation to the character limit
if nb_dots > char_limit/4:
nb_dots = char_limit/5
nb_char_url = char_limit - nb_dots
nb_char_end = nb_char_url/4
nb_char_beg = nb_char_url - nb_char_end
return url[:nb_char_beg] + '.'*nb_dots + url[-nb_char_end:]
else:
return url
diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py
index cd0a2ebaf..aa4c13e5b 100644
--- a/modules/webcomment/lib/webcomment.py
+++ b/modules/webcomment/lib/webcomment.py
@@ -1,1448 +1,1447 @@
# -*- coding: utf-8 -*-
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
""" Comments and reviews for records """
__revision__ = "$Id$"
# non CDS Invenio imports:
import time
import math
from datetime import datetime, timedelta
# CDS Invenio imports:
from invenio.dbquery import run_sql
from invenio.config import CFG_SITE_LANG, \
CFG_WEBALERT_ALERT_ENGINE_EMAIL,\
CFG_SITE_ADMIN_EMAIL,\
CFG_SITE_SUPPORT_EMAIL,\
CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,\
CFG_SITE_URL,\
CFG_SITE_NAME,\
CFG_WEBCOMMENT_ALLOW_REVIEWS,\
CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS,\
CFG_WEBCOMMENT_ALLOW_COMMENTS,\
CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL,\
CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,\
CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS,\
CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_REVIEWS_IN_SECONDS,\
CFG_WEBCOMMENT_DEFAULT_MODERATOR
from invenio.webmessage_mailutils import \
email_quote_txt, \
email_quoted_txt2html
from invenio.webuser import get_user_info, get_email, collect_user_info
from invenio.dateutils import convert_datetext_to_dategui, \
datetext_default, \
convert_datestruct_to_datetext
from invenio.mailutils import send_email
from invenio.messages import wash_language, gettext_set_language
from invenio.urlutils import wash_url_argument
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.webcomment_config import CFG_WEBCOMMENT_ACTION_CODE, \
CFG_WEBCOMMENT_EMAIL_REPLIES_TO
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_admin import acc_is_role
from invenio.access_control_config import CFG_WEBACCESS_WARNING_MSGS
from invenio.search_engine import \
guess_primary_collection_of_a_record, \
check_user_can_view_record, \
get_all_collections_of_a_record, \
get_fieldvalues
from invenio.webcomment_washer import EmailWasher
try:
import invenio.template
webcomment_templates = invenio.template.load('webcomment')
except:
pass
def perform_request_display_comments_or_remarks(req, recID, display_order='od', display_since='all', nb_per_page=100, page=1, ln=CFG_SITE_LANG, voted=-1, reported=-1, subscribed=0, reviews=0, uid=-1, can_send_comments=False, can_attach_files=False, user_is_subscribed_to_discussion=False, user_can_unsubscribe_from_discussion=False):
"""
Returns all the comments (reviews) of a specific internal record or external basket record.
@param recID: record id where (internal record IDs > 0) or (external basket record IDs < -100)
@param display_order: hh = highest helpful score, review only
lh = lowest helpful score, review only
hs = highest star score, review only
ls = lowest star score, review only
od = oldest date
nd = newest date
@param display_since: all= no filtering by date
nd = n days ago
nw = n weeks ago
nm = n months ago
ny = n years ago
where n is a single digit integer between 0 and 9
@param nb_per_page: number of results per page
@param page: results page
@param voted: boolean, active if user voted for a review, see perform_request_vote function
@param reported: boolean, active if user reported a certain comment/review, perform_request_report function
@param subscribed: int, 1 if user just subscribed to discussion, -1 if unsubscribed
@param reviews: boolean, enabled if reviews, disabled for comments
@param uid: the id of the user who is reading comments
@param can_send_comments: if user can send comment or not
@param can_attach_files: if user can attach file to comment or not
@param user_is_subscribed_to_discussion: True if user already receives new comments by email
@param user_can_unsubscribe_from_discussion: True is user is allowed to unsubscribe from discussion
@return html body.
"""
errors = []
warnings = []
nb_reviews = 0
nb_comments = 0
# wash arguments
recID = wash_url_argument(recID, 'int')
ln = wash_language(ln)
display_order = wash_url_argument(display_order, 'str')
display_since = wash_url_argument(display_since, 'str')
nb_per_page = wash_url_argument(nb_per_page, 'int')
page = wash_url_argument(page, 'int')
voted = wash_url_argument(voted, 'int')
reported = wash_url_argument(reported, 'int')
reviews = wash_url_argument(reviews, 'int')
# vital argument check
(valid, error_body) = check_recID_is_in_range(recID, warnings, ln)
if not(valid):
return (error_body, errors, warnings)
# Query the database and filter results
res = query_retrieve_comments_or_remarks(recID, display_order, display_since, reviews)
res2 = query_retrieve_comments_or_remarks(recID, display_order, display_since, not reviews)
nb_res = len(res)
if reviews:
nb_reviews = nb_res
nb_comments = len(res2)
else:
nb_reviews = len(res2)
nb_comments = nb_res
# checking non vital arguemnts - will be set to default if wrong
#if page <= 0 or page.lower() != 'all':
if page < 0:
page = 1
warnings.append(('WRN_WEBCOMMENT_INVALID_PAGE_NB',))
if nb_per_page < 0:
nb_per_page = 100
warnings.append(('WRN_WEBCOMMENT_INVALID_NB_RESULTS_PER_PAGE',))
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
if display_order not in ['od', 'nd', 'hh', 'lh', 'hs', 'ls']:
display_order = 'hh'
warnings.append(('WRN_WEBCOMMENT_INVALID_REVIEW_DISPLAY_ORDER',))
else:
if display_order not in ['od', 'nd']:
display_order = 'od'
warnings.append(('WRN_WEBCOMMENT_INVALID_DISPLAY_ORDER',))
# filter results according to page and number of reults per page
if nb_per_page > 0:
if nb_res > 0:
last_page = int(math.ceil(nb_res / float(nb_per_page)))
else:
last_page = 1
if page > last_page:
page = 1
warnings.append(("WRN_WEBCOMMENT_INVALID_PAGE_NB",))
if nb_res > nb_per_page: # if more than one page of results
if page < last_page:
res = res[(page-1)*(nb_per_page) : (page*nb_per_page)]
else:
res = res[(page-1)*(nb_per_page) : ]
else: # one page of results
pass
else:
last_page = 1
# Send to template
avg_score = 0.0
if not CFG_WEBCOMMENT_ALLOW_COMMENTS and not CFG_WEBCOMMENT_ALLOW_REVIEWS: # comments not allowed by admin
errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
if reported > 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED',))
elif reported == 0:
warnings.append(('WRN_WEBCOMMENT_ALREADY_REPORTED',))
elif reported == -2:
warnings.append(('WRN_WEBCOMMENT_INVALID_REPORT',))
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
avg_score = calculate_avg_score(res)
if voted > 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED',))
elif voted == 0:
warnings.append(('WRN_WEBCOMMENT_ALREADY_VOTED',))
if subscribed == 1:
warnings.append(('WRN_WEBCOMMENT_SUBSCRIBED',))
elif subscribed == -1:
warnings.append(('WRN_WEBCOMMENT_UNSUBSCRIBED',))
body = webcomment_templates.tmpl_get_comments(req,
recID,
ln,
nb_per_page, page, last_page,
display_order, display_since,
CFG_WEBCOMMENT_ALLOW_REVIEWS,
res, nb_comments, avg_score,
warnings,
border=0,
reviews=reviews,
total_nb_reviews=nb_reviews,
uid=uid,
can_send_comments=can_send_comments,
can_attach_files=can_attach_files,
user_is_subscribed_to_discussion=\
user_is_subscribed_to_discussion,
user_can_unsubscribe_from_discussion=\
user_can_unsubscribe_from_discussion)
return (body, errors, warnings)
def perform_request_vote(cmt_id, client_ip_address, value, uid=-1):
"""
Vote positively or negatively for a comment/review
@param cmt_id: review id
@param value: +1 for voting positively
-1 for voting negatively
@return: integer 1 if successful, integer 0 if not
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
value = wash_url_argument(value, 'int')
uid = wash_url_argument(uid, 'int')
if cmt_id > 0 and value in [-1, 1] and check_user_can_vote(cmt_id, client_ip_address, uid):
action_date = convert_datestruct_to_datetext(time.localtime())
action_code = CFG_WEBCOMMENT_ACTION_CODE['VOTE']
query = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT,
id_bibrec, id_user, client_host, action_time,
action_code)
VALUES (%s, NULL ,%s, inet_aton(%s), %s, %s)"""
params = (cmt_id, uid, client_ip_address, action_date, action_code)
run_sql(query, params)
return query_record_useful_review(cmt_id, value)
else:
return 0
def check_user_can_comment(recID, client_ip_address, uid=-1):
""" Check if a user hasn't already commented within the last seconds
time limit: CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS
@param recID: record id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
recID = wash_url_argument(recID, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
max_action_time = time.time() - CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS
max_action_time = convert_datestruct_to_datetext(time.localtime(max_action_time))
action_code = CFG_WEBCOMMENT_ACTION_CODE['ADD_COMMENT']
query = """SELECT id_bibrec
FROM cmtACTIONHISTORY
WHERE id_bibrec=%s AND
action_code=%s AND
action_time>%s
"""
params = (recID, action_code, max_action_time)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return len(res) == 0
def check_user_can_review(recID, client_ip_address, uid=-1):
""" Check if a user hasn't already reviewed within the last seconds
time limit: CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_REVIEWS_IN_SECONDS
@param recID: record ID
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
action_code = CFG_WEBCOMMENT_ACTION_CODE['ADD_REVIEW']
query = """SELECT id_bibrec
FROM cmtACTIONHISTORY
WHERE id_bibrec=%s AND
action_code=%s
"""
params = (recID, action_code)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return len(res) == 0
def check_user_can_vote(cmt_id, client_ip_address, uid=-1):
""" Checks if a user hasn't already voted
@param cmt_id: comment id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
query = """SELECT id_cmtRECORDCOMMENT
FROM cmtACTIONHISTORY
WHERE id_cmtRECORDCOMMENT=%s"""
params = (cmt_id,)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid, )
res = run_sql(query, params)
return (len(res) == 0)
def get_comment_collection(cmt_id):
"""
Extract the collection where the comment is written
"""
query = "SELECT id_bibrec FROM cmtRECORDCOMMENT WHERE id=%s"
recid = run_sql(query, (cmt_id,))
record_primary_collection = guess_primary_collection_of_a_record(recid[0][0])
return record_primary_collection
def get_collection_moderators(collection):
"""
Return the list of comment moderators for the given collection.
"""
from invenio.access_control_engine import acc_get_authorized_emails
res = list(acc_get_authorized_emails('moderatecomments', collection=collection))
if not res:
return [CFG_WEBCOMMENT_DEFAULT_MODERATOR,]
return res
def perform_request_report(cmt_id, client_ip_address, uid=-1):
"""
Report a comment/review for inappropriate content.
Will send an email to the administrator if number of reports is a multiple of CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN
@param cmt_id: comment id
@return: integer 1 if successful, integer 0 if not. -2 if comment does not exist
"""
cmt_id = wash_url_argument(cmt_id, 'int')
if cmt_id <= 0:
return 0
(query_res, nb_abuse_reports) = query_record_report_this(cmt_id)
if query_res == 0:
return 0
elif query_res == -2:
return -2
if not(check_user_can_report(cmt_id, client_ip_address, uid)):
return 0
action_date = convert_datestruct_to_datetext(time.localtime())
action_code = CFG_WEBCOMMENT_ACTION_CODE['REPORT_ABUSE']
query = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT, id_bibrec,
id_user, client_host, action_time, action_code)
VALUES (%s, NULL, %s, inet_aton(%s), %s, %s)"""
params = (cmt_id, uid, client_ip_address, action_date, action_code)
run_sql(query, params)
if nb_abuse_reports % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN == 0:
(cmt_id2,
id_bibrec,
id_user,
cmt_body,
cmt_date,
cmt_star,
cmt_vote, cmt_nb_votes_total,
cmt_title,
cmt_reported) = query_get_comment(cmt_id)
(user_nb_abuse_reports,
user_votes,
user_nb_votes_total) = query_get_user_reports_and_votes(int(id_user))
(nickname, user_email, last_login) = query_get_user_contact_info(id_user)
from_addr = '%s Alert Engine <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(cmt_id)
to_addrs = get_collection_moderators(comment_collection)
subject = "A comment has been reported as inappropriate by a user"
body = '''
The following comment has been reported a total of %(cmt_reported)s times.
Author: nickname = %(nickname)s
email = %(user_email)s
user_id = %(uid)s
This user has:
total number of reports = %(user_nb_abuse_reports)s
%(votes)s
Comment: comment_id = %(cmt_id)s
record_id = %(id_bibrec)s
date written = %(cmt_date)s
nb reports = %(cmt_reported)s
%(review_stuff)s
body =
---start body---
%(cmt_body)s
---end body---
Please go to the record page %(comment_admin_link)s to delete this message if necessary. A warning will be sent to the user in question.''' % \
{ 'cfg-report_max' : CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,
'nickname' : nickname,
'user_email' : user_email,
'uid' : id_user,
'user_nb_abuse_reports' : user_nb_abuse_reports,
'user_votes' : user_votes,
'votes' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"total number of positive votes\t= %s\n\t\ttotal number of negative votes\t= %s" % \
(user_votes, (user_nb_votes_total - user_votes)) or "\n",
'cmt_id' : cmt_id,
'id_bibrec' : id_bibrec,
'cmt_date' : cmt_date,
'cmt_reported' : cmt_reported,
'review_stuff' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"star score\t= %s\n\treview title\t= %s" % (cmt_star, cmt_title) or "",
'cmt_body' : cmt_body,
'comment_admin_link' : CFG_SITE_URL + "/record/" + str(id_bibrec) + '/comments#' + str(cmt_id),
'user_admin_link' : "user_admin_link" #! FIXME
}
#FIXME to be added to email when websession module is over:
#If you wish to ban the user, you can do so via the User Admin Panel %(user_admin_link)s.
send_email(from_addr, to_addrs, subject, body)
return 1
def check_user_can_report(cmt_id, client_ip_address, uid=-1):
""" Checks if a user hasn't already reported a comment
@param cmt_id: comment id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
query = """SELECT id_cmtRECORDCOMMENT
FROM cmtACTIONHISTORY
WHERE id_cmtRECORDCOMMENT=%s"""
params = (uid,)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return (len(res) == 0)
def query_get_user_contact_info(uid):
"""
Get the user contact information
@return: tuple (nickname, email, last_login), if none found return ()
Note: for the moment, if no nickname, will return email address up to the '@'
"""
query1 = """SELECT nickname, email,
DATE_FORMAT(last_login, '%%Y-%%m-%%d %%H:%%i:%%s')
FROM user WHERE id=%s"""
params1 = (uid,)
res1 = run_sql(query1, params1)
if res1:
return res1[0]
else:
return ()
def query_get_user_reports_and_votes(uid):
"""
Retrieve total number of reports and votes of a particular user
@param uid: user id
@return: tuple (total_nb_reports, total_nb_votes_yes, total_nb_votes_total)
if none found return ()
"""
query1 = """SELECT nb_votes_yes,
nb_votes_total,
nb_abuse_reports
FROM cmtRECORDCOMMENT
WHERE id_user=%s"""
params1 = (uid,)
res1 = run_sql(query1, params1)
if len(res1) == 0:
return ()
nb_votes_yes = nb_votes_total = nb_abuse_reports = 0
for cmt_tuple in res1:
nb_votes_yes += int(cmt_tuple[0])
nb_votes_total += int(cmt_tuple[1])
nb_abuse_reports += int(cmt_tuple[2])
return (nb_abuse_reports, nb_votes_yes, nb_votes_total)
def query_get_comment(comID):
"""
Get all fields of a comment
@param comID: comment id
@return: tuple (comID, id_bibrec, id_user, body, date_creation, star_score, nb_votes_yes, nb_votes_total, title, nb_abuse_reports)
if none found return ()
"""
query1 = """SELECT id,
id_bibrec,
id_user,
body,
DATE_FORMAT(date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
star_score,
nb_votes_yes,
nb_votes_total,
title,
nb_abuse_reports
FROM cmtRECORDCOMMENT
WHERE id=%s"""
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1)>0:
return res1[0]
else:
return ()
def query_record_report_this(comID):
"""
Increment the number of reports for a comment
@param comID: comment id
@return: tuple (success, new_total_nb_reports_for_this_comment) where
success is integer 1 if success, integer 0 if not, -2 if comment does not exist
"""
#retrieve nb_abuse_reports
query1 = "SELECT nb_abuse_reports FROM cmtRECORDCOMMENT WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1) == 0:
return (-2, 0)
#increment and update
nb_abuse_reports = int(res1[0][0]) + 1
query2 = "UPDATE cmtRECORDCOMMENT SET nb_abuse_reports=%s WHERE id=%s"
params2 = (nb_abuse_reports, comID)
res2 = run_sql(query2, params2)
return (int(res2), nb_abuse_reports)
def query_record_useful_review(comID, value):
"""
private funciton
Adjust the number of useful votes and number of total votes for a comment.
@param comID: comment id
@param value: +1 or -1
@return: integer 1 if successful, integer 0 if not
"""
# retrieve nb_useful votes
query1 = "SELECT nb_votes_total, nb_votes_yes FROM cmtRECORDCOMMENT WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1)==0:
return 0
# modify and insert new nb_useful votes
nb_votes_yes = int(res1[0][1])
if value >= 1:
nb_votes_yes = int(res1[0][1]) + 1
nb_votes_total = int(res1[0][0]) + 1
query2 = "UPDATE cmtRECORDCOMMENT SET nb_votes_total=%s, nb_votes_yes=%s WHERE id=%s"
params2 = (nb_votes_total, nb_votes_yes, comID)
res2 = run_sql(query2, params2)
return int(res2)
def query_retrieve_comments_or_remarks (recID, display_order='od', display_since='0000-00-00 00:00:00',
ranking=0):
"""
Private function
Retrieve tuple of comments or remarks from the database
@param recID: record id
@param display_order: hh = highest helpful score
lh = lowest helpful score
hs = highest star score
ls = lowest star score
od = oldest date
nd = newest date
@param display_since: datetime, e.g. 0000-00-00 00:00:00
@param ranking: boolean, enabled if reviews, disabled for comments
@return: tuple of comment where comment is
tuple (nickname, uid, date_creation, body, status, id) if ranking disabled or
tuple (nickname, uid, date_creation, body, status, nb_votes_yes, nb_votes_total, star_score, title, id)
Note: for the moment, if no nickname, will return email address up to '@'
"""
display_since = calculate_start_date(display_since)
order_dict = { 'hh' : "cmt.nb_votes_yes/(cmt.nb_votes_total+1) DESC, cmt.date_creation DESC ",
'lh' : "cmt.nb_votes_yes/(cmt.nb_votes_total+1) ASC, cmt.date_creation ASC ",
'ls' : "cmt.star_score ASC, cmt.date_creation DESC ",
'hs' : "cmt.star_score DESC, cmt.date_creation DESC ",
'od' : "cmt.date_creation ASC ",
'nd' : "cmt.date_creation DESC "
}
# Ranking only done for comments and when allowed
if ranking and recID > 0:
try:
display_order = order_dict[display_order]
except:
display_order = order_dict['od']
else:
# in case of recID > 0 => external record => no ranking!
ranking = 0
try:
if display_order[-1] == 'd':
display_order = order_dict[display_order]
else:
display_order = order_dict['od']
except:
display_order = order_dict['od']
#display_order = order_dict['nd']
query = """SELECT user.nickname,
cmt.id_user,
DATE_FORMAT(cmt.date_creation, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
cmt.body,
cmt.status,
cmt.nb_abuse_reports,
%(ranking)s cmt.id
FROM %(table)s cmt LEFT JOIN user ON
user.id=cmt.id_user
WHERE %(id_bibrec)s=%%s
%(ranking_only)s
%(display_since)s
ORDER BY %%s
""" % {'ranking' : ranking and ' cmt.nb_votes_yes, cmt.nb_votes_total, cmt.star_score, cmt.title, ' or '',
'ranking_only' : ranking and ' AND cmt.star_score>0 ' or ' AND cmt.star_score=0 ',
'id_bibrec' : recID > 0 and 'cmt.id_bibrec' or 'cmt.id_bibrec_or_bskEXTREC',
'table' : recID > 0 and 'cmtRECORDCOMMENT' or 'bskRECORDCOMMENT',
'display_since' : display_since == '0000-00-00 00:00:00' and ' ' or 'AND cmt.date_creation>=\'%s\' ' % display_since}
params = (recID, display_order)
res = run_sql(query, params)
if res:
return res
return ()
def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
note="", score=0, priority=0,
client_ip_address='', editor_type='textarea'):
"""
Private function
Insert a comment/review or remarkinto the database
@param recID: record id
@param uid: user id
@param msg: comment body
@param note: comment title
@param score: review star score
@param priority: remark priority #!FIXME
@param editor_type: the kind of editor used to submit the comment: 'textarea', 'fckeditor'
@return: integer >0 representing id if successful, integer 0 if not
"""
current_date = calculate_start_date('0d')
#change utf-8 message into general unicode
msg = msg.decode('utf-8')
note = note.decode('utf-8')
#change general unicode back to utf-8
msg = msg.encode('utf-8')
note = note.encode('utf-8')
if editor_type == 'fckeditor':
# Here we remove the line feeds introduced by FCKeditor (they
# have no meaning for the user) and replace the HTML line
# breaks by linefeeds, so that we are close to an input that
# would be done without the FCKeditor. That's much better if a
# reply to a comment is made with a browser that does not
# support FCKeditor.
msg = msg.replace('\n', '').replace('\r', '').replace(' ', '\n')
query = """INSERT INTO cmtRECORDCOMMENT (id_bibrec,
id_user,
body,
date_creation,
star_score,
nb_votes_total,
title)
VALUES (%s, %s, %s, %s, %s, %s, %s)"""
params = (recID, uid, msg, current_date, score, 0, note)
res = run_sql(query, params)
if res:
action_code = CFG_WEBCOMMENT_ACTION_CODE[reviews and 'ADD_REVIEW' or 'ADD_COMMENT']
action_time = convert_datestruct_to_datetext(time.localtime())
query2 = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT,
id_bibrec, id_user, client_host, action_time, action_code)
VALUES ('', %s, %s, inet_aton(%s), %s, %s)"""
params2 = (recID, uid, client_ip_address, action_time, action_code)
run_sql(query2, params2)
# Email this comment to 'subscribers'
(subscribers_emails1, subscribers_emails2) = \
get_users_subscribed_to_discussion(recID)
email_subscribers_about_new_comment(recID, reviews=reviews,
emails1=subscribers_emails1,
emails2=subscribers_emails2,
comID=res, msg=msg,
note=note, score=score,
editor_type=editor_type, uid=uid)
return int(res)
def subscribe_user_to_discussion(recID, uid):
"""
Subscribe a user to a discussion, so the she receives by emails
all new new comments for this record.
@param recID: record ID corresponding to the discussion we want to
subscribe the user
@param uid: user id
"""
- query = """INSERT INTO cmtSUBSCRIPTION
- (id_bibrec, id_user, creation_time)
+ query = """INSERT INTO cmtSUBSCRIPTION (id_bibrec, id_user, creation_time)
VALUES (%s, %s, %s)"""
params = (recID, uid, convert_datestruct_to_datetext(time.localtime()))
try:
- res = run_sql(query, params)
+ run_sql(query, params)
except:
return 0
return 1
def unsubscribe_user_from_discussion(recID, uid):
"""
Unsubscribe users from a discussion.
@param recID: record ID corresponding to the discussion we want to
unsubscribe the user
@param uid: user id
@return 1 if successful, 0 if not
"""
query = """DELETE FROM cmtSUBSCRIPTION
WHERE id_bibrec=%s AND id_user=%s"""
params = (recID, uid)
try:
res = run_sql(query, params)
except:
return 0
if res > 0:
return 1
return 0
def get_user_subscription_to_discussion(recID, uid):
"""
Returns the type of subscription for the given user to this
discussion.
@param recID: record ID
@param uid: user id
@return:
- 0 if user is not subscribed to discussion
- 1 if user is subscribed, and is allowed to unsubscribe
- 2 if user is subscribed, but cannot unsubscribe
"""
user_email = get_email(uid)
(emails1, emails2) = get_users_subscribed_to_discussion(recID, check_authorizations=False)
if user_email in emails1:
return 1
elif user_email in emails2:
return 2
else:
return 0
def get_users_subscribed_to_discussion(recID, check_authorizations=True):
"""
Returns the lists of users subscribed to a given discussion.
Two lists are returned: the first one is the list of emails for
users who can unsubscribe from the discussion, the second list
contains the emails of users who cannot unsubscribe (for eg. author
of the document, etc).
Users appear in only one list. If a user has manually subscribed
to a discussion AND is an automatic recipients for updates, it
will only appear in the second list.
@param recID: record ID for which we want to retrieve subscribed users
@param check_authorizations: if True, check again if users are authorized to view comment
@return tuple (emails1, emails2)
"""
subscribers_emails = {}
# Get users that have subscribed to this discussion
query = """SELECT id_user FROM cmtSUBSCRIPTION WHERE id_bibrec=%s"""
params = (recID,)
res = run_sql(query, params)
for row in res:
uid = row[0]
user_info = collect_user_info(uid)
(auth_code, auth_msg) = check_user_can_view_comments(user_info, recID)
if auth_code:
# User is no longer authorized to view comments.
# Delete subscription
unsubscribe_user_from_discussion(recID, uid)
else:
email = get_email(uid)
if '@' in email:
subscribers_emails[email] = True
# Get users automatically subscribed, based on the record metadata
collections = get_all_collections_of_a_record(recID)
primary_collection = guess_primary_collection_of_a_record(recID)
if primary_collection not in collections:
collections.append(primary_collection)
for collection in collections:
if CFG_WEBCOMMENT_EMAIL_REPLIES_TO.has_key(collection):
fields = CFG_WEBCOMMENT_EMAIL_REPLIES_TO[collection]
for field in fields:
emails = get_fieldvalues(recID, field)
for email in emails:
if not '@' in email:
# Is a group: add domain name
subscribers_emails[email + '@' + \
CFG_SITE_SUPPORT_EMAIL.split('@')[1]] = False
else:
subscribers_emails[email] = False
return ([email for email, can_unsubscribe_p \
in subscribers_emails.iteritems() if can_unsubscribe_p],
[email for email, can_unsubscribe_p \
in subscribers_emails.iteritems() if not can_unsubscribe_p] )
def email_subscribers_about_new_comment(recID, reviews, emails1,
emails2, comID, msg="",
note="", score=0,
editor_type='textarea',
ln=CFG_SITE_LANG, uid=-1):
"""
Notify subscribers that a new comment was posted.
FIXME: consider recipient preference to send email in correct language.
@param recID: record id
@param emails1: list of emails for users who can unsubscribe from discussion
@param emails2: list of emails for users who cannot unsubscribe from discussion
@param comID: the comment id
@param msg: comment body
@param note: comment title
@param score: review star score
@param editor_type: the kind of editor used to submit the comment: 'textarea', 'fckeditor'
@rtype: bool
@return: True if email was sent okay, False if it was not.
"""
_ = gettext_set_language(ln)
if not emails1 and not emails2:
return 0
# Get title
titles = get_fieldvalues(recID, "245__a")
if not titles:
# usual title not found, try conference title:
titles = get_fieldvalues(recID, "111__a")
title = ''
if titles:
title = titles[0]
else:
title = _("Record %i") % recID
# Get report number
report_numbers = get_fieldvalues(recID, "037__a")
if not report_numbers:
report_numbers = get_fieldvalues(recID, "088__a")
if not report_numbers:
report_numbers = get_fieldvalues(recID, "021__a")
# Prepare email subject and body
if reviews:
email_subject = _('%(report_number)s"%(title)s" has been reviewed') % \
{'report_number': report_numbers and ('[' + report_numbers[0] + '] ') or '',
'title': title}
else:
email_subject = _('%(report_number)s"%(title)s" has been commented') % \
{'report_number': report_numbers and ('[' + report_numbers[0] + '] ') or '',
'title': title}
washer = EmailWasher()
msg = washer.wash(msg)
email_content = msg
if note:
email_content = note + email_content
# Send emails to people who can unsubscribe
email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=True,
ln=ln,
uid=uid)
email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=True,
ln=ln)
res1 = True
if emails1:
res1 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
toaddr=emails1,
subject=email_subject,
content=email_content,
header=email_header,
footer=email_footer,
ln=ln)
# Then send email to people who have been automatically
# subscribed to the discussion (they cannot unsubscribe)
email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=False,
ln=ln,
uid=uid)
email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=False,
ln=ln)
res2 = True
if emails2:
res2 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
toaddr=emails2,
subject=email_subject,
content=email_content,
header=email_header,
footer=email_footer,
ln=ln)
return res1 and res2
def calculate_start_date(display_since):
"""
Private function
Returns the datetime of display_since argument in MYSQL datetime format
calculated according to the local time.
@param display_since: = all= no filtering
nd = n days ago
nw = n weeks ago
nm = n months ago
ny = n years ago
where n is a single digit number
@return: string of wanted datetime.
If 'all' given as argument, will return datetext_default
datetext_default is defined in miscutils/lib/dateutils and
equals 0000-00-00 00:00:00 => MySQL format
If bad arguement given, will return datetext_default
"""
time_types = {'d':0, 'w':0, 'm':0, 'y':0}
today = datetime.today()
try:
nb = int(display_since[:-1])
except:
return datetext_default
if (display_since==(None or 'all')):
return datetext_default
if str(display_since[-1]) in time_types:
time_type = str(display_since[-1])
else:
return datetext_default
# year
if time_type == 'y':
if (int(display_since[:-1]) > today.year - 1) or (int(display_since[:-1]) < 1):
# 1 < nb years < 2008
return datetext_default
else:
final_nb_year = today.year - nb
yesterday = today.replace(year=final_nb_year)
# month
elif time_type == 'm':
# to convert nb of monthes in years
nb_year = nb / 12 # nb_year = number of year to substract
nb = nb % 12
if nb > today.month-1: # ex: july(07)-9 monthes = -1year -3monthes
nb_year += 1
nb_month = 12 - (today.month % nb)
else:
nb_month = today.month - nb
final_nb_year = today.year - nb_year # final_nb_year = number of year to print
yesterday = today.replace(year=final_nb_year, month=nb_month)
# week
elif time_type == 'w':
delta = timedelta(weeks=nb)
yesterday = today - delta
# day
elif time_type == 'd':
delta = timedelta(days=nb)
yesterday = today - delta
return yesterday.strftime("%Y-%m-%d %H:%M:%S")
def count_comments(recID):
"""
Returns the number of comments made on a record.
"""
recID = int(recID)
query = """SELECT count(id) FROM cmtRECORDCOMMENT
WHERE id_bibrec=%s AND star_score=0"""
return run_sql(query, (recID,))[0][0]
def count_reviews(recID):
"""
Returns the number of reviews made on a record.
"""
recID = int(recID)
query = """SELECT count(id) FROM cmtRECORDCOMMENT
WHERE id_bibrec=%s AND star_score>0"""
return run_sql(query, (recID,))[0][0]
def get_first_comments_or_remarks(recID=-1,
ln=CFG_SITE_LANG,
nb_comments='all',
nb_reviews='all',
voted=-1,
reported=-1):
"""
Gets nb number comments/reviews or remarks.
In the case of comments, will get both comments and reviews
Comments and remarks sorted by most recent date, reviews sorted by highest helpful score
@param recID: record id
@param ln: language
@param nb_comments: number of comment or remarks to get
@param nb_reviews: number of reviews or remarks to get
@param voted: 1 if user has voted for a remark
@param reported: 1 if user has reported a comment or review
@return: if comment, tuple (comments, reviews) both being html of first nb comments/reviews
if remark, tuple (remakrs, None)
"""
warnings = []
errors = []
voted = wash_url_argument(voted, 'int')
reported = wash_url_argument(reported, 'int')
## check recID argument
if type(recID) is not int:
return ()
if recID >= 1: #comment or review. NB: suppressed reference to basket (handled in webbasket)
if CFG_WEBCOMMENT_ALLOW_REVIEWS:
res_reviews = query_retrieve_comments_or_remarks(recID=recID, display_order="hh", ranking=1)
nb_res_reviews = len(res_reviews)
## check nb argument
if type(nb_reviews) is int and nb_reviews < len(res_reviews):
first_res_reviews = res_reviews[:nb_reviews]
else:
first_res_reviews = res_reviews
if CFG_WEBCOMMENT_ALLOW_COMMENTS:
res_comments = query_retrieve_comments_or_remarks(recID=recID, display_order="od", ranking=0)
nb_res_comments = len(res_comments)
## check nb argument
if type(nb_comments) is int and nb_comments < len(res_comments):
first_res_comments = res_comments[:nb_comments]
else:
first_res_comments = res_comments
else: #error
errors.append(('ERR_WEBCOMMENT_RECID_INVALID', recID)) #!FIXME dont return error anywhere since search page
# comment
if recID >= 1:
comments = reviews = ""
if reported > 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED_GREEN_TEXT',))
elif reported == 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_NOT_RECORDED_RED_TEXT',))
if CFG_WEBCOMMENT_ALLOW_COMMENTS: # normal comments
comments = webcomment_templates.tmpl_get_first_comments_without_ranking(recID, ln, first_res_comments, nb_res_comments, warnings)
if CFG_WEBCOMMENT_ALLOW_REVIEWS: # ranked comments
#calculate average score
avg_score = calculate_avg_score(res_reviews)
if voted > 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_RECORDED_GREEN_TEXT',))
elif voted == 0:
warnings.append(('WRN_WEBCOMMENT_FEEDBACK_NOT_RECORDED_RED_TEXT',))
reviews = webcomment_templates.tmpl_get_first_comments_with_ranking(recID, ln, first_res_reviews, nb_res_reviews, avg_score, warnings)
return (comments, reviews)
# remark
else:
return(webcomment_templates.tmpl_get_first_remarks(first_res_comments, ln, nb_res_comments), None)
def calculate_avg_score(res):
"""
private function
Calculate the avg score of reviews present in res
@param res: tuple of tuple returned from query_retrieve_comments_or_remarks
@return: a float of the average score rounded to the closest 0.5
"""
c_star_score = 6
avg_score = 0.0
nb_reviews = 0
for comment in res:
if comment[c_star_score] > 0:
avg_score += comment[c_star_score]
nb_reviews += 1
if nb_reviews == 0:
return 0.0
avg_score = avg_score / nb_reviews
avg_score_unit = avg_score - math.floor(avg_score)
if avg_score_unit < 0.25:
avg_score = math.floor(avg_score)
elif avg_score_unit > 0.75:
avg_score = math.floor(avg_score) + 1
else:
avg_score = math.floor(avg_score) + 0.5
if avg_score > 5:
avg_score = 5.0
return avg_score
def perform_request_add_comment_or_remark(recID=0,
uid=-1,
action='DISPLAY',
ln=CFG_SITE_LANG,
msg=None,
score=None,
note=None,
priority=None,
reviews=0,
comID=-1,
client_ip_address=None,
editor_type='textarea',
can_attach_files=False,
subscribe=False):
"""
Add a comment/review or remark
@param recID: record id
@param uid: user id
@param action: 'DISPLAY' to display add form
'SUBMIT' to submit comment once form is filled
'REPLY' to reply to an existing comment
@param ln: language
@param msg: the body of the comment/review or remark
@param score: star score of the review
@param note: title of the review
@param priority: priority of remark (int)
@param reviews: boolean, if enabled will add a review, if disabled will add a comment
@param comID: if replying, this is the comment id of the commetn are replying to
@param editor_type: the kind of editor/input used for the comment: 'textarea', 'fckeditor'
@param can_attach_files: if user can attach files to comments or not
@param subscribe: if True, subscribe user to receive new comments by email
@return:
- html add form if action is display or reply
- html successful added form if action is submit
"""
warnings = []
errors = []
actions = ['DISPLAY', 'REPLY', 'SUBMIT']
_ = gettext_set_language(ln)
## check arguments
check_recID_is_in_range(recID, warnings, ln)
if uid <= 0:
errors.append(('ERR_WEBCOMMENT_UID_INVALID', uid))
return ('', errors, warnings)
user_contact_info = query_get_user_contact_info(uid)
nickname = ''
if user_contact_info:
if user_contact_info[0]:
nickname = user_contact_info[0]
# show the form
if action == 'DISPLAY':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return (webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files), errors, warnings)
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
return (webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files), errors, warnings)
else:
errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
elif action == 'REPLY':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
errors.append(('ERR_WEBCOMMENT_REPLY_REVIEW',))
return (webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files), errors, warnings)
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
textual_msg = msg
if comID > 0:
comment = query_get_comment(comID)
if comment:
user_info = get_user_info(comment[2])
if user_info:
date_creation = convert_datetext_to_dategui(str(comment[4]))
# Build two msg: one mostly textual, the other one with HTML markup, for the FCKeditor.
msg = _("%(x_name)s wrote on %(x_date)s:")% {'x_name': user_info[2], 'x_date': date_creation}
textual_msg = msg
# 1 For FCKeditor input
msg += '
'
msg += comment[3]
msg = email_quote_txt(text=msg)
msg = email_quoted_txt2html(text=msg)
msg = ' ' + msg + ' '
# 2 For textarea input
textual_msg += "\n\n"
textual_msg += comment[3]
textual_msg = email_quote_txt(text=textual_msg)
return (webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, textual_msg, can_attach_files=can_attach_files), errors, warnings)
else:
errors.append(('ERR_WEBCOMMENT_COMMENTS_NOT_ALLOWED',))
# check before submitting form
elif action == 'SUBMIT':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
if note.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
warnings.append(('WRN_WEBCOMMENT_ADD_NO_TITLE',))
if score == 0 or score > 5:
warnings.append(("WRN_WEBCOMMENT_ADD_NO_SCORE",))
if msg.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
warnings.append(('WRN_WEBCOMMENT_ADD_NO_BODY',))
# if no warnings, submit
if len(warnings) == 0:
if reviews:
if check_user_can_review(recID, client_ip_address, uid):
success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
note=note, score=score, priority=0,
client_ip_address=client_ip_address,
editor_type=editor_type)
else:
warnings.append('WRN_WEBCOMMENT_CANNOT_REVIEW_TWICE')
success = 1
else:
if check_user_can_comment(recID, client_ip_address, uid):
success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
note=note, score=score, priority=0,
client_ip_address=client_ip_address,
editor_type=editor_type)
if success > 0 and subscribe:
subscribe_user_to_discussion(recID, uid)
else:
warnings.append('WRN_WEBCOMMENT_TIMELIMIT')
success = 1
if success > 0:
if CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL > 0:
notify_admin_of_new_comment(comID=success)
return (webcomment_templates.tmpl_add_comment_successful(recID, ln, reviews, warnings, success), errors, warnings)
else:
errors.append(('ERR_WEBCOMMENT_DB_INSERT_ERROR'))
# if are warnings or if inserting comment failed, show user where warnings are
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return (webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files), errors, warnings)
else:
return (webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files), errors, warnings)
# unknown action send to display
else:
warnings.append(('WRN_WEBCOMMENT_ADD_UNKNOWN_ACTION',))
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return (webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, ln, msg, score, note, warnings, can_attach_files=can_attach_files), errors, warnings)
else:
return (webcomment_templates.tmpl_add_comment_form(recID, uid, ln, msg, warnings, can_attach_files=can_attach_files), errors, warnings)
return ('', errors, warnings)
def notify_admin_of_new_comment(comID):
"""
Sends an email to the admin with details regarding comment with ID = comID
"""
comment = query_get_comment(comID)
if len(comment) > 0:
(comID2,
id_bibrec,
id_user,
body,
date_creation,
star_score, nb_votes_yes, nb_votes_total,
title,
nb_abuse_reports) = comment
else:
return
user_info = query_get_user_contact_info(id_user)
if len(user_info) > 0:
(nickname, email, last_login) = user_info
if not len(nickname) > 0:
nickname = email.split('@')[0]
else:
nickname = email = last_login = "ERROR: Could not retrieve"
review_stuff = '''
Star score = %s
Title = %s''' % (star_score, title)
washer = EmailWasher()
body = washer.wash(body)
record_info = webcomment_templates.tmpl_email_new_comment_admin(id_bibrec)
out = '''
The following %(comment_or_review)s has just been posted (%(date)s).
AUTHOR:
Nickname = %(nickname)s
Email = %(email)s
User ID = %(uid)s
RECORD CONCERNED:
Record ID = %(recID)s
%(record_details)s
%(comment_or_review_caps)s:
%(comment_or_review)s ID = %(comID)s %(review_stuff)s
Body =
<--------------->
%(body)s
<--------------->
ADMIN OPTIONS:
To moderate the %(comment_or_review)s go to %(siteurl)s/record/%(recID)s/%(comments_or_reviews)s/display?%(arguments)s
''' % \
{ 'comment_or_review' : star_score > 0 and 'review' or 'comment',
'comment_or_review_caps': star_score > 0 and 'REVIEW' or 'COMMENT',
'comments_or_reviews' : star_score > 0 and 'reviews' or 'comments',
'date' : date_creation,
'nickname' : nickname,
'email' : email,
'uid' : id_user,
'recID' : id_bibrec,
'record_details' : record_info,
'comID' : comID2,
'review_stuff' : star_score > 0 and review_stuff or "",
'body' : body.replace(' ','\n'),
'siteurl' : CFG_SITE_URL,
'arguments' : 'ln=en&do=od#%s' % comID
}
from_addr = '%s WebComment <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(comID)
to_addrs = get_collection_moderators(comment_collection)
rec_collection = guess_primary_collection_of_a_record(id_bibrec)
report_nums = get_fieldvalues(id_bibrec, "037__a")
report_nums += get_fieldvalues(id_bibrec, "088__a")
report_nums = ', '.join(report_nums)
subject = "A new comment/review has just been posted [%s|%s]" % (rec_collection, report_nums)
send_email(from_addr, to_addrs, subject, out)
def check_recID_is_in_range(recID, warnings=[], ln=CFG_SITE_LANG):
"""
Check that recID is >= 0
Append error messages to errors listi
@param recID: record id
@param warnings: the warnings list of the calling function
@return: tuple (boolean, html) where boolean (1=true, 0=false)
and html is the body of the page to display if there was a problem
"""
# Make errors into a list if needed
if type(warnings) is not list:
errors = [warnings]
try:
recID = int(recID)
except:
pass
if type(recID) is int:
if recID > 0:
from invenio.search_engine import record_exists
success = record_exists(recID)
if success == 1:
return (1,"")
else:
warnings.append(('ERR_WEBCOMMENT_RECID_INEXISTANT', recID))
return (0, webcomment_templates.tmpl_record_not_found(status='inexistant', recID=recID, ln=ln))
elif recID == 0:
warnings.append(('ERR_WEBCOMMENT_RECID_MISSING',))
return (0, webcomment_templates.tmpl_record_not_found(status='missing', recID=recID, ln=ln))
else:
warnings.append(('ERR_WEBCOMMENT_RECID_INVALID', recID))
return (0, webcomment_templates.tmpl_record_not_found(status='invalid', recID=recID, ln=ln))
else:
warnings.append(('ERR_WEBCOMMENT_RECID_NAN', recID))
return (0, webcomment_templates.tmpl_record_not_found(status='nan', recID=recID, ln=ln))
def check_int_arg_is_in_range(value, name, errors, gte_value, lte_value=None):
"""
Check that variable with name 'name' >= gte_value and optionally <= lte_value
Append error messages to errors list
@param value: variable value
@param name: variable name
@param errors: list of error tuples (error_id, value)
@param gte_value: greater than or equal to value
@param lte_value: less than or equal to value
@return: boolean (1=true, 0=false)
"""
# Make errors into a list if needed
if type(errors) is not list:
errors = [errors]
if type(value) is not int or type(gte_value) is not int:
errors.append(('ERR_WEBCOMMENT_PROGRAMNING_ERROR',))
return 0
if type(value) is not int:
errors.append(('ERR_WEBCOMMENT_ARGUMENT_NAN', value))
return 0
if value < gte_value:
errors.append(('ERR_WEBCOMMENT_ARGUMENT_INVALID', value))
return 0
if lte_value:
if type(lte_value) is not int:
errors.append(('ERR_WEBCOMMENT_PROGRAMNING_ERROR',))
return 0
if value > lte_value:
errors.append(('ERR_WEBCOMMENT_ARGUMENT_INVALID', value))
return 0
return 1
def get_mini_reviews(recid, ln=CFG_SITE_LANG):
"""
Returns the web controls to add reviews to a record from the
detailed record pages mini-panel.
@param recid: the id of the displayed record
@param ln: the user's language
"""
if CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
action = 'SUBMIT'
else:
action = 'DISPLAY'
reviews = query_retrieve_comments_or_remarks(recid, ranking=1)
return webcomment_templates.tmpl_mini_review(recid, ln, action=action,
avg_score=calculate_avg_score(reviews),
nb_comments_total=len(reviews))
def check_user_can_view_comments(user_info, recid):
"""Check if the user is authorized to view comments for given
recid.
Returns the same type as acc_authorize_action
"""
# Check user can view the record itself first
(auth_code, auth_msg) = check_user_can_view_record(user_info, recid)
if auth_code:
return (auth_code, auth_msg)
# Check if user can view the comments
## But first can we find an authorization for this case action,
## for this collection?
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'viewcomment', authorized_if_no_roles=True, collection=record_primary_collection)
def check_user_can_send_comments(user_info, recid):
"""Check if the user is authorized to comment the given
recid. This function does not check that user can view the record
or view the comments
Returns the same type as acc_authorize_action
"""
## First can we find an authorization for this case, action + collection
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'sendcomment', authorized_if_no_roles=True, collection=record_primary_collection)
def check_user_can_attach_file_to_comments(user_info, recid):
"""Check if the user is authorized to attach a file to comments
for given recid. This function does not check that user can view
the comments or send comments.
Returns the same type as acc_authorize_action
"""
## First can we find an authorization for this case action, for
## this collection?
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'attachcommentfile', authorized_if_no_roles=True, collection=record_primary_collection)
diff --git a/modules/websearch/lib/websearchadminlib.py b/modules/websearch/lib/websearchadminlib.py
index 39b0c2fad..12ae7da26 100644
--- a/modules/websearch/lib/websearchadminlib.py
+++ b/modules/websearch/lib/websearchadminlib.py
@@ -1,3440 +1,3440 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable-msg=C0301
"""CDS Invenio WebSearch Administrator Interface."""
__revision__ = "$Id$"
import cgi
import random
import time
import sys
if sys.hexversion < 0x2040000:
# pylint: disable-msg=W0622
from sets import Set as set
# pylint: enable-msg=W0622
from invenio.config import \
CFG_CACHEDIR, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_SITE_URL,\
CFG_WEBCOMMENT_ALLOW_COMMENTS,\
CFG_WEBCOMMENT_ALLOW_REVIEWS,\
CFG_INSPIRE_SITE
from invenio.bibrankadminlib import \
write_outcome, \
modify_translations, \
get_def_name, \
get_name, \
get_languages, \
addadminbox, \
tupletotable, \
createhiddenform
from invenio.dbquery import \
run_sql, \
get_table_update_time
from invenio.websearch_external_collections import \
external_collections_dictionary, \
external_collection_sort_engine_by_name, \
external_collection_get_state, \
external_collection_get_update_state_list, \
external_collection_apply_changes
from invenio.websearch_external_collections_utils import \
get_collection_descendants
from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_STATES_NAME
#from invenio.bibformat_elements import bfe_references
#from invenio.bibformat_engine import BibFormatObject
from invenio.bibdocfile import BibRecDocs
from invenio.messages import gettext_set_language
#from invenio.bibrank_citation_searcher import get_cited_by
from invenio.access_control_admin import acc_get_action_id
from invenio.access_control_config import VIEWRESTRCOLL
from invenio.errorlib import register_exception
def getnavtrail(previous = ''):
"""Get the navtrail"""
navtrail = """Admin Area """ % (CFG_SITE_URL,)
navtrail = navtrail + previous
return navtrail
def perform_modifytranslations(colID, ln, sel_type='', trans=[], confirm=-1, callback='yes'):
"""Modify the translations of a collection
sel_type - the nametype to modify
trans - the translations in the same order as the languages from get_languages()"""
output = ''
subtitle = ''
sitelangs = get_languages()
if confirm in ["2", 2] and colID:
finresult = modify_translations(colID, sitelangs, sel_type, trans, "collection")
col_dict = dict(get_def_name('', "collection"))
if colID and col_dict.has_key(int(colID)):
colID = int(colID)
subtitle = """3. Modify translations for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
if type(trans) is str:
trans = [trans]
if sel_type == '':
sel_type = get_col_nametypes()[0][0]
header = ['Language', 'Translation']
actions = []
types = get_col_nametypes()
if len(types) > 1:
text = """
Name type
"""
output += createhiddenform(action="modifytranslations#3",
text=text,
button="Select",
colID=colID,
ln=ln,
confirm=0)
if confirm in [-1, "-1", 0, "0"]:
trans = []
for (key, value) in sitelangs:
try:
trans_names = get_name(colID, key, sel_type, "collection")
trans.append(trans_names[0][0])
except StandardError, e:
trans.append('')
for nr in range(0, len(sitelangs)):
actions.append(["%s %s" % (sitelangs[nr][1], (sitelangs[nr][0]==CFG_SITE_LANG and '(def)' or ''))])
actions[-1].append('' % trans[nr])
text = tupletotable(header=header, tuple=actions)
output += createhiddenform(action="modifytranslations#3",
text=text,
button="Modify",
colID=colID,
sel_type=sel_type,
ln=ln,
confirm=2)
if sel_type and len(trans) and confirm in ["2", 2]:
output += write_outcome(finresult)
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_modifytranslations", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_modifyrankmethods(colID, ln, func='', rnkID='', confirm=0, callback='yes'):
"""Modify which rank methods is visible to the collection
func - remove or add rank method
rnkID - the id of the rank method."""
output = ""
subtitle = ""
col_dict = dict(get_def_name('', "collection"))
rnk_dict = dict(get_def_name('', "rnkMETHOD"))
if colID and col_dict.has_key(int(colID)):
colID = int(colID)
if func in ["0", 0] and confirm in ["1", 1]:
finresult = attach_rnk_col(colID, rnkID)
elif func in ["1", 1] and confirm in ["1", 1]:
finresult = detach_rnk_col(colID, rnkID)
subtitle = """9. Modify rank options for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
output = """
The rank methods enabled for the collection '%s' is:
"""
if not rnkmethods:
output += """No rank methods"""
else:
for id, name in rnkmethods:
output += """%s, """ % name
output += """
"""
rnk_list = get_def_name('', "rnkMETHOD")
rnk_dict_in_col = dict(get_col_rnk(colID, ln))
rnk_list = filter(lambda x: not rnk_dict_in_col.has_key(x[0]), rnk_list)
if rnk_list:
text = """
Enable:
"""
output += createhiddenform(action="modifyrankmethods#9",
text=text,
button="Enable",
colID=colID,
ln=ln,
func=0,
confirm=1)
if confirm in ["1", 1] and func in ["0", 0] and int(rnkID) != -1:
output += write_outcome(finresult)
elif confirm not in ["0", 0] and func in ["0", 0]:
output += """Please select a rank method."""
coll_list = get_col_rnk(colID, ln)
if coll_list:
text = """
Disable:
"""
output += createhiddenform(action="modifyrankmethods#9",
text=text,
button="Disable",
colID=colID,
ln=ln,
func=1,
confirm=1)
if confirm in ["1", 1] and func in ["1", 1] and int(rnkID) != -1:
output += write_outcome(finresult)
elif confirm not in ["0", 0] and func in ["1", 1]:
output += """Please select a rank method."""
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_modifyrankmethods", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_addcollectiontotree(colID, ln, add_dad='', add_son='', rtype='', mtype='', callback='yes', confirm=-1):
"""Form to add a collection to the tree.
add_dad - the dad to add the collection to
add_son - the collection to add
rtype - add it as a regular or virtual
mtype - add it to the regular or virtual tree."""
output = ""
output2 = ""
subtitle = """Attach collection to tree [?]""" % (CFG_SITE_URL)
col_dict = dict(get_def_name('', "collection"))
if confirm not in [-1, "-1"] and not (add_son and add_dad and rtype):
output2 += """All fields must be filled.
"""
elif add_son and add_dad and rtype:
add_son = int(add_son)
add_dad = int(add_dad)
if confirm not in [-1, "-1"]:
if add_son == add_dad:
output2 += """Cannot add a collection as a pointer to itself.
"""
elif check_col(add_dad, add_son):
res = add_col_dad_son(add_dad, add_son, rtype)
output2 += write_outcome(res)
if res[0] == 1:
output2 += """ The collection will appear on your website after the next webcoll run. You can either run it manually or wait until bibsched does it for you.
"""
else:
output2 += """Cannot add the collection '%s' as a %s subcollection of '%s' since it will either create a loop, or the association already exists.
""" % (col_dict[add_son], (rtype=="r" and 'regular' or 'virtual'), col_dict[add_dad])
add_son = ''
add_dad = ''
rtype = ''
tree = get_col_tree(colID)
col_list = col_dict.items()
col_list.sort(compare_on_val)
output = show_coll_not_in_tree(colID, ln, col_dict)
text = """
Attach collection: to parent collection:
"""
text += """
with relationship:
""" % ((rtype=="r" and 'selected="selected"' or ''), (rtype=="v" and 'selected="selected"' or ''))
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/addcollectiontotree" % CFG_SITE_URL,
text=text,
button="Add",
colID=colID,
ln=ln,
confirm=1)
output += output2
#output += perform_showtree(colID, ln)
body = [output]
if callback:
return perform_index(colID, ln, mtype="perform_addcollectiontotree", content=addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_addcollection(colID, ln, colNAME='', dbquery='', callback="yes", confirm=-1):
"""form to add a new collection.
colNAME - the name of the new collection
dbquery - the dbquery of the new collection"""
output = ""
subtitle = """Create new collection [?]""" % (CFG_SITE_URL)
text = """
Default name
""" % colNAME
output = createhiddenform(action="%s/admin/websearch/websearchadmin.py/addcollection" % CFG_SITE_URL,
text=text,
colID=colID,
ln=ln,
button="Add collection",
confirm=1)
if colNAME and confirm in ["1", 1]:
res = add_col(colNAME, '')
output += write_outcome(res)
if res[0] == 1:
output += perform_addcollectiontotree(colID=colID, ln=ln, add_son=res[1], callback='')
elif confirm not in ["-1", -1]:
output += """Please give the collection a name."""
body = [output]
if callback:
return perform_index(colID, ln=ln, mtype="perform_addcollection", content=addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_modifydbquery(colID, ln, dbquery='', callback='yes', confirm=-1):
"""form to modify the dbquery of the collection.
dbquery - the dbquery of the collection."""
subtitle = ''
output = ""
col_dict = dict(get_def_name('', "collection"))
if colID and col_dict.has_key(int(colID)):
colID = int(colID)
subtitle = """1. Modify collection query for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
if confirm == -1:
res = run_sql("SELECT dbquery FROM collection WHERE id=%s" % colID)
dbquery = res[0][0]
if not dbquery:
dbquery = ''
reg_sons = len(get_col_tree(colID, 'r'))
vir_sons = len(get_col_tree(colID, 'v'))
if reg_sons > 1:
if dbquery:
output += "Warning: This collection got subcollections, and should because of this not have a collection query, for further explanation, check the WebSearch Guide "
elif reg_sons <= 1:
if not dbquery:
output += "Warning: This collection does not have any subcollections, and should because of this have a collection query, for further explanation, check the WebSearch Guide "
text = """
Query
""" % cgi.escape(dbquery, 1)
output += createhiddenform(action="modifydbquery",
text=text,
button="Modify",
colID=colID,
ln=ln,
confirm=1)
if confirm in ["1", 1]:
res = modify_dbquery(colID, dbquery)
if res:
if dbquery == "":
text = """Query removed for this collection."""
else:
text = """Query set for this collection."""
else:
text = """Sorry, could not change query."""
output += text
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_modifydbquery", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_modifycollectiontree(colID, ln, move_up='', move_down='', move_from='', move_to='', delete='', rtype='', callback='yes', confirm=0):
"""to modify the collection tree: move a collection up and down, delete a collection, or change the father of the collection.
colID - the main collection of the tree, the root
move_up - move this collection up (is not the collection id, but the place in the tree)
move_up - move this collection down (is not the collection id, but the place in the tree)
move_from - move this collection from the current positon (is not the collection id, but the place in the tree)
move_to - move the move_from collection and set this as it's father. (is not the collection id, but the place in the tree)
delete - delete this collection from the tree (is not the collection id, but the place in the tree)
rtype - the type of the collection in the tree, regular or virtual"""
colID = int(colID)
tree = get_col_tree(colID, rtype)
col_dict = dict(get_def_name('', "collection"))
subtitle = """Modify collection tree: %s [?] Printer friendly version""" % (col_dict[colID], CFG_SITE_URL, CFG_SITE_URL, colID, ln)
fin_output = ""
output = ""
try:
if move_up:
move_up = int(move_up)
switch = find_last(tree, move_up)
if switch and switch_col_treescore(tree[move_up], tree[switch]):
output += """Moved the %s collection '%s' up and '%s' down.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]], col_dict[tree[switch][0]])
else:
output += """Could not move the %s collection '%s' up and '%s' down.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]], col_dict[tree[switch][0]])
elif move_down:
move_down = int(move_down)
switch = find_next(tree, move_down)
if switch and switch_col_treescore(tree[move_down], tree[switch]):
output += """Moved the %s collection '%s' down and '%s' up.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_down][0]], col_dict[tree[switch][0]])
else:
output += """Could not move the %s collection '%s' up and '%s' down.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_up][0]],col_dict[tree[switch][0]])
elif delete:
delete = int(delete)
if confirm in [0, "0"]:
if col_dict[tree[delete][0]] != col_dict[tree[delete][3]]:
text = """Do you want to remove the %s collection '%s' and its subcollections in the %s collection '%s'.
""" % ((tree[delete][4]=="r" and 'regular' or 'virtual'), col_dict[tree[delete][0]], (rtype=="r" and 'regular' or 'virtual'), col_dict[tree[delete][3]])
else:
text = """Do you want to remove all subcollections of the %s collection '%s'.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[delete][3]])
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/modifycollectiontree#tree" % CFG_SITE_URL,
text=text,
button="Confirm",
colID=colID,
delete=delete,
rtype=rtype,
ln=ln,
confirm=1)
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/index?mtype=perform_modifycollectiontree#tree" % CFG_SITE_URL,
text="To cancel",
button="Cancel",
colID=colID,
ln=ln)
else:
if remove_col_subcol(tree[delete][0], tree[delete][3], rtype):
if col_dict[tree[delete][0]] != col_dict[tree[delete][3]]:
output += """Removed the %s collection '%s' and its subcollections in subdirectory '%s'.
""" % ((tree[delete][4]=="r" and 'regular' or 'virtual'), col_dict[tree[delete][0]], col_dict[tree[delete][3]])
else:
output += """Removed the subcollections of the %s collection '%s'.
""" % ((rtype=="r" and 'regular' or 'virtual'), col_dict[tree[delete][3]])
else:
output += """Could not remove the collection from the tree.
"""
delete = ''
elif move_from and not move_to:
move_from_rtype = move_from[0]
move_from_id = int(move_from[1:len(move_from)])
text = """Select collection to place the %s collection '%s' under.
""" % ((move_from_rtype=="r" and 'regular' or 'virtual'), col_dict[tree[move_from_id][0]])
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/index?mtype=perform_modifycollectiontree#tree" % CFG_SITE_URL,
text=text,
button="Cancel",
colID=colID,
ln=ln)
elif move_from and move_to:
move_from_rtype = move_from[0]
move_from_id = int(move_from[1:len(move_from)])
move_to_rtype = move_to[0]
move_to_id = int(move_to[1:len(move_to)])
tree_from = get_col_tree(colID, move_from_rtype)
tree_to = get_col_tree(colID, move_to_rtype)
if confirm in [0, '0']:
if move_from_id == move_to_id and move_from_rtype == move_to_rtype:
output += """Cannot move to itself.
"""
elif tree_from[move_from_id][3] == tree_to[move_to_id][0] and move_from_rtype==move_to_rtype:
output += """The collection is already there.
"""
elif check_col(tree_to[move_to_id][0], tree_from[move_from_id][0]) or (tree_to[move_to_id][0] == 1 and tree_from[move_from_id][3] == tree_to[move_to_id][0] and move_from_rtype != move_to_rtype):
text = """Move %s collection '%s' to the %s collection '%s'.
""" % ((tree_from[move_from_id][4]=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (tree_to[move_to_id][4]=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/modifycollectiontree#tree" % CFG_SITE_URL,
text=text,
button="Confirm",
colID=colID,
move_from=move_from,
move_to=move_to,
ln=ln,
rtype=rtype,
confirm=1)
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/index?mtype=perform_modifycollectiontree#tree" % CFG_SITE_URL,
text="""To cancel""",
button="Cancel",
colID=colID,
ln=ln)
else:
output += """Cannot move the collection '%s' and set it as a subcollection of '%s' since it will create a loop.
""" % (col_dict[tree_from[move_from_id][0]], col_dict[tree_to[move_to_id][0]])
else:
if (move_to_id != 0 and move_col_tree(tree_from[move_from_id], tree_to[move_to_id])) or (move_to_id == 0 and move_col_tree(tree_from[move_from_id], tree_to[move_to_id], move_to_rtype)):
output += """Moved %s collection '%s' to the %s collection '%s'.
""" % ((move_from_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (move_to_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
else:
output += """Could not move %s collection '%s' to the %s collection '%s'.
""" % ((move_from_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_from[move_from_id][0]], (move_to_rtype=="r" and 'regular' or 'virtual'), col_dict[tree_to[move_to_id][0]])
move_from = ''
move_to = ''
else:
output += """
"""
except StandardError, e:
register_exception()
return """An error occured.
"""
output += """
"""
body = [output]
return addadminbox(subtitle, body)
def perform_addportalbox(colID, ln, title='', body='', callback='yes', confirm=-1):
"""form to add a new portalbox
title - the title of the portalbox
body - the body of the portalbox"""
col_dict = dict(get_def_name('', "collection"))
colID = int(colID)
subtitle = """Create new portalbox"""
text = """
Title Body
""" % (cgi.escape(title), cgi.escape(body))
output = createhiddenform(action="addportalbox#5.1",
text=text,
button="Add",
colID=colID,
ln=ln,
confirm=1)
if body and confirm in [1, "1"]:
res = add_pbx(title, body)
output += write_outcome(res)
if res[1] == 1:
output += """Add portalbox to collection""" % (colID, ln, res[1])
elif confirm not in [-1, "-1"]:
output += """Body field must be filled.
"""
body = [output]
return perform_showportalboxes(colID, ln, content=addadminbox(subtitle, body))
def perform_addexistingportalbox(colID, ln, pbxID=-1, score=0, position='', sel_ln='', callback='yes', confirm=-1):
"""form to add an existing portalbox to a collection.
colID - the collection to add the portalbox to
pbxID - the portalbox to add
score - the importance of the portalbox.
position - the position of the portalbox on the page
sel_ln - the language of the portalbox"""
subtitle = """Add existing portalbox to collection"""
output = ""
colID = int(colID)
res = get_pbx()
pos = get_pbx_pos()
lang = dict(get_languages())
col_dict = dict(get_def_name('', "collection"))
pbx_dict = dict(map(lambda x: (x[0], x[1]), res))
col_pbx = get_col_pbx(colID)
col_pbx = dict(map(lambda x: (x[0], x[5]), col_pbx))
if len(res) > 0:
text = """
Portalbox Language Position
"
output += createhiddenform(action="addexistingportalbox#5.2",
text=text,
button="Add",
colID=colID,
ln=ln,
confirm=1)
else:
output = """No existing portalboxes to add, please create a new one.
"""
if pbxID > -1 and position and sel_ln and confirm in [1, "1"]:
pbxID = int(pbxID)
res = add_col_pbx(colID, pbxID, sel_ln, position, '')
output += write_outcome(res)
elif pbxID > -1 and confirm not in [-1, "-1"]:
output += """All fields must be filled.
"""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showportalboxes(colID, ln, content=output)
def perform_deleteportalbox(colID, ln, pbxID=-1, callback='yes', confirm=-1):
"""form to delete a portalbox which is not in use.
colID - the current collection.
pbxID - the id of the portalbox"""
subtitle = """Delete an unused portalbox"""
output = ""
colID = int(colID)
if pbxID not in [-1, "-1"] and confirm in [1, "1"]:
ares = get_pbx()
pbx_dict = dict(map(lambda x: (x[0], x[1]), ares))
if pbx_dict.has_key(int(pbxID)):
pname = pbx_dict[int(pbxID)]
ares = delete_pbx(int(pbxID))
else:
return """This portalbox does not exist"""
res = get_pbx()
col_dict = dict(get_def_name('', "collection"))
pbx_dict = dict(map(lambda x: (x[0], x[1]), res))
col_pbx = get_col_pbx()
col_pbx = dict(map(lambda x: (x[0], x[5]), col_pbx))
if len(res) > 0:
text = """
Portalbox """
output += createhiddenform(action="deleteportalbox#5.3",
text=text,
button="Delete",
colID=colID,
ln=ln,
confirm=1)
if pbxID not in [-1, "-1"]:
pbxID = int(pbxID)
if confirm in [1, "1"]:
output += write_outcome(ares)
elif confirm not in [-1, "-1"]:
output += """Choose a portalbox to delete.
"""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showportalboxes(colID, ln, content=output)
def perform_modifyportalbox(colID, ln, pbxID=-1, score='', position='', sel_ln='', title='', body='', callback='yes', confirm=-1):
"""form to modify a portalbox in a collection, or change the portalbox itself.
colID - the id of the collection.
pbxID - the portalbox to change
score - the score of the portalbox connected to colID which should be changed.
position - the position of the portalbox in collection colID to change."""
subtitle = ""
output = ""
colID = int(colID)
res = get_pbx()
pos = get_pbx_pos()
lang = dict(get_languages())
col_dict = dict(get_def_name('', "collection"))
pbx_dict = dict(map(lambda x: (x[0], x[1]), res))
col_pbx = get_col_pbx(colID)
col_pbx = dict(map(lambda x: (x[0], x[5]), col_pbx))
if pbxID not in [-1, "-1"]:
pbxID = int(pbxID)
subtitle = """Modify portalbox '%s' for this collection""" % pbx_dict[pbxID]
col_pbx = get_col_pbx(colID)
if not (score and position) and not (body and title):
for (id_pbx, id_collection, tln, score, position, title, body) in col_pbx:
if id_pbx == pbxID:
break
output += """Collection (presentation) specific values (Changes implies only to this collection.) """
text = """
Position """
output += createhiddenform(action="modifyportalbox#5.4",
text=text,
button="Modify",
colID=colID,
pbxID=pbxID,
score=score,
title=title,
body=cgi.escape(body, 1),
sel_ln=sel_ln,
ln=ln,
confirm=3)
if pbxID > -1 and score and position and confirm in [3, "3"]:
pbxID = int(pbxID)
res = modify_pbx(colID, pbxID, sel_ln, score, position, '', '')
res2 = get_pbx()
pbx_dict = dict(map(lambda x: (x[0], x[1]), res2))
output += write_outcome(res)
output += """ Portalbox (content) specific values (any changes appears everywhere the portalbox is used.)"""
text = """
Title
""" % cgi.escape(title)
text += """
Body
""" % cgi.escape(body)
output += createhiddenform(action="modifyportalbox#5.4",
text=text,
button="Modify",
colID=colID,
pbxID=pbxID,
sel_ln=sel_ln,
score=score,
position=position,
ln=ln,
confirm=4)
if pbxID > -1 and confirm in [4, "4"]:
pbxID = int(pbxID)
res = modify_pbx(colID, pbxID, sel_ln, '', '', title, body)
output += write_outcome(res)
else:
output = """No portalbox to modify."""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showportalboxes(colID, ln, content=output)
def perform_switchpbxscore(colID, id_1, id_2, sel_ln, ln):
"""Switch the score of id_1 and id_2 in collection_portalbox.
colID - the current collection
id_1/id_2 - the id's to change the score for.
sel_ln - the language of the portalbox"""
output = ""
res = get_pbx()
pbx_dict = dict(map(lambda x: (x[0], x[1]), res))
res = switch_pbx_score(colID, id_1, id_2, sel_ln)
output += write_outcome(res)
return perform_showportalboxes(colID, ln, content=output)
def perform_showportalboxes(colID, ln, callback='yes', content='', confirm=-1):
"""show the portalboxes of this collection.
colID - the portalboxes to show the collection for."""
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
subtitle = """5. Modify portalboxes for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
output = ""
pos = get_pbx_pos()
output = """
Portalbox actions (not related to this collection)
"
i += 1
if i != len(res):
move += """""" % (CFG_SITE_URL, colID, ln, pbxID, res[i][0], tln, random.randint(0, 1000), CFG_SITE_URL)
move += """
"""
actions.append(["%s" % (i==1 and pos[position] or ''), "%s" % (i==1 and lang[tln] or ''), move, "%s" % title])
for col in [(('Modify', 'modifyportalbox'), ('Remove', 'removeportalbox'),)]:
actions[-1].append('%s' % (CFG_SITE_URL, col[0][1], colID, ln, pbxID, tln, col[0][0]))
for (str, function) in col[1:]:
actions[-1][-1] += ' / %s' % (CFG_SITE_URL, function, colID, ln, pbxID, tln, str)
output += tupletotable(header=header, tuple=actions)
else:
output += """No portalboxes exists for this collection"""
output += content
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_showportalboxes", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_removeportalbox(colID, ln, pbxID='', sel_ln='', callback='yes', confirm=0):
"""form to remove a portalbox from a collection.
colID - the current collection, remove the portalbox from this collection.
sel_ln - remove the portalbox with this language
pbxID - remove the portalbox with this id"""
subtitle = """Remove portalbox"""
output = ""
col_dict = dict(get_def_name('', "collection"))
res = get_pbx()
pbx_dict = dict(map(lambda x: (x[0], x[1]), res))
if colID and pbxID and sel_ln:
colID = int(colID)
pbxID = int(pbxID)
if confirm in ["0", 0]:
text = """Do you want to remove the portalbox '%s' from the collection '%s'.""" % (pbx_dict[pbxID], col_dict[colID])
output += createhiddenform(action="removeportalbox#5.5",
text=text,
button="Confirm",
colID=colID,
pbxID=pbxID,
sel_ln=sel_ln,
confirm=1)
elif confirm in ["1", 1]:
res = remove_pbx(colID, pbxID, sel_ln)
output += write_outcome(res)
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showportalboxes(colID, ln, content=output)
def perform_switchfmtscore(colID, type, id_1, id_2, ln):
"""Switch the score of id_1 and id_2 in the table type.
colID - the current collection
id_1/id_2 - the id's to change the score for.
type - like "format" """
fmt_dict = dict(get_def_name('', "format"))
res = switch_score(colID, id_1, id_2, type)
output = write_outcome(res)
return perform_showoutputformats(colID, ln, content=output)
def perform_switchfldscore(colID, id_1, id_2, fmeth, ln):
"""Switch the score of id_1 and id_2 in collection_field_fieldvalue.
colID - the current collection
id_1/id_2 - the id's to change the score for."""
fld_dict = dict(get_def_name('', "field"))
res = switch_fld_score(colID, id_1, id_2)
output = write_outcome(res)
if fmeth == "soo":
return perform_showsortoptions(colID, ln, content=output)
elif fmeth == "sew":
return perform_showsearchfields(colID, ln, content=output)
elif fmeth == "seo":
return perform_showsearchoptions(colID, ln, content=output)
def perform_switchfldvaluescore(colID, id_1, id_fldvalue_1, id_fldvalue_2, ln):
"""Switch the score of id_1 and id_2 in collection_field_fieldvalue.
colID - the current collection
id_1/id_2 - the id's to change the score for."""
name_1 = run_sql("SELECT name from fieldvalue where id=%s", (id_fldvalue_1, ))[0][0]
name_2 = run_sql("SELECT name from fieldvalue where id=%s", (id_fldvalue_2, ))[0][0]
res = switch_fld_value_score(colID, id_1, id_fldvalue_1, id_fldvalue_2)
output = write_outcome(res)
return perform_modifyfield(colID, fldID=id_1, ln=ln, content=output)
def perform_addnewfieldvalue(colID, fldID, ln, name='', value='', callback="yes", confirm=-1):
"""form to add a new fieldvalue.
name - the name of the new fieldvalue
value - the value of the new fieldvalue
"""
output = ""
subtitle = """Add new value"""
text = """
Display name Search value
""" % (name, value)
output = createhiddenform(action="%s/admin/websearch/websearchadmin.py/addnewfieldvalue" % CFG_SITE_URL,
text=text,
colID=colID,
fldID=fldID,
ln=ln,
button="Add",
confirm=1)
if name and value and confirm in ["1", 1]:
res = add_fldv(name, value)
output += write_outcome(res)
if res[0] == 1:
res = add_col_fld(colID, fldID, 'seo', res[1])
if res[0] == 0:
output += " " + write_outcome(res)
elif confirm not in ["-1", -1]:
output += """Please fill in name and value.
"""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_modifyfield(colID, fldID=fldID, ln=ln, content=output)
def perform_modifyfieldvalue(colID, fldID, fldvID, ln, name='', value='', callback="yes", confirm=-1):
"""form to modify a fieldvalue.
name - the name of the fieldvalue
value - the value of the fieldvalue
"""
if confirm in [-1, "-1"]:
res = get_fld_value(fldvID)
(id, name, value) = res[0]
output = ""
subtitle = """Modify existing value"""
output = """
Warning: Modifications done below will also inflict on all places the modified data is used.
"""
text = """
Display name Search value
""" % (name, value)
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/modifyfieldvalue" % CFG_SITE_URL,
text=text,
colID=colID,
fldID=fldID,
fldvID=fldvID,
ln=ln,
button="Update",
confirm=1)
output += createhiddenform(action="%s/admin/websearch/websearchadmin.py/modifyfieldvalue" % CFG_SITE_URL,
text="Delete value and all associations",
colID=colID,
fldID=fldID,
fldvID=fldvID,
ln=ln,
button="Delete",
confirm=2)
if name and value and confirm in ["1", 1]:
res = update_fldv(fldvID, name, value)
output += write_outcome(res)
#if res:
# output += """Operation successfully completed."""
#else:
# output += """Operation failed."""
elif confirm in ["2", 2]:
res = delete_fldv(fldvID)
output += write_outcome(res)
elif confirm not in ["-1", -1]:
output += """Please fill in name and value."""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_modifyfield(colID, fldID=fldID, ln=ln, content=output)
def perform_removefield(colID, ln, fldID='', fldvID='', fmeth='', callback='yes', confirm=0):
"""form to remove a field from a collection.
colID - the current collection, remove the field from this collection.
sel_ln - remove the field with this language
fldID - remove the field with this id"""
if fmeth == "soo":
field = "sort option"
elif fmeth == "sew":
field = "search field"
elif fmeth == "seo":
field = "search option"
else:
field = "field"
subtitle = """Remove %s""" % field
output = ""
col_dict = dict(get_def_name('', "collection"))
fld_dict = dict(get_def_name('', "field"))
res = get_fld_value()
fldv_dict = dict(map(lambda x: (x[0], x[1]), res))
if colID and fldID:
colID = int(colID)
fldID = int(fldID)
if fldvID and fldvID != "None":
fldvID = int(fldvID)
if confirm in ["0", 0]:
text = """Do you want to remove the %s '%s' %s from the collection '%s'.""" % (field, fld_dict[fldID], (fldvID not in["", "None"] and "with value '%s'" % fldv_dict[fldvID] or ''), col_dict[colID])
output += createhiddenform(action="removefield#6.5",
text=text,
button="Confirm",
colID=colID,
fldID=fldID,
fldvID=fldvID,
fmeth=fmeth,
confirm=1)
elif confirm in ["1", 1]:
res = remove_fld(colID, fldID, fldvID)
output += write_outcome(res)
body = [output]
output = " " + addadminbox(subtitle, body)
if fmeth == "soo":
return perform_showsortoptions(colID, ln, content=output)
elif fmeth == "sew":
return perform_showsearchfields(colID, ln, content=output)
elif fmeth == "seo":
return perform_showsearchoptions(colID, ln, content=output)
def perform_removefieldvalue(colID, ln, fldID='', fldvID='', fmeth='', callback='yes', confirm=0):
"""form to remove a field from a collection.
colID - the current collection, remove the field from this collection.
sel_ln - remove the field with this language
fldID - remove the field with this id"""
subtitle = """Remove value"""
output = ""
col_dict = dict(get_def_name('', "collection"))
fld_dict = dict(get_def_name('', "field"))
res = get_fld_value()
fldv_dict = dict(map(lambda x: (x[0], x[1]), res))
if colID and fldID:
colID = int(colID)
fldID = int(fldID)
if fldvID and fldvID != "None":
fldvID = int(fldvID)
if confirm in ["0", 0]:
text = """Do you want to remove the value '%s' from the search option '%s'.""" % (fldv_dict[fldvID], fld_dict[fldID])
output += createhiddenform(action="removefieldvalue#7.4",
text=text,
button="Confirm",
colID=colID,
fldID=fldID,
fldvID=fldvID,
fmeth=fmeth,
confirm=1)
elif confirm in ["1", 1]:
res = remove_fld(colID, fldID, fldvID)
output += write_outcome(res)
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_modifyfield(colID, fldID=fldID, ln=ln, content=output)
def perform_rearrangefieldvalue(colID, fldID, ln, callback='yes', confirm=-1):
"""rearrang the fieldvalues alphabetically
colID - the collection
fldID - the field to rearrange the fieldvalue for
"""
subtitle = "Order values alphabetically"
output = ""
col_fldv = get_col_fld(colID, 'seo', fldID)
col_fldv = dict(map(lambda x: (x[1], x[0]), col_fldv))
fldv_names = get_fld_value()
fldv_names = map(lambda x: (x[0], x[1]), fldv_names)
if not col_fldv.has_key(None):
vscore = len(col_fldv)
for (fldvID, name) in fldv_names:
if col_fldv.has_key(fldvID):
run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s", (vscore, colID, fldID, fldvID))
vscore -= 1
output += write_outcome((1, ""))
else:
output += write_outcome((0, (0, "No values to order")))
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_modifyfield(colID, fldID, ln, content=output)
def perform_rearrangefield(colID, ln, fmeth, callback='yes', confirm=-1):
"""rearrang the fields alphabetically
colID - the collection
"""
subtitle = "Order fields alphabetically"
output = ""
col_fld = dict(map(lambda x: (x[0], x[1]), get_col_fld(colID, fmeth)))
fld_names = get_def_name('', "field")
if len(col_fld) > 0:
score = len(col_fld)
for (fldID, name) in fld_names:
if col_fld.has_key(fldID):
run_sql("UPDATE collection_field_fieldvalue SET score=%s WHERE id_collection=%s and id_field=%s", (score, colID, fldID))
score -= 1
output += write_outcome((1, ""))
else:
output += write_outcome((0, (0, "No fields to order")))
body = [output]
output = " " + addadminbox(subtitle, body)
if fmeth == "soo":
return perform_showsortoptions(colID, ln, content=output)
elif fmeth == "sew":
return perform_showsearchfields(colID, ln, content=output)
elif fmeth == "seo":
return perform_showsearchoptions(colID, ln, content=output)
def perform_addexistingfieldvalue(colID, fldID, fldvID=-1, ln=CFG_SITE_LANG, callback='yes', confirm=-1):
"""form to add an existing fieldvalue to a field.
colID - the collection
fldID - the field to add the fieldvalue to
fldvID - the fieldvalue to add"""
subtitle = """Add existing value to search option"""
output = ""
if fldvID not in [-1, "-1"] and confirm in [1, "1"]:
fldvID = int(fldvID)
ares = add_col_fld(colID, fldID, 'seo', fldvID)
colID = int(colID)
fldID = int(fldID)
lang = dict(get_languages())
res = get_def_name('', "field")
col_dict = dict(get_def_name('', "collection"))
fld_dict = dict(res)
col_fld = dict(map(lambda x: (x[0], x[1]), get_col_fld(colID, 'seo')))
fld_value = get_fld_value()
fldv_dict = dict(map(lambda x: (x[0], x[1]), fld_value))
text = """
Value """
output += createhiddenform(action="addexistingfieldvalue#7.4",
text=text,
button="Add",
colID=colID,
fldID=fldID,
ln=ln,
confirm=1)
if fldvID not in [-1, "-1"] and confirm in [1, "1"]:
output += write_outcome(ares)
elif confirm in [1, "1"]:
output += """Select a value to add and try again."""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_modifyfield(colID, fldID, ln, content=output)
def perform_addexistingfield(colID, ln, fldID=-1, fldvID=-1, fmeth='', callback='yes', confirm=-1):
"""form to add an existing field to a collection.
colID - the collection to add the field to
fldID - the field to add
sel_ln - the language of the field"""
subtitle = """Add existing field to collection"""
output = ""
if fldID not in [-1, "-1"] and confirm in [1, "1"]:
fldID = int(fldID)
ares = add_col_fld(colID, fldID, fmeth, fldvID)
colID = int(colID)
lang = dict(get_languages())
res = get_def_name('', "field")
col_dict = dict(get_def_name('', "collection"))
fld_dict = dict(res)
col_fld = dict(map(lambda x: (x[0], x[1]), get_col_fld(colID, fmeth)))
fld_value = get_fld_value()
fldv_dict = dict(map(lambda x: (x[0], x[1]), fld_value))
if fldvID:
fldvID = int(fldvID)
text = """
Field """
output += createhiddenform(action="addexistingfield#6.2",
text=text,
button="Add",
colID=colID,
fmeth=fmeth,
ln=ln,
confirm=1)
if fldID not in [-1, "-1"] and confirm in [1, "1"]:
output += write_outcome(ares)
elif fldID in [-1, "-1"] and confirm not in [-1, "-1"]:
output += """Select a field.
"""
body = [output]
output = " " + addadminbox(subtitle, body)
if fmeth == "soo":
return perform_showsortoptions(colID, ln, content=output)
elif fmeth == "sew":
return perform_showsearchfields(colID, ln, content=output)
elif fmeth == "seo":
return perform_showsearchoptions(colID, ln, content=output)
def perform_showsortoptions(colID, ln, callback='yes', content='', confirm=-1):
"""show the sort fields of this collection.."""
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
fld_dict = dict(get_def_name('', "field"))
fld_type = get_sort_nametypes()
subtitle = """8. Modify sort options for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
output = """
Field actions (not related to this collection)
Go to the BibIndex interface to modify the available sort options
"
i += 1
if i != len(col_fmt):
move += '' % (CFG_SITE_URL, colID, ln, id_format, col_fmt[i][0], random.randint(0, 1000), CFG_SITE_URL)
move += """
"""
actions.append([move, code, fmt_dict[int(id_format)]])
for col in [(('Remove', 'removeoutputformat'),)]:
actions[-1].append('%s' % (CFG_SITE_URL, col[0][1], colID, ln, id_format, col[0][0]))
for (str, function) in col[1:]:
actions[-1][-1] += ' / %s' % (CFG_SITE_URL, function, colID, ln, id_format, str)
output += tupletotable(header=header, tuple=actions)
else:
output += """No output formats exists for this collection"""
output += content
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_showoutputformats", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def external_collections_build_select(colID, external_collection):
output = '\n'
return output
def perform_manage_external_collections(colID, ln, callback='yes', content='', confirm=-1):
"""Show the interface to configure external collections to the user."""
colID = int(colID)
subtitle = """11. Configuration of related external collections
[?]""" % CFG_SITE_URL
output = ''
return addadminbox(subtitle, [output])
def perform_update_external_collections(colID, ln, state_list, recurse_list):
colID = int(colID)
changes = []
output = ""
if not state_list:
return 'Warning : No state found. ' + perform_manage_external_collections(colID, ln)
external_collections = external_collection_sort_engine_by_name(external_collections_dictionary.values())
if len(external_collections) != len(state_list):
return 'Warning : Size of state_list different from external_collections! ' + perform_manage_external_collections(colID, ln)
for (external_collection, state) in zip(external_collections, state_list):
state = int(state)
collection_name = external_collection.name
recurse = recurse_list and collection_name in recurse_list
oldstate = external_collection_get_state(external_collection, colID)
if oldstate != state or recurse:
changes += external_collection_get_update_state_list(external_collection, colID, state, recurse)
external_collection_apply_changes(changes)
return output + '
' + perform_manage_external_collections(colID, ln)
def perform_showdetailedrecordoptions(colID, ln, callback='yes', content='', confirm=-1):
"""Show the interface to configure detailed record page to the user."""
colID = int(colID)
subtitle = """12. Configuration of detailed record page
[?]""" % CFG_SITE_URL
output = ''''
return addadminbox(subtitle, [output])
def perform_update_detailed_record_options(colID, ln, tabs, recurse):
"""Update the preferences for the tab to show/hide in the detailed record page."""
colID = int(colID)
changes = []
output = 'Operation successfully completed.'
if '' in tabs:
tabs.remove('')
tabs.append('metadata')
def update_settings(colID, tabs, recurse):
run_sql("DELETE FROM collectiondetailedrecordpagetabs WHERE id_collection=%s", (colID, ))
run_sql("REPLACE INTO collectiondetailedrecordpagetabs" + \
" SET id_collection=%s, tabs=%s", (colID, ';'.join(tabs)))
## for enabled_tab in tabs:
## run_sql("REPLACE INTO collectiondetailedrecordpagetabs" + \
## " SET id_collection='%s', tabs='%s'" % (colID, ';'.join(tabs)))
if recurse:
for descendant_id in get_collection_descendants(colID):
update_settings(descendant_id, tabs, recurse)
update_settings(colID, tabs, recurse)
## for colID in colIDs:
## run_sql("DELETE FROM collectiondetailedrecordpagetabs WHERE id_collection='%s'" % colID)
## for enabled_tab in tabs:
## run_sql("REPLACE INTO collectiondetailedrecordpagetabs" + \
## " SET id_collection='%s', tabs='%s'" % (colID, ';'.join(tabs)))
#if callback:
return perform_editcollection(colID, ln, "perform_modifytranslations",
'
' + perform_showdetailedrecordoptions(colID, ln)
def perform_addexistingoutputformat(colID, ln, fmtID=-1, callback='yes', confirm=-1):
"""form to add an existing output format to a collection.
colID - the collection the format should be added to
fmtID - the format to add."""
subtitle = """Add existing output format to collection"""
output = ""
if fmtID not in [-1, "-1"] and confirm in [1, "1"]:
ares = add_col_fmt(colID, fmtID)
colID = int(colID)
res = get_def_name('', "format")
fmt_dict = dict(res)
col_dict = dict(get_def_name('', "collection"))
col_fmt = get_col_fmt(colID)
col_fmt = dict(map(lambda x: (x[0], x[2]), col_fmt))
if len(res) > 0:
text = """
Output format
"""
output += createhiddenform(action="addexistingoutputformat#10.2",
text=text,
button="Add",
colID=colID,
ln=ln,
confirm=1)
else:
output = """No existing output formats to add, please create a new one."""
if fmtID not in [-1, "-1"] and confirm in [1, "1"]:
output += write_outcome(ares)
elif fmtID in [-1, "-1"] and confirm not in [-1, "-1"]:
output += """Please select output format."""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showoutputformats(colID, ln, content=output)
def perform_deleteoutputformat(colID, ln, fmtID=-1, callback='yes', confirm=-1):
"""form to delete an output format not in use.
colID - the collection id of the current collection.
fmtID - the format id to delete."""
subtitle = """Delete an unused output format"""
output = """
Deleting an output format will also delete the translations associated.
"""
colID = int(colID)
if fmtID not in [-1, "-1"] and confirm in [1, "1"]:
fmt_dict = dict(get_def_name('', "format"))
old_colNAME = fmt_dict[int(fmtID)]
ares = delete_fmt(int(fmtID))
res = get_def_name('', "format")
fmt_dict = dict(res)
col_dict = dict(get_def_name('', "collection"))
col_fmt = get_col_fmt()
col_fmt = dict(map(lambda x: (x[0], x[2]), col_fmt))
if len(res) > 0:
text = """
Output format """
output += createhiddenform(action="deleteoutputformat#10.3",
text=text,
button="Delete",
colID=colID,
ln=ln,
confirm=0)
if fmtID not in [-1, "-1"]:
fmtID = int(fmtID)
if confirm in [0, "0"]:
text = """Do you want to delete the output format '%s'.
""" % fmt_dict[fmtID]
output += createhiddenform(action="deleteoutputformat#10.3",
text=text,
button="Confirm",
colID=colID,
fmtID=fmtID,
ln=ln,
confirm=1)
elif confirm in [1, "1"]:
output += write_outcome(ares)
elif confirm not in [-1, "-1"]:
output += """Choose a output format to delete.
"""
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showoutputformats(colID, ln, content=output)
def perform_removeoutputformat(colID, ln, fmtID='', callback='yes', confirm=0):
"""form to remove an output format from a collection.
colID - the collection id of the current collection.
fmtID - the format id.
"""
subtitle = """Remove output format"""
output = ""
col_dict = dict(get_def_name('', "collection"))
fmt_dict = dict(get_def_name('', "format"))
if colID and fmtID:
colID = int(colID)
fmtID = int(fmtID)
if confirm in ["0", 0]:
text = """Do you want to remove the output format '%s' from the collection '%s'.""" % (fmt_dict[fmtID], col_dict[colID])
output += createhiddenform(action="removeoutputformat#10.5",
text=text,
button="Confirm",
colID=colID,
fmtID=fmtID,
confirm=1)
elif confirm in ["1", 1]:
res = remove_fmt(colID, fmtID)
output += write_outcome(res)
body = [output]
output = " " + addadminbox(subtitle, body)
return perform_showoutputformats(colID, ln, content=output)
def perform_index(colID=1, ln=CFG_SITE_LANG, mtype='', content='', confirm=0):
"""The index method, calling methods to show the collection tree, create new collections and add collections to tree.
"""
subtitle = "Overview"
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
output = ""
fin_output = ""
if not col_dict.has_key(1):
res = add_col(CFG_SITE_NAME, '')
if res:
fin_output += """Created root collection. """
else:
return "Cannot create root collection, please check database."
if CFG_SITE_NAME != run_sql("SELECT name from collection WHERE id=1")[0][0]:
res = run_sql("update collection set name=%s where id=1", (CFG_SITE_NAME, ))
if res:
fin_output += """The name of the root collection has been modified to be the same as the %(sitename)s installation name given prior to installing %(sitename)s. """ % {'sitename' : CFG_SITE_NAME}
else:
return "Error renaming root collection."
fin_output += """
To manage the collections, select an item from the menu. """
if mtype == "perform_addcollection" and content:
fin_output += content
elif mtype == "perform_addcollection" or mtype == "perform_showall":
fin_output += perform_addcollection(colID=colID, ln=ln, callback='')
fin_output += " "
if mtype == "perform_addcollectiontotree" and content:
fin_output += content
elif mtype == "perform_addcollectiontotree" or mtype == "perform_showall":
fin_output += perform_addcollectiontotree(colID=colID, ln=ln, callback='')
fin_output += " "
if mtype == "perform_modifycollectiontree" and content:
fin_output += content
elif mtype == "perform_modifycollectiontree" or mtype == "perform_showall":
fin_output += perform_modifycollectiontree(colID=colID, ln=ln, callback='')
fin_output += " "
if mtype == "perform_checkwebcollstatus" and content:
fin_output += content
elif mtype == "perform_checkwebcollstatus" or mtype == "perform_showall":
fin_output += perform_checkwebcollstatus(colID, ln, callback='')
if mtype == "perform_checkcollectionstatus" and content:
fin_output += content
elif mtype == "perform_checkcollectionstatus" or mtype == "perform_showall":
fin_output += perform_checkcollectionstatus(colID, ln, callback='')
if mtype == "perform_checkexternalcollections" and content:
fin_output += content
elif mtype == "perform_checkexternalcollections" or mtype == "perform_showall":
fin_output += perform_checkexternalcollections(colID, ln, callback='')
body = [fin_output]
body = [fin_output]
return addadminbox('Menu', body)
def show_coll_not_in_tree(colID, ln, col_dict):
"""Returns collections not in tree"""
tree = get_col_tree(colID)
in_tree = {}
output = "These collections are not in the tree, and should be added: "
for (id, up, down, dad, reltype) in tree:
in_tree[id] = 1
in_tree[dad] = 1
res = run_sql("SELECT id from collection")
if len(res) != len(in_tree):
for id in res:
if not in_tree.has_key(id[0]):
output += """%s ,
""" % (CFG_SITE_URL, id[0], ln, col_dict[id[0]])
output += "
"
else:
output = ""
return output
def create_colltree(tree, col_dict, colID, ln, move_from='', move_to='', rtype='', edit=''):
"""Creates the presentation of the collection tree, with the buttons for modifying it.
tree - the tree to present, from get_tree()
col_dict - the name of the collections in a dictionary
colID - the collection id to start with
move_from - if a collection to be moved has been chosen
move_to - the collection which should be set as father of move_from
rtype - the type of the tree, regular or virtual
edit - if the method should output the edit buttons."""
if move_from:
move_from_rtype = move_from[0]
move_from_id = int(move_from[1:len(move_from)])
tree_from = get_col_tree(colID, move_from_rtype)
tree_to = get_col_tree(colID, rtype)
tables = 0
tstack = []
i = 0
text = """
"""
for i in range(0, len(tree)):
id_son = tree[i][0]
up = tree[i][1]
down = tree[i][2]
dad = tree[i][3]
reltype = tree[i][4]
tmove_from = ""
j = i
while j > 0:
j = j - 1
try:
if tstack[j][1] == dad:
table = tstack[j][2]
for k in range(0, tables - table):
tables = tables - 1
text += """
"""
break
except StandardError, e:
pass
text += """
"""
if i > 0 and tree[i][1] == 0:
tables = tables + 1
text += """
"""
if i == 0:
tstack.append((id_son, dad, 1))
else:
tstack.append((id_son, dad, tables))
if up == 1 and edit:
text += """""" % (CFG_SITE_URL, colID, ln, i, rtype, tree[i][0], CFG_SITE_URL)
else:
text += """ """
text += "
"
if down == 1 and edit:
text += """""" % (CFG_SITE_URL, colID, ln, i, rtype, tree[i][0], CFG_SITE_URL)
else:
text += """ """
text += "
"
if edit:
if move_from and move_to:
tmove_from = move_from
move_from = ''
if not (move_from == "" and i == 0) and not (move_from != "" and int(move_from[1:len(move_from)]) == i and rtype == move_from[0]):
check = "true"
if move_from:
#if tree_from[move_from_id][0] == tree_to[i][0] or not check_col(tree_to[i][0], tree_from[move_from_id][0]):
# check = ''
#elif not check_col(tree_to[i][0], tree_from[move_from_id][0]):
# check = ''
#if not check and (tree_to[i][0] == 1 and tree_from[move_from_id][3] == tree_to[i][0] and move_from_rtype != rtype):
# check = "true"
if check:
text += """
""" % (CFG_SITE_URL, colID, ln, move_from, rtype, i, rtype, CFG_SITE_URL, col_dict[tree_from[int(move_from[1:len(move_from)])][0]], col_dict[tree_to[i][0]])
else:
try:
text += """""" % (CFG_SITE_URL, colID, ln, rtype, i, rtype, tree[i][0], CFG_SITE_URL, col_dict[tree[i][0]])
except KeyError:
pass
else:
text += """
""" % CFG_SITE_URL
else:
text += """
""" % CFG_SITE_URL
text += """
"""
if edit:
try:
text += """""" % (CFG_SITE_URL, colID, ln, i, rtype, tree[i][0], CFG_SITE_URL)
except KeyError:
pass
elif i != 0:
text += """
""" % CFG_SITE_URL
text += """
"""
if tmove_from:
move_from = tmove_from
try:
text += """%s%s%s%s%s""" % (tree[i][0], (reltype=="v" and '' or ''), CFG_SITE_URL, tree[i][0], ln, col_dict[id_son], (move_to=="%s%s" %(rtype, i) and ' ' % CFG_SITE_URL or ''), (move_from=="%s%s" % (rtype, i) and ' ' % CFG_SITE_URL or ''), (reltype=="v" and '' or ''))
except KeyError:
pass
text += """
"""
while tables > 0:
text += """
"""
tables = tables - 1
text += """
"""
return text
def perform_deletecollection(colID, ln, confirm=-1, callback='yes'):
"""form to delete a collection
colID - id of collection
"""
subtitle =''
output = """
WARNING:
When deleting a collection, you also deletes all data related to the collection like translations, relations to other collections and information about which rank methods to use.
For more information, please go to the WebSearch guide and read the section regarding deleting a collection.
""" % CFG_SITE_URL
col_dict = dict(get_def_name('', "collection"))
if colID != 1 and colID and col_dict.has_key(int(colID)):
colID = int(colID)
subtitle = """4. Delete collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
- res = run_sql("SELECT * from collection_collection WHERE id_dad=%s", (colID, ))
- res2 = run_sql("SELECT * from collection_collection WHERE id_son=%s", (colID, ))
+ res = run_sql("SELECT id_dad,id_son,type,score from collection_collection WHERE id_dad=%s", (colID, ))
+ res2 = run_sql("SELECT id_dad,id_son,type,score from collection_collection WHERE id_son=%s", (colID, ))
if not res and not res2:
if confirm in ["-1", -1]:
text = """Do you want to delete this collection."""
output += createhiddenform(action="deletecollection#4",
text=text,
colID=colID,
button="Delete",
confirm=0)
elif confirm in ["0", 0]:
text = """Are you sure you want to delete this collection."""
output += createhiddenform(action="deletecollection#4",
text=text,
colID=colID,
button="Confirm",
confirm=1)
elif confirm in ["1", 1]:
result = delete_col(colID)
if not result:
raise Exception
else:
output = """Can not delete a collection that is a part of the collection tree, remove collection from the tree and try again."""
else:
subtitle = """4. Delete collection"""
output = """Not possible to delete the root collection"""
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_deletecollection", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_editcollection(colID=1, ln=CFG_SITE_LANG, mtype='', content=''):
"""interface to modify a collection. this method is calling other methods which again is calling this and sending back the output of the method.
if callback, the method will call perform_editcollection, if not, it will just return its output.
colID - id of the collection
mtype - the method that called this method.
content - the output from that method."""
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
if not col_dict.has_key(colID):
return """Collection deleted.
"""
fin_output = """
""" % (colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln, colID, ln)
if mtype == "perform_modifydbquery" and content:
fin_output += content
elif mtype == "perform_modifydbquery" or not mtype:
fin_output += perform_modifydbquery(colID, ln, callback='')
if mtype == "perform_modifyrestricted" and content:
fin_output += content
elif mtype == "perform_modifyrestricted" or not mtype:
fin_output += perform_modifyrestricted(colID, ln, callback='')
if mtype == "perform_modifytranslations" and content:
fin_output += content
elif mtype == "perform_modifytranslations" or not mtype:
fin_output += perform_modifytranslations(colID, ln, callback='')
if mtype == "perform_deletecollection" and content:
fin_output += content
elif mtype == "perform_deletecollection" or not mtype:
fin_output += perform_deletecollection(colID, ln, callback='')
if mtype == "perform_showportalboxes" and content:
fin_output += content
elif mtype == "perform_showportalboxes" or not mtype:
fin_output += perform_showportalboxes(colID, ln, callback='')
if mtype == "perform_showsearchfields" and content:
fin_output += content
elif mtype == "perform_showsearchfields" or not mtype:
fin_output += perform_showsearchfields(colID, ln, callback='')
if mtype == "perform_showsearchoptions" and content:
fin_output += content
elif mtype == "perform_showsearchoptions" or not mtype:
fin_output += perform_showsearchoptions(colID, ln, callback='')
if mtype == "perform_showsortoptions" and content:
fin_output += content
elif mtype == "perform_showsortoptions" or not mtype:
fin_output += perform_showsortoptions(colID, ln, callback='')
if mtype == "perform_modifyrankmethods" and content:
fin_output += content
elif mtype == "perform_modifyrankmethods" or not mtype:
fin_output += perform_modifyrankmethods(colID, ln, callback='')
if mtype == "perform_showoutputformats" and content:
fin_output += content
elif mtype == "perform_showoutputformats" or not mtype:
fin_output += perform_showoutputformats(colID, ln, callback='')
if mtype == "perform_manage_external_collections" and content:
fin_output += content
elif mtype == "perform_manage_external_collections" or not mtype:
fin_output += perform_manage_external_collections(colID, ln, callback='')
if mtype == "perform_showdetailedrecordoptions" and content:
fin_output += content
elif mtype == "perform_showdetailedrecordoptions" or not mtype:
fin_output += perform_showdetailedrecordoptions(colID, ln, callback='')
return addadminbox("Overview of edit options for collection '%s'" % col_dict[colID], [fin_output])
def perform_checkwebcollstatus(colID, ln, confirm=0, callback='yes'):
"""Check status of the collection tables with respect to the webcoll cache."""
subtitle = """Webcoll Status [?]""" % CFG_SITE_URL
output = ""
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
output += """ Last updates: """
collection_table_update_time = ""
collection_web_update_time = ""
collection_table_update_time = get_table_update_time('collection')
output += "Collection table last updated: %s " % collection_table_update_time
try:
file = open("%s/collections/last_updated" % CFG_CACHEDIR)
collection_web_update_time = file.readline().strip()
output += "Collection cache last updated: %s " % collection_web_update_time
file.close()
except:
pass
# reformat collection_web_update_time to the format suitable for comparisons
try:
collection_web_update_time = time.strftime("%Y-%m-%d %H:%M:%S",
time.strptime(collection_web_update_time, "%d %b %Y %H:%M:%S"))
except ValueError, e:
pass
if collection_table_update_time > collection_web_update_time:
output += """ Warning: The collections have been modified since last time Webcoll was executed, to process the changes, Webcoll must be executed. """
header = ['ID', 'Name', 'Time', 'Status', 'Progress']
actions = []
output += """ Last BibSched tasks: """
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='webcoll' and runtime< now() ORDER by runtime")
if len(res) > 0:
(id, proc, host, user, runtime, sleeptime, arguments, status, progress) = res[len(res) - 1]
webcoll__update_time = runtime
actions.append([id, proc, runtime, (status !="" and status or ''), (progress !="" and progress or '')])
else:
actions.append(['', 'webcoll', '', '', 'Not executed yet'])
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='bibindex' and runtime< now() ORDER by runtime")
if len(res) > 0:
(id, proc, host, user, runtime, sleeptime, arguments, status, progress) = res[len(res) - 1]
actions.append([id, proc, runtime, (status !="" and status or ''), (progress !="" and progress or '')])
else:
actions.append(['', 'bibindex', '', '', 'Not executed yet'])
output += tupletotable(header=header, tuple=actions)
output += """ Next scheduled BibSched run: """
actions = []
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='webcoll' and runtime > now() ORDER by runtime")
webcoll_future = ""
if len(res) > 0:
(id, proc, host, user, runtime, sleeptime, arguments, status, progress) = res[0]
webcoll__update_time = runtime
actions.append([id, proc, runtime, (status !="" and status or ''), (progress !="" and progress or '')])
webcoll_future = "yes"
else:
actions.append(['', 'webcoll', '', '', 'Not scheduled'])
res = run_sql("select id, proc, host, user, runtime, sleeptime, arguments, status, progress from schTASK where proc='bibindex' and runtime > now() ORDER by runtime")
bibindex_future = ""
if len(res) > 0:
(id, proc, host, user, runtime, sleeptime, arguments, status, progress) = res[0]
actions.append([id, proc, runtime, (status !="" and status or ''), (progress !="" and progress or '')])
bibindex_future = "yes"
else:
actions.append(['', 'bibindex', '', '', 'Not scheduled'])
output += tupletotable(header=header, tuple=actions)
if webcoll_future == "":
output += """ Warning: Webcoll is not scheduled for a future run by bibsched, any updates to the collection will not be processed. """
if bibindex_future == "":
output += """ Warning: Bibindex is not scheduled for a future run by bibsched, any updates to the records will not be processed. """
body = [output]
if callback:
return perform_index(colID, ln, "perform_checkwebcollstatus", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_modifyrestricted(colID, ln, rest='', callback='yes', confirm=-1):
"""modify which apache group is allowed to access the collection.
rest - the groupname"""
subtitle = ''
output = ""
col_dict = dict(get_def_name('', "collection"))
action_id = acc_get_action_id(VIEWRESTRCOLL)
if colID and col_dict.has_key(int(colID)):
colID = int(colID)
subtitle = """2. Modify access restrictions for collection '%s' [?]""" % (col_dict[colID], CFG_SITE_URL)
output = """
Please note that CDS Invenio versions greater than 0.92.1 manage collection restriction via the standard
WebAccess Admin Interface (action '%s').
""" % (action_id, VIEWRESTRCOLL)
body = [output]
if callback:
return perform_editcollection(colID, ln, "perform_modifyrestricted", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_checkcollectionstatus(colID, ln, confirm=0, callback='yes'):
"""Check the configuration of the collections."""
from invenio.search_engine import collection_restricted_p
subtitle = """Collection Status [?]""" % CFG_SITE_URL
output = ""
colID = int(colID)
col_dict = dict(get_def_name('', "collection"))
collections = run_sql("SELECT id, name, dbquery FROM collection ORDER BY id")
header = ['ID', 'Name', 'Query', 'Subcollections', 'Restricted', 'Hosted', 'I18N', 'Status']
rnk_list = get_def_name('', "rnkMETHOD")
actions = []
for (id, name, dbquery) in collections:
reg_sons = len(get_col_tree(id, 'r'))
vir_sons = len(get_col_tree(id, 'v'))
status = ""
hosted = ""
if str(dbquery).startswith("hostedcollection:"): hosted = """Yes"""
else: hosted = """No"""
langs = run_sql("SELECT ln from collectionname where id_collection=%s", (id, ))
i8n = ""
if len(langs) > 0:
for lang in langs:
i8n += "%s, " % lang
else:
i8n = """None"""
if (reg_sons > 1 and dbquery) or dbquery=="":
status = """1:Query"""
elif dbquery is None and reg_sons == 1:
status = """2:Query"""
elif dbquery == "" and reg_sons == 1:
status = """3:Query"""
if (reg_sons > 1 or vir_sons > 1):
subs = """Yes"""
else:
subs = """No"""
if dbquery is None:
dbquery = """No"""
restricted = collection_restricted_p(name)
if restricted:
restricted = """Yes"""
if status:
status += """,4:Restricted"""
else:
status += """4:Restricted"""
else:
restricted = """No"""
if status == "":
status = """OK"""
actions.append([id, """%s""" % (CFG_SITE_URL, id, ln, name), dbquery, subs, restricted, hosted, i8n, status])
output += tupletotable(header=header, tuple=actions)
body = [output]
return addadminbox(subtitle, body)
if callback:
return perform_index(colID, ln, "perform_checkcollectionstatus", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_checkexternalcollections(colID, ln, icl=None, update="", confirm=0, callback='yes'):
"""Check the external collections for inconsistencies."""
subtitle = """Check external collections [?]""" % CFG_SITE_URL
output = ""
colID = int(colID)
if icl:
if update == "add":
# icl : the "inconsistent list" comes as a string, it has to be converted back into a list
icl = eval(icl)
#icl = icl[1:-1].split(',')
for collection in icl:
#collection = str(collection[1:-1])
query_select = "SELECT name FROM externalcollection WHERE name like '%(name)s';" % {'name': collection}
results_select = run_sql(query_select)
if not results_select:
query_insert = "INSERT INTO externalcollection (name) VALUES ('%(name)s');" % {'name': collection}
run_sql(query_insert)
output += """ New collection \"%s\" has been added to the database table \"externalcollection\". """ % (collection)
else:
output += """ Collection \"%s\" has already been added to the database table \"externalcollection\" or was already there. """ % (collection)
elif update == "del":
# icl : the "inconsistent list" comes as a string, it has to be converted back into a list
icl = eval(icl)
#icl = icl[1:-1].split(',')
for collection in icl:
#collection = str(collection[1:-1])
query_select = "SELECT id FROM externalcollection WHERE name like '%(name)s';" % {'name': collection}
results_select = run_sql(query_select)
if results_select:
query_delete = "DELETE FROM externalcollection WHERE id like '%(id)s';" % {'id': results_select[0][0]}
query_delete_states = "DELETE FROM collection_externalcollection WHERE id_externalcollection like '%(id)s';" % {'id': results_select[0][0]}
run_sql(query_delete)
run_sql(query_delete_states)
output += """ Collection \"%s\" has been deleted from the database table \"externalcollection\". """ % (collection)
else:
output += """ Collection \"%s\" has already been delete from the database table \"externalcollection\" or was never there. """ % (collection)
external_collections_file = []
external_collections_db = []
for coll in external_collections_dictionary.values():
external_collections_file.append(coll.name)
external_collections_file.sort()
query = """SELECT name from externalcollection"""
results = run_sql(query)
for result in results:
external_collections_db.append(result[0])
external_collections_db.sort()
number_file = len(external_collections_file)
number_db = len(external_collections_db)
if external_collections_file == external_collections_db:
output += """ External collections are consistent.
- database table \"externalcollection\" has %(number_db)s collections
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections""" % {
"number_db" : number_db,
"number_file" : number_file}
elif len(external_collections_file) > len(external_collections_db):
external_collections_diff = list(set(external_collections_file) - set(external_collections_db))
external_collections_db.extend(external_collections_diff)
external_collections_db.sort()
if external_collections_file == external_collections_db:
output += """ There is an inconsistency:
- database table \"externalcollection\" has %(number_db)s collections
(missing: %(diff)s)
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections
Click here to update your database adding the missing collections. If the problem persists please check your configuration manually.""" % {
"number_db" : number_db,
"number_file" : number_file,
"diff" : external_collections_diff,
"site_url" : CFG_SITE_URL,
"colID" : colID,
"ln" : ln}
else:
output += """ There is an inconsistency:
- database table \"externalcollection\" has %(number_db)s collections
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections
The external collections do not match. To fix the problem please check your configuration manually.""" % {
"number_db" : number_db,
"number_file" : number_file}
elif len(external_collections_file) < len(external_collections_db):
external_collections_diff = list(set(external_collections_db) - set(external_collections_file))
external_collections_file.extend(external_collections_diff)
external_collections_file.sort()
if external_collections_file == external_collections_db:
output += """ There is an inconsistency:
- database table \"externalcollection\" has %(number_db)s collections
(extra: %(diff)s)
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections
Click here to force remove the extra collections from your database (warning: use with caution!). If the problem persists please check your configuration manually.""" % {
"number_db" : number_db,
"number_file" : number_file,
"diff" : external_collections_diff,
"site_url" : CFG_SITE_URL,
"colID" : colID,
"ln" : ln}
else:
output += """ There is an inconsistency:
- database table \"externalcollection\" has %(number_db)s collections
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections
The external collections do not match. To fix the problem please check your configuration manually.""" % {
"number_db" : number_db,
"number_file" : number_file}
else:
output += """ There is an inconsistency:
- database table \"externalcollection\" has %(number_db)s collections
- configuration file \"websearch_external_collections_config.py\" has %(number_file)s collections
The number of external collections is the same but the collections do not match. To fix the problem please check your configuration manually.""" % {
"number_db" : number_db,
"number_file" : number_file}
body = [output]
return addadminbox(subtitle, body)
if callback:
return perform_index(colID, ln, "perform_checkexternalcollections", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def get_col_tree(colID, rtype=''):
"""Returns a presentation of the tree as a list. TODO: Add loop detection
colID - startpoint for the tree
rtype - get regular or virtual part of the tree"""
try:
colID = int(colID)
stack = [colID]
ssize = 0
tree = [(colID, 0, 0, colID, 'r')]
while len(stack) > 0:
ccolID = stack.pop()
if ccolID == colID and rtype:
res = run_sql("SELECT id_son, score, type FROM collection_collection WHERE id_dad=%s AND type=%s ORDER BY score ASC,id_son", (ccolID, rtype))
else:
res = run_sql("SELECT id_son, score, type FROM collection_collection WHERE id_dad=%s ORDER BY score ASC,id_son", (ccolID, ))
ssize += 1
ntree = []
for i in range(0, len(res)):
id_son = res[i][0]
score = res[i][1]
rtype = res[i][2]
stack.append(id_son)
if i == (len(res) - 1):
up = 0
else:
up = 1
if i == 0:
down = 0
else:
down = 1
ntree.insert(0, (id_son, up, down, ccolID, rtype))
tree = tree[0:ssize] + ntree + tree[ssize:len(tree)]
return tree
except StandardError, e:
register_exception()
return ()
def add_col_dad_son(add_dad, add_son, rtype):
"""Add a son to a collection (dad)
add_dad - add to this collection id
add_son - add this collection id
rtype - either regular or virtual"""
try:
res = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s ORDER BY score ASC", (add_dad, ))
highscore = 0
for score in res:
if int(score[0]) > highscore:
highscore = int(score[0])
highscore += 1
res = run_sql("INSERT INTO collection_collection(id_dad,id_son,score,type) values(%s,%s,%s,%s)", (add_dad, add_son, highscore, rtype))
return (1, highscore)
except StandardError, e:
register_exception()
return (0, e)
def compare_on_val(first, second):
"""Compare the two values"""
return cmp(first[1], second[1])
def get_col_fld(colID=-1, type = '', id_field=''):
"""Returns either all portalboxes associated with a collection, or based on either colID or language or both.
colID - collection id
ln - language id"""
sql = "SELECT id_field,id_fieldvalue,type,score,score_fieldvalue FROM collection_field_fieldvalue, field WHERE id_field=field.id"
params = []
if colID > -1:
sql += " AND id_collection=%s"
params.append(colID)
if id_field:
sql += " AND id_field=%s"
params.append(id_field)
if type:
sql += " AND type=%s"
params.append(type)
sql += " ORDER BY type, score desc, score_fieldvalue desc"
res = run_sql(sql, tuple(params))
return res
def get_col_pbx(colID=-1, ln='', position = ''):
"""Returns either all portalboxes associated with a collection, or based on either colID or language or both.
colID - collection id
ln - language id"""
sql = "SELECT id_portalbox, id_collection, ln, score, position, title, body FROM collection_portalbox, portalbox WHERE id_portalbox = portalbox.id"
params = []
if colID > -1:
sql += " AND id_collection=%s"
params.append(colID)
if ln:
sql += " AND ln=%s"
params.append(ln)
if position:
sql += " AND position=%s"
params.append(position)
sql += " ORDER BY position, ln, score desc"
res = run_sql(sql, tuple(params))
return res
def get_col_fmt(colID=-1):
"""Returns all formats currently associated with a collection, or for one specific collection
colID - the id of the collection"""
if colID not in [-1, "-1"]:
res = run_sql("SELECT id_format, id_collection, code, score FROM collection_format, format WHERE id_format = format.id AND id_collection=%s ORDER BY score desc", (colID, ))
else:
res = run_sql("SELECT id_format, id_collection, code, score FROM collection_format, format WHERE id_format = format.id ORDER BY score desc")
return res
def get_col_rnk(colID, ln):
""" Returns a list of the rank methods the given collection is attached to
colID - id from collection"""
try:
res1 = dict(run_sql("SELECT id_rnkMETHOD, '' FROM collection_rnkMETHOD WHERE id_collection=%s", (colID, )))
res2 = get_def_name('', "rnkMETHOD")
result = filter(lambda x: res1.has_key(x[0]), res2)
return result
except StandardError, e:
return ()
def get_pbx():
"""Returns all portalboxes"""
res = run_sql("SELECT id, title, body FROM portalbox ORDER by title,body")
return res
def get_fld_value(fldvID = ''):
"""Returns fieldvalue"""
sql = "SELECT id, name, value FROM fieldvalue"
params = []
if fldvID:
sql += " WHERE id=%s"
params.append(fldvID)
sql += " ORDER BY name"
res = run_sql(sql, tuple(params))
return res
def get_pbx_pos():
"""Returns a list of all the positions for a portalbox"""
position = {}
position["rt"] = "Right Top"
position["lt"] = "Left Top"
position["te"] = "Title Epilog"
position["tp"] = "Title Prolog"
position["ne"] = "Narrow by coll epilog"
position["np"] = "Narrow by coll prolog"
return position
def get_sort_nametypes():
"""Return a list of the various translationnames for the fields"""
type = {}
type['soo'] = 'Sort options'
type['seo'] = 'Search options'
type['sew'] = 'Search within'
return type
def get_fmt_nametypes():
"""Return a list of the various translationnames for the output formats"""
type = []
type.append(('ln', 'Long name'))
return type
def get_fld_nametypes():
"""Return a list of the various translationnames for the fields"""
type = []
type.append(('ln', 'Long name'))
return type
def get_col_nametypes():
"""Return a list of the various translationnames for the collections"""
type = []
type.append(('ln', 'Long name'))
return type
def find_last(tree, start_son):
"""Find the previous collection in the tree with the same father as start_son"""
id_dad = tree[start_son][3]
while start_son > 0:
start_son -= 1
if tree[start_son][3] == id_dad:
return start_son
def find_next(tree, start_son):
"""Find the next collection in the tree with the same father as start_son"""
id_dad = tree[start_son][3]
while start_son < len(tree):
start_son += 1
if tree[start_son][3] == id_dad:
return start_son
def remove_col_subcol(id_son, id_dad, type):
"""Remove a collection as a son of another collection in the tree, if collection isn't used elsewhere in the tree, remove all registered sons of the id_son.
id_son - collection id of son to remove
id_dad - the id of the dad"""
try:
if id_son != id_dad:
tree = get_col_tree(id_son)
- res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s", (id_son, id_dad))
+ run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s", (id_son, id_dad))
else:
tree = get_col_tree(id_son, type)
- res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s and type=%s", (id_son, id_dad, type))
- if not run_sql("SELECT * from collection_collection WHERE id_son=%s and type=%s", (id_son, type)):
+ run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s and type=%s", (id_son, id_dad, type))
+ if not run_sql("SELECT id_dad,id_son,type,score from collection_collection WHERE id_son=%s and type=%s", (id_son, type)):
for (id, up, down, dad, rtype) in tree:
- res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s", (id, dad))
+ run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s", (id, dad))
return (1, "")
except StandardError, e:
return (0, e)
def check_col(add_dad, add_son):
"""Check if the collection can be placed as a son of the dad without causing loops.
add_dad - collection id
add_son - collection id"""
try:
stack = [add_dad]
res = run_sql("SELECT id_dad FROM collection_collection WHERE id_dad=%s AND id_son=%s", (add_dad, add_son))
if res:
raise StandardError
while len(stack) > 0:
colID = stack.pop()
res = run_sql("SELECT id_dad FROM collection_collection WHERE id_son=%s", (colID, ))
for id in res:
if int(id[0]) == int(add_son):
raise StandardError
else:
stack.append(id[0])
return (1, "")
except StandardError, e:
return (0, e)
def attach_rnk_col(colID, rnkID):
"""attach rank method to collection
rnkID - id from rnkMETHOD table
colID - id of collection, as in collection table """
try:
res = run_sql("INSERT INTO collection_rnkMETHOD(id_collection, id_rnkMETHOD) values (%s,%s)", (colID, rnkID))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def detach_rnk_col(colID, rnkID):
"""detach rank method from collection
rnkID - id from rnkMETHOD table
colID - id of collection, as in collection table """
try:
res = run_sql("DELETE FROM collection_rnkMETHOD WHERE id_collection=%s AND id_rnkMETHOD=%s", (colID, rnkID))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def switch_col_treescore(col_1, col_2):
try:
res1 = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s and id_son=%s", (col_1[3], col_1[0]))
res2 = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s and id_son=%s", (col_2[3], col_2[0]))
res = run_sql("UPDATE collection_collection SET score=%s WHERE id_dad=%s and id_son=%s", (res2[0][0], col_1[3], col_1[0]))
res = run_sql("UPDATE collection_collection SET score=%s WHERE id_dad=%s and id_son=%s", (res1[0][0], col_2[3], col_2[0]))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def move_col_tree(col_from, col_to, move_to_rtype=''):
"""Move a collection from one point in the tree to another. becomes a son of the endpoint.
col_from - move this collection from current point
col_to - and set it as a son of this collection.
move_to_rtype - either virtual or regular collection"""
try:
res = run_sql("SELECT score FROM collection_collection WHERE id_dad=%s ORDER BY score asc", (col_to[0], ))
highscore = 0
for score in res:
if int(score[0]) > highscore:
highscore = int(score[0])
highscore += 1
if not move_to_rtype:
move_to_rtype = col_from[4]
res = run_sql("DELETE FROM collection_collection WHERE id_son=%s and id_dad=%s", (col_from[0], col_from[3]))
res = run_sql("INSERT INTO collection_collection(id_dad,id_son,score,type) values(%s,%s,%s,%s)", (col_to[0], col_from[0], highscore, move_to_rtype))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def remove_pbx(colID, pbxID, ln):
"""Removes a portalbox from the collection given.
colID - the collection the format is connected to
pbxID - the portalbox which should be removed from the collection.
ln - the language of the portalbox to be removed"""
try:
res = run_sql("DELETE FROM collection_portalbox WHERE id_collection=%s AND id_portalbox=%s AND ln=%s", (colID, pbxID, ln))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def remove_fmt(colID, fmtID):
"""Removes a format from the collection given.
colID - the collection the format is connected to
fmtID - the format which should be removed from the collection."""
try:
res = run_sql("DELETE FROM collection_format WHERE id_collection=%s AND id_format=%s", (colID, fmtID))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def remove_fld(colID, fldID, fldvID=''):
"""Removes a field from the collection given.
colID - the collection the format is connected to
fldID - the field which should be removed from the collection."""
try:
sql = "DELETE FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s"
params = [colID, fldID]
if fldvID:
if fldvID != "None":
sql += " AND id_fieldvalue=%s"
params.append(fldvID)
else:
sql += " AND id_fieldvalue is NULL"
res = run_sql(sql, tuple(params))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def delete_fldv(fldvID):
"""Deletes all data for the given fieldvalue
fldvID - delete all data in the tables associated with fieldvalue and this id"""
try:
res = run_sql("DELETE FROM collection_field_fieldvalue WHERE id_fieldvalue=%s", (fldvID, ))
res = run_sql("DELETE FROM fieldvalue WHERE id=%s", (fldvID, ))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def delete_pbx(pbxID):
"""Deletes all data for the given portalbox
pbxID - delete all data in the tables associated with portalbox and this id """
try:
res = run_sql("DELETE FROM collection_portalbox WHERE id_portalbox=%s", (pbxID, ))
res = run_sql("DELETE FROM portalbox WHERE id=%s", (pbxID, ))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def delete_fmt(fmtID):
"""Deletes all data for the given format
fmtID - delete all data in the tables associated with format and this id """
try:
res = run_sql("DELETE FROM format WHERE id=%s", (fmtID, ))
res = run_sql("DELETE FROM collection_format WHERE id_format=%s", (fmtID, ))
res = run_sql("DELETE FROM formatname WHERE id_format=%s", (fmtID, ))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def delete_col(colID):
"""Deletes all data for the given collection
colID - delete all data in the tables associated with collection and this id """
try:
res = run_sql("DELETE FROM collection WHERE id=%s", (colID, ))
res = run_sql("DELETE FROM collectionname WHERE id_collection=%s", (colID, ))
res = run_sql("DELETE FROM collection_rnkMETHOD WHERE id_collection=%s", (colID, ))
res = run_sql("DELETE FROM collection_collection WHERE id_dad=%s", (colID, ))
res = run_sql("DELETE FROM collection_collection WHERE id_son=%s", (colID, ))
res = run_sql("DELETE FROM collection_portalbox WHERE id_collection=%s", (colID, ))
res = run_sql("DELETE FROM collection_format WHERE id_collection=%s", (colID, ))
res = run_sql("DELETE FROM collection_field_fieldvalue WHERE id_collection=%s", (colID, ))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def add_fmt(code, name, rtype):
"""Add a new output format. Returns the id of the format.
code - the code for the format, max 6 chars.
name - the default name for the default language of the format.
rtype - the default nametype"""
try:
res = run_sql("INSERT INTO format (code, name) values (%s,%s)", (code, name))
fmtID = run_sql("SELECT id FROM format WHERE code=%s", (code,))
res = run_sql("INSERT INTO formatname(id_format, type, ln, value) VALUES (%s,%s,%s,%s)",
(fmtID[0][0], rtype, CFG_SITE_LANG, name))
return (1, fmtID)
except StandardError, e:
register_exception()
return (0, e)
def update_fldv(fldvID, name, value):
"""Modify existing fieldvalue
fldvID - id of fieldvalue to modify
value - the value of the fieldvalue
name - the name of the fieldvalue."""
try:
res = run_sql("UPDATE fieldvalue set name=%s where id=%s", (name, fldvID))
res = run_sql("UPDATE fieldvalue set value=%s where id=%s", (value, fldvID))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def add_fldv(name, value):
"""Add a new fieldvalue, returns id of fieldvalue
value - the value of the fieldvalue
name - the name of the fieldvalue."""
try:
res = run_sql("SELECT id FROM fieldvalue WHERE name=%s and value=%s", (name, value))
if not res:
res = run_sql("INSERT INTO fieldvalue (name, value) values (%s,%s)", (name, value))
res = run_sql("SELECT id FROM fieldvalue WHERE name=%s and value=%s", (name, value))
if res:
return (1, res[0][0])
else:
raise StandardError
except StandardError, e:
register_exception()
return (0, e)
def add_pbx(title, body):
try:
res = run_sql("INSERT INTO portalbox (title, body) values (%s,%s)", (title, body))
res = run_sql("SELECT id FROM portalbox WHERE title=%s AND body=%s", (title, body))
if res:
return (1, res[0][0])
else:
raise StandardError
except StandardError, e:
register_exception()
return (0, e)
def add_col(colNAME, dbquery=None):
"""Adds a new collection to collection table
colNAME - the default name for the collection, saved to collection and collectionname
dbquery - query related to the collection"""
# BTW, sometimes '' are passed instead of None, so change them to None
if not dbquery:
dbquery = None
try:
rtype = get_col_nametypes()[0][0]
colID = run_sql("SELECT id FROM collection WHERE id=1")
if colID:
res = run_sql("INSERT INTO collection (name,dbquery) VALUES (%s,%s)",
(colNAME,dbquery))
else:
res = run_sql("INSERT INTO collection (id,name,dbquery) VALUES (1,%s,%s)",
(colNAME,dbquery))
colID = run_sql("SELECT id FROM collection WHERE name=%s", (colNAME,))
res = run_sql("INSERT INTO collectionname(id_collection, type, ln, value) VALUES (%s,%s,%s,%s)",
(colID[0][0], rtype, CFG_SITE_LANG, colNAME))
if colID:
return (1, colID[0][0])
else:
raise StandardError
except StandardError, e:
register_exception()
return (0, e)
def add_col_pbx(colID, pbxID, ln, position, score=''):
"""add a portalbox to the collection.
colID - the id of the collection involved
pbxID - the portalbox to add
ln - which language the portalbox is for
score - decides which portalbox is the most important
position - position on page the portalbox should appear."""
try:
if score:
res = run_sql("INSERT INTO collection_portalbox(id_portalbox, id_collection, ln, score, position) values (%s,%s,'%s',%s,%s)", (pbxID, colID, ln, score, position))
else:
res = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and ln=%s and position=%s ORDER BY score desc, ln, position", (colID, ln, position))
if res:
score = int(res[0][0])
else:
score = 0
res = run_sql("INSERT INTO collection_portalbox(id_portalbox, id_collection, ln, score, position) values (%s,%s,%s,%s,%s)", (pbxID, colID, ln, (score + 1), position))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def add_col_fmt(colID, fmtID, score=''):
"""Add a output format to the collection.
colID - the id of the collection involved
fmtID - the id of the format.
score - the score of the format, decides sorting, if not given, place the format on top"""
try:
if score:
res = run_sql("INSERT INTO collection_format(id_format, id_collection, score) values (%s,%s,%s)", (fmtID, colID, score))
else:
res = run_sql("SELECT score FROM collection_format WHERE id_collection=%s ORDER BY score desc", (colID, ))
if res:
score = int(res[0][0])
else:
score = 0
res = run_sql("INSERT INTO collection_format(id_format, id_collection, score) values (%s,%s,%s)", (fmtID, colID, (score + 1)))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def add_col_fld(colID, fldID, type, fldvID=''):
"""Add a sort/search/field to the collection.
colID - the id of the collection involved
fldID - the id of the field.
fldvID - the id of the fieldvalue.
type - which type, seo, sew...
score - the score of the format, decides sorting, if not given, place the format on top"""
try:
if fldvID and fldvID not in [-1, "-1"]:
run_sql("DELETE FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type=%s and id_fieldvalue is NULL", (colID, fldID, type))
res = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type=%s ORDER BY score desc", (colID, fldID, type))
if res:
score = int(res[0][0])
res = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s AND id_field=%s and type=%s ORDER BY score_fieldvalue desc", (colID, fldID, type))
else:
res = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and type=%s ORDER BY score desc", (colID, type))
if res:
score = int(res[0][0]) + 1
else:
score = 1
- res = run_sql("SELECT * FROM collection_field_fieldvalue where id_field=%s and id_collection=%s and type=%s and id_fieldvalue=%s", (fldID, colID, type, fldvID))
+ res = run_sql("SELECT id_collection,id_field,id_fieldvalue,type,score,score_fieldvalue FROM collection_field_fieldvalue where id_field=%s and id_collection=%s and type=%s and id_fieldvalue=%s", (fldID, colID, type, fldvID))
if not res:
run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=score_fieldvalue+1 WHERE id_field=%s AND id_collection=%s and type=%s", (fldID, colID, type))
res = run_sql("INSERT INTO collection_field_fieldvalue(id_field, id_fieldvalue, id_collection, type, score, score_fieldvalue) values (%s,%s,%s,%s,%s,%s)", (fldID, fldvID, colID, type, score, 1))
else:
return (0, (1, "Already exists"))
else:
- res = run_sql("SELECT * FROM collection_field_fieldvalue WHERE id_collection=%s AND type=%s and id_field=%s and id_fieldvalue is NULL", (colID, type, fldID))
+ res = run_sql("SELECT id_collection,id_field,id_fieldvalue,type,score,score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s AND type=%s and id_field=%s and id_fieldvalue is NULL", (colID, type, fldID))
if res:
return (0, (1, "Already exists"))
else:
run_sql("UPDATE collection_field_fieldvalue SET score=score+1")
res = run_sql("INSERT INTO collection_field_fieldvalue(id_field, id_collection, type, score,score_fieldvalue) values (%s,%s,%s,%s, 0)", (fldID, colID, type, 1))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def modify_dbquery(colID, dbquery=None):
"""Modify the dbquery of an collection.
colID - the id of the collection involved
dbquery - the new dbquery"""
# BTW, sometimes '' is passed instead of None, so change it to None
if not dbquery:
dbquery = None
try:
res = run_sql("UPDATE collection SET dbquery=%s WHERE id=%s", (dbquery, colID))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def modify_pbx(colID, pbxID, sel_ln, score='', position='', title='', body=''):
"""Modify a portalbox
colID - the id of the collection involved
pbxID - the id of the portalbox that should be modified
sel_ln - the language of the portalbox that should be modified
title - the title
body - the content
score - if several portalboxes in one position, who should appear on top.
position - position on page"""
try:
if title:
res = run_sql("UPDATE portalbox SET title=%s WHERE id=%s", (title, pbxID))
if body:
res = run_sql("UPDATE portalbox SET body=%s WHERE id=%s", (body, pbxID))
if score:
res = run_sql("UPDATE collection_portalbox SET score=%s WHERE id_collection=%s and id_portalbox=%s and ln=%s", (score, colID, pbxID, sel_ln))
if position:
res = run_sql("UPDATE collection_portalbox SET position=%s WHERE id_collection=%s and id_portalbox=%s and ln=%s", (position, colID, pbxID, sel_ln))
return (1, "")
except Exception, e:
register_exception()
return (0, e)
def switch_fld_score(colID, id_1, id_2):
"""Switch the scores of id_1 and id_2 in collection_field_fieldvalue
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s", (colID, id_1))
res2 = run_sql("SELECT score FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s", (colID, id_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
else:
res = run_sql("UPDATE collection_field_fieldvalue SET score=%s WHERE id_collection=%s and id_field=%s", (res2[0][0], colID, id_1))
res = run_sql("UPDATE collection_field_fieldvalue SET score=%s WHERE id_collection=%s and id_field=%s", (res1[0][0], colID, id_2))
return (1, "")
except StandardError, e:
register_exception()
return (0, e)
def switch_fld_value_score(colID, id_1, fldvID_1, fldvID_2):
"""Switch the scores of two field_value
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s", (colID, id_1, fldvID_1))
res2 = run_sql("SELECT score_fieldvalue FROM collection_field_fieldvalue WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s", (colID, id_1, fldvID_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
else:
res = run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s", (res2[0][0], colID, id_1, fldvID_1))
res = run_sql("UPDATE collection_field_fieldvalue SET score_fieldvalue=%s WHERE id_collection=%s and id_field=%s and id_fieldvalue=%s", (res1[0][0], colID, id_1, fldvID_2))
return (1, "")
except Exception, e:
register_exception()
return (0, e)
def switch_pbx_score(colID, id_1, id_2, sel_ln):
"""Switch the scores of id_1 and id_2 in the table given by the argument.
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and id_portalbox=%s and ln=%s", (colID, id_1, sel_ln))
res2 = run_sql("SELECT score FROM collection_portalbox WHERE id_collection=%s and id_portalbox=%s and ln=%s", (colID, id_2, sel_ln))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
res = run_sql("UPDATE collection_portalbox SET score=%s WHERE id_collection=%s and id_portalbox=%s and ln=%s", (res2[0][0], colID, id_1, sel_ln))
res = run_sql("UPDATE collection_portalbox SET score=%s WHERE id_collection=%s and id_portalbox=%s and ln=%s", (res1[0][0], colID, id_2, sel_ln))
return (1, "")
except Exception, e:
register_exception()
return (0, e)
def switch_score(colID, id_1, id_2, table):
"""Switch the scores of id_1 and id_2 in the table given by the argument.
colID - collection the id_1 or id_2 is connected to
id_1/id_2 - id field from tables like format..portalbox...
table - name of the table"""
try:
res1 = run_sql("SELECT score FROM collection_%s WHERE id_collection=%%s and id_%s=%%s" % (table, table), (colID, id_1))
res2 = run_sql("SELECT score FROM collection_%s WHERE id_collection=%%s and id_%s=%%s" % (table, table), (colID, id_2))
if res1[0][0] == res2[0][0]:
return (0, (1, "Cannot rearrange the selected fields, either rearrange by name or use the mySQL client to fix the problem."))
res = run_sql("UPDATE collection_%s SET score=%%s WHERE id_collection=%%s and id_%s=%%s" % (table, table), (res2[0][0], colID, id_1))
res = run_sql("UPDATE collection_%s SET score=%%s WHERE id_collection=%%s and id_%s=%%s" % (table, table), (res1[0][0], colID, id_2))
return (1, "")
except Exception, e:
register_exception()
return (0, e)
def get_detailed_page_tabs(colID=None, recID=None, ln=CFG_SITE_LANG):
"""
Returns the complete list of tabs to be displayed in the
detailed record pages.
Returned structured is a dict with
- key : last component of the url that leads to detailed record tab: http:www.../record/74/key
- values: a dictionary with the following keys:
- label: *string* label to be printed as tab (Not localized here)
- visible: *boolean* if False, tab should not be shown
- enabled: *boolean* if True, tab should be disabled
- order: *int* position of the tab in the list of tabs
- ln: language of the tab labels
returns dict
"""
_ = gettext_set_language(ln)
tabs = {'metadata' : {'label': _('Information'), 'visible': False, 'enabled': True, 'order': 1},
'references': {'label': _('References'), 'visible': False, 'enabled': True, 'order': 2},
'citations' : {'label': _('Citations'), 'visible': False, 'enabled': True, 'order': 3},
'keywords' : {'label': _('Keywords'), 'visible': False, 'enabled': True, 'order': 4},
'comments' : {'label': _('Discussion'), 'visible': False, 'enabled': True, 'order': 5},
'usage' : {'label': _('Usage statistics'), 'visible': False, 'enabled': True, 'order': 6},
'files' : {'label': _('Fulltext'), 'visible': False, 'enabled': True, 'order': 7},
'holdings' : {'label': _('Holdings'), 'visible': False, 'enabled': True, 'order': 8},
}
res = run_sql("SELECT tabs FROM collectiondetailedrecordpagetabs " + \
"WHERE id_collection=%s", (colID, ))
if len(res) > 0:
tabs_state = res[0][0].split(';')
for tab_state in tabs_state:
if tabs.has_key(tab_state):
tabs[tab_state]['visible'] = True;
else:
# no preference set for this collection.
# assume all tabs are displayed
for key in tabs.keys():
tabs[key]['visible'] = True
if not CFG_WEBCOMMENT_ALLOW_COMMENTS and \
not CFG_WEBCOMMENT_ALLOW_REVIEWS:
tabs['comments']['visible'] = False
tabs['comments']['enabled'] = False
if recID is not None:
# Disable references if no references found
#bfo = BibFormatObject(recID)
#if bfe_references.format(bfo, '', '') == '':
# tabs['references']['enabled'] = False
## FIXME: the above was commented out because bfe_references
## may be too slow. And we do not really need this anyway
## because we can disable tabs in WebSearch Admin on a
## collection-by-collection basis. If we need this, then we
## should probably call bfo.fields('999') here that should be
## much faster than calling bfe_references.
# Disable citations if not citations found
#if len(get_cited_by(recID)) == 0:
# tabs['citations']['enabled'] = False
## FIXME: the above was commented out because get_cited_by()
## may be too slow. And we do not really need this anyway
## because we can disable tags in WebSearch Admin on a
## collection-by-collection basis.
# Disable fulltext if no file found
if not CFG_INSPIRE_SITE:
brd = BibRecDocs(recID)
if len(brd.list_bibdocs()) == 0:
tabs['files']['enabled'] = False
#Disable holdings tab if collection != Books
collection = run_sql("""select name from collection where id=%s""", (colID, ))
if collection[0][0] != 'Books':
tabs['holdings']['enabled'] = False
tabs[''] = tabs['metadata']
del tabs['metadata']
return tabs
diff --git a/modules/webstat/lib/webstat_engine.py b/modules/webstat/lib/webstat_engine.py
index e7ea2c228..fbb4ff76c 100644
--- a/modules/webstat/lib/webstat_engine.py
+++ b/modules/webstat/lib/webstat_engine.py
@@ -1,733 +1,733 @@
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__revision__ = "$Id$"
__lastupdated__ = "$Date$"
import calendar, commands, datetime, time, os, cPickle
from invenio.config import CFG_TMPDIR, CFG_SITE_URL
from invenio.urlutils import redirect_to_url
from invenio.search_engine import perform_request_search
from invenio.dbquery import run_sql, escape_string
WEBSTAT_SESSION_LENGTH = 48*60*60 # seconds
WEBSTAT_GRAPH_TOKENS = '-=#+@$%&XOSKEHBC'
# KEY EVENT TREND SECTION
def get_keyevent_trend_collection_population(args):
"""
Returns the quantity of documents in CDS Invenio for
the given timestamp range.
@param args['collection']: A collection name
@type args['collection']: str
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
"""
# Collect list of timestamps of insertion in the specific collection
ids = perform_request_search(cc=args['collection'])
if len(ids) == 0:
return []
# collect action dates
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
ids_str = str(ids).replace('[', '(').replace(']', ')')
sql_query = ("SELECT creation_date FROM bibrec WHERE id IN %s AND creation_date > '%s'" + \
"AND creation_date < '%s' ORDER BY creation_date DESC") % \
(ids_str, lower, upper)
action_dates = [x[0] for x in run_sql(sql_query)]
sql_query = "SELECT COUNT(id) FROM bibrec WHERE id IN %s AND creation_date < '%s'" % \
(ids_str,lower)
initial_quantity = run_sql(sql_query)[0][0]
return _get_trend_from_actions(action_dates, initial_quantity,
args['t_start'], args['t_end'], args['granularity'], args['t_format'])
def get_keyevent_trend_search_frequency(args):
"""
Returns the number of searches (of any kind) carried out
during the given timestamp range.
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
"""
# collect action dates
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
sql = "SELECT date FROM query INNER JOIN user_query ON id=id_query " + \
"WHERE date > '%s' AND date < '%s' ORDER BY date DESC" % \
(lower, upper)
action_dates = [x[0] for x in run_sql(sql)]
return _get_trend_from_actions(action_dates, 0, args['t_start'], args['t_end'], args['granularity'], args['t_format'])
def get_keyevent_trend_search_type_distribution(args):
"""
Returns the number of searches carried out during the given
timestamp range, but also partion them by type Simple and
Advanced.
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
"""
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
# SQL to determine all simple searches:
sql = "SELECT date FROM query INNER JOIN user_query ON id=id_query WHERE urlargs LIKE '%p=%' " + \
"AND date > '%s' AND date < '%s' ORDER BY date DESC" % (lower, upper)
simple = [x[0] for x in run_sql(sql)]
# SQL to determine all advanced searches:
sql = "SELECT date FROM query INNER JOIN user_query ON id=id_query WHERE urlargs LIKE '%as=1%' " + \
"AND date > '%s' AND date < '%s' ORDER BY date DESC" % (lower, upper)
advanced = [x[0] for x in run_sql(sql)]
# Compute the trend for both types
s_trend = _get_trend_from_actions(simple, 0, args['t_start'], args['t_end'], args['granularity'], args['t_format'])
a_trend = _get_trend_from_actions(advanced, 0, args['t_start'], args['t_end'], args['granularity'], args['t_format'])
# Assemble, according to return type
return [(s_trend[i][0], (s_trend[i][1], a_trend[i][1])) for i in range(len(s_trend))]
def get_keyevent_trend_download_frequency(args):
"""
Returns the number of full text downloads carried out
during the given timestamp range.
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
"""
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
sql = "SELECT download_time FROM rnkDOWNLOADS WHERE download_time > '%s' \
AND download_time < '%s' ORDER BY download_time DESC" % (lower, upper)
actions = [x[0] for x in run_sql(sql)]
return _get_trend_from_actions(actions, 0, args['t_start'], args['t_end'], args['granularity'], args['t_format'])
# KEY EVENT SNAPSHOT SECTION
def get_keyevent_snapshot_uptime_cmd():
"""
A specific implementation of get_current_event().
@return: The std-out from the UNIX command 'uptime'.
@type: str
"""
return _run_cmd('uptime').strip().replace(' ', ' ')
def get_keyevent_snapshot_apache_processes():
"""
A specific implementation of get_current_event().
@return: The std-out from the UNIX command 'uptime'.
@type: str
"""
# The number of Apache processes (root+children)
return _run_cmd('ps -e | grep apache2 | grep -v grep | wc -l')
def get_keyevent_snapshot_bibsched_status():
"""
A specific implementation of get_current_event().
@return: Information about the number of tasks in the different status modes.
@type: [(str, int)]
"""
sql = "SELECT status, COUNT(status) FROM schTASK GROUP BY status"
return [(x[0], int(x[1])) for x in run_sql(sql)]
def get_keyevent_snapshot_sessions():
"""
A specific implementation of get_current_event().
@return: The current number of website visitors (guests, logged in)
@type: (int, int)
"""
# SQL to retrieve sessions in the Guests
sql = "SELECT COUNT(session_expiry) FROM session INNER JOIN user ON uid=id " + \
"WHERE email = '' AND " + \
"session_expiry-%d < unix_timestamp() AND " % WEBSTAT_SESSION_LENGTH + \
"unix_timestamp() < session_expiry"
guests = run_sql(sql)[0][0]
# SQL to retrieve sessions in the Logged in users
sql = "SELECT COUNT(session_expiry) FROM session INNER JOIN user ON uid=id " + \
"WHERE email <> '' AND " + \
"session_expiry-%d < unix_timestamp() AND " % WEBSTAT_SESSION_LENGTH + \
"unix_timestamp() < session_expiry"
logged_ins = run_sql(sql)[0][0]
# Assemble, according to return type
return (guests, logged_ins)
# CUSTOM EVENT SECTION
def get_customevent_trend(args):
"""
Returns trend data for a custom event over a give
timestamp range.
@param args['id']: The event id
@type args['id']: str
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
@param args['cols']: Columns and it's content that will be include
if don't exist or it's empty it will include all cols
@type args['cols']: [ [ str, str ], ]
"""
# Get a MySQL friendly date
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
tbl_name = get_customevent_table(args['id'])
col_names = get_customevent_args(args['id'])
sql_query = ["SELECT creation_time FROM %s WHERE creation_time > '%s'" % (tbl_name, lower)]
sql_query.append("AND creation_time < '%s'" % upper)
sql_param = []
for col_bool, col_title, col_content in args['cols']:
if not col_title in col_names:
continue
if col_content:
if col_bool == "and" or col_bool == "":
sql_query.append("AND `%s`" % escape_string(col_title))
elif col_bool == "or":
sql_query.append("OR `%s`" % escape_string(col_title))
elif col_bool == "and_not":
sql_query.append("AND NOT `%s`" % escape_string(col_title))
else:
continue
sql_query.append(" LIKE %s")
sql_param.append("%" + col_content + "%")
sql_query.append("ORDER BY creation_time DESC")
sql = ' '.join(sql_query)
dates = [x[0] for x in run_sql(sql, tuple(sql_param))]
return _get_trend_from_actions(dates, 0, args['t_start'], args['t_end'], args['granularity'], args['t_format'])
def get_customevent_dump(args):
"""
Similar to a get_event_trend implemention, but NO refining aka frequency
handling is carried out what so ever. This is just a dump. A dump!
@param args['id']: The event id
@type args['id']: str
@param args['t_start']: Date and time of start point
@type args['t_start']: str
@param args['t_end']: Date and time of end point
@type args['t_end']: str
@param args['granularity']: Granularity of date and time
@type args['granularity']: str
@param args['t_format']: Date and time formatting string
@type args['t_format']: str
@param args['cols']: Columns and it's content that will be include
if don't exist or it's empty it will include all cols
@type args['cols']: [ [ str, str ], ]
"""
# Get a MySQL friendly date
lower = _to_datetime(args['t_start'], args['t_format']).isoformat()
upper = _to_datetime(args['t_end'], args['t_format']).isoformat()
# Get customevents
# events_list = [(creation_time, event, [arg1, arg2, ...]), ...]
event_list = []
event_cols = {}
for id, i in [ (args['ids'][i], str(i)) for i in range(len(args['ids']))]:
# Get all the event arguments and creation times
tbl_name = get_customevent_table(id)
col_names = get_customevent_args(id)
- sql_query = ["SELECT * FROM %s WHERE creation_time > '%s'" % (tbl_name, lower)]
+ sql_query = ["SELECT * FROM %s WHERE creation_time > '%s'" % (tbl_name, lower)] # Note: SELECT * technique is okay here
sql_query.append("AND creation_time < '%s'" % upper)
sql_param = []
for col_bool, col_title, col_content in args['cols'+i]:
if not col_title in col_names: continue
if col_content:
if col_bool == "and" or col_bool == "":
sql_query.append("AND `%s`" % escape_string(col_title))
elif col_bool == "or":
sql_query.append("OR `%s`" % escape_string(col_title))
elif col_bool == "and_not":
sql_query.append("AND NOT `%s`" % escape_string(col_title))
else:
continue
sql_query.append(" LIKE %s")
sql_param.append("%" + col_content + "%")
sql_query.append("ORDER BY creation_time DESC")
sql = ' '.join(sql_query)
res = run_sql(sql, tuple(sql_param))
for row in res:
event_list.append((row[1],id,row[2:]))
# Get the event col names
try:
event_cols[id] = cPickle.loads(run_sql("SELECT cols FROM staEVENT WHERE id = %s", (id,))[0][0])
except TypeError:
event_cols[id] = ["Unnamed"]
event_list.sort()
output = []
for row in event_list:
temp = [row[1], row[0].strftime('%Y-%m-%d %H:%M:%S')]
arguments = ["%s: %s" % (event_cols[row[1]][i], row[2][i]) for i in range(len(row[2]))]
temp.extend(arguments)
output.append(tuple(temp))
return output
def get_customevent_table(id):
"""
Helper function that for a certain event id retrives the corresponding
event table name.
"""
res = run_sql("SELECT CONCAT('staEVENT', number) FROM staEVENT WHERE id = %s", (id,))
try:
return res[0][0]
except IndexError:
# No such event table
return None
def get_customevent_args(id):
"""
Helper function that for a certain event id retrives the corresponding
event argument (column) names.
"""
res = run_sql("SELECT cols FROM staEVENT WHERE id = %s", (id,))
try:
if res[0][0]:
return cPickle.loads(res[0][0])
else:
return []
except IndexError:
# No such event table
return None
# GRAPHER
def create_graph_trend(trend, path, settings):
"""
Creates a graph representation out of data produced from get_event_trend.
@param trend: The trend data
@type trend: [(str, str|int|(str|int,...))]
@param path: Where to store the graph
@type path: str
@param settings: Dictionary of graph parameters
@type settings: dict
"""
# If no input, we don't bother about anything
if len(trend) == 0:
return
# If no filename is given, we'll assume STD-out format and ASCII.
if path == '':
settings["format"] = 'asciiart'
if settings["format"] == 'asciiart':
out = ""
if settings["multiple"] is not None:
# Tokens that will represent the different data sets (maximum 16 sets)
# Set index (=100) to the biggest of the histogram sums
index = max([sum(x[1]) for x in trend])
# Print legend box
out += "Legend: %s\n\n" % ", ".join(["%s (%s)" % x for x in zip(settings["multiple"], WEBSTAT_GRAPH_TOKENS)])
else:
index = max([x[1] for x in trend])
width = 82
# Figure out the max length of the xtics, in order to left align
xtic_max_len = max([len(_to_datetime(x[0]).strftime(settings["xtic_format"])) for x in trend])
for row in trend:
# Print the xtic
xtic = _to_datetime(row[0]).strftime(settings["xtic_format"])
out_row = xtic + ': ' + ' '*(xtic_max_len-len(xtic)) + '|'
try:
col_width = (1.0*width/index)
except ZeroDivisionError:
col_width = 0
if settings["multiple"] is not None:
# The second value of the row-tuple, represents the n values from the n data
# sets. Each set, will be represented by a different ASCII character, chosen
# from the randomized string 'WEBSTAT_GRAPH_TOKENS'. NOTE: Only up to 16 (len(WEBSTAT_GRAPH_TOKENS)) data
# sets are supported.
total = sum(row[1])
for i in range(len(row[1])):
col = row[1][i]
try:
out_row += WEBSTAT_GRAPH_TOKENS[i]*int(1.0*col*col_width)
except ZeroDivisionError:
break
if len([i for i in row[1] if type(i) is int and i > 0]) - 1 > 0:
out_row += out_row[-1]
else:
total = row[1]
try:
out_row += '-'*int(1.0*total*col_width)
except ZeroDivisionError:
break
# Print sentinel, and the total
out += out_row + '>' + ' '*(xtic_max_len+4+width-len(out_row)) + str(total) + '\n'
# Write to destination file
if path == '':
print out
else:
open(path, 'w').write(out)
elif settings["format"] == 'gnuplot':
try:
import Gnuplot
except ImportError:
return
g = Gnuplot.Gnuplot()
g('set style data linespoints')
g('set terminal png small')
g('set output "%s"' % path)
if settings["title"] != '':
g.title(settings["title"])
if settings["xlabel"] != '':
g.xlabel(settings["xlabel"])
if settings["ylabel"] != '':
g.ylabel(settings["ylabel"])
if settings["xtic_format"] != '':
xtics = 'set xtics ('
xtics += ', '.join(['"%s" %d' %
(_to_datetime(trend[i][0], '%Y-%m-%d \
%H:%M:%S').strftime(settings["xtic_format"]), i)
for i in range(len(trend))]) + ')'
g(xtics)
g('set format y "%.0f"')
# If we have multiple data sets, we need to do some magic to make Gnuplot eat it,
# This is basically a matrix transposition, and the addition of index numbers.
if settings["multiple"] is not None:
cols = len(trend[0][1])
rows = len(trend)
plot_items = []
y_max = 0
for col in range(cols):
data = []
for row in range(rows):
data.append([row, trend[row][1][col]])
plot_items.append(Gnuplot.PlotItems.Data(data, title=settings["multiple"][col]))
tmp_max = max(data[col])
if tmp_max > y_max:
y_max = tmp_max
if y_max < 5:
g('set ytic 1')
g.plot(*plot_items)
else:
data = [x[1] for x in trend]
y_max = max(data)
if y_max < 5:
g('set ytic 1')
g.plot(data)
def create_graph_dump(dump, path, settings):
"""
Creates a graph representation out of data produced from get_event_trend.
@param dump: The dump data
@type dump: [(str|int,...)]
@param path: Where to store the graph
@type path: str
@param graph_settings: Dictionary of graph parameters
@type graph_settings: dict
"""
out = ""
if len(dump) == 0:
out += "No actions for this custom event are registered in the given time range."
else:
# Make every row in dump equally long, insert None if appropriate.
max_len = max([len(x) for x in dump])
events = [tuple(list(x) + [None]*(max_len-len(x))) for x in dump]
cols = ["Event", "Date and time"] + ["Argument %d" % i for i in range(max_len-2)]
column_widths = [max([len(str(x[i])) for x in events + [cols]])+3 for i in range(len(events[0]))]
for i in range(len(cols)):
out += cols[i] + ' '*(column_widths[i] - len(cols[i]))
out += "\n"
for i in range(len(cols)):
out += '='*(len(cols[i])) + ' '*(column_widths[i] - len(cols[i]))
out += "\n\n"
for action in dump:
for i in range(len(action)):
if action[i] is None:
temp = ''
else:
temp = action[i]
out += str(temp) + ' '*(column_widths[i] - len(str(temp)))
out += "\n"
# Write to destination file
if path == '':
print out
else:
open(path, 'w').write(out)
# EXPORTER
def export_to_python(data, req):
"""
Exports the data to Python code.
@param data: The Python data that should be exported
@type data: []
@param req: The Apache request object
@type req:
"""
_export("text/x-python", str(data), req)
def export_to_csv(data, req):
"""
Exports the data to CSV.
@param data: The Python data that should be exported
@type data: []
@param req: The Apache request object
@type req:
"""
csv_list = [""""%s",%s""" % (x[0], ",".join([str(y) for y in ((type(x[1]) is tuple) and x[1] or (x[1],))])) for x in data]
_export('text/csv', '\n'.join(csv_list), req)
# INTERNAL
def _export(mime, content, req):
"""
Helper function to pass on the export call. Create a
temporary file in which the content is stored, then let
redirect to the export web interface.
"""
filename = CFG_TMPDIR + "/webstat_export_" + str(time.time()).replace('.', '')
open(filename, 'w').write(content)
redirect_to_url(req, '%s/stats/export?filename=%s&mime=%s' % (CFG_SITE_URL, os.path.basename(filename), mime))
def _get_trend_from_actions(action_dates, initial_value, t_start, t_end, granularity, format):
"""
Given a list of dates reflecting some sort of action/event, and some additional parameters,
an internal data format is returned. 'initial_value' set to zero, means that the frequency
will not be accumulative, but rather non-causal.
@param action_dates: A list of dates, indicating some sort of action/event.
@type action_dates: [datetime.datetime]
@param initial_value: The numerical offset the first action's value should make use of.
@type initial_value: int
@param t_start: Start time for the time domain in format %Y-%m-%d %H:%M:%S
@type t_start: str
@param t_stop: End time for the time domain in format %Y-%m-%d %H:%M:%S
@type t_stop: str
@param granularity: The granularity of the time domain, span between values.
Possible values are [year,month,day,hour,minute,second].
@type granularity: str
@param format: Format of the 't_start' and 't_stop' parameters
@type format: str
@return: A list of tuples zipping a time-domain and a value-domain
@type: [(str, int)]
"""
# Append the maximum date as a sentinel indicating we're done
action_dates.insert(0, datetime.datetime.max)
# Create an iterator running from the first day of activity
dt_iter = _get_datetime_iter(t_start, granularity, format)
# Construct the datetime tuple for the stop time
stop_at = _to_datetime(t_end, format) - datetime.timedelta(seconds=1)
# If our t_start is more recent than the initial action_dates, we need to
# drop those.
t_start_dt = _to_datetime(t_start, format)
while action_dates[-1] < t_start_dt:
action_dates = action_dates[:-1]
vector = [(None, initial_value)]
old = dt_iter.next()
upcoming_action = action_dates.pop()
for current in dt_iter:
# Counter of action_dates in the current span, set the initial value to
# zero to avoid accumlation.
if initial_value != 0:
actions_here = vector[-1][1]
else:
actions_here = 0
# Check to see if there's an action date in the current span
while old <= upcoming_action < current:
actions_here += 1
try:
upcoming_action = action_dates.pop()
except IndexError:
upcoming_action = datetime.datetime.max
vector.append((old.strftime('%Y-%m-%d %H:%M:%S'), actions_here))
old = current
# Make sure to stop the iteration at the end time
if current > stop_at:
break
# Remove the first bogus tuple, and return
return vector[1:]
def _get_datetime_iter(t_start, granularity='day', format='%Y-%m-%d %H:%M:%S'):
"""
Returns an iterator over datetime elements starting at an arbitrary time,
with granularity of a [year,month,day,hour,minute,second].
@param t_start: An arbitrary starting time in format %Y-%m-%d %H:%M:%S
@type t_start: str
@param granularity: The span between iterable elements, default is 'days'.
Possible values are [year,month,day,hour,minute,second].
@type granularity: str
@param format: Format of the 't_start' parameter
@type format: str
@return: An iterator of points in time
@type: iterator over datetime elements
"""
t = _to_datetime(t_start, format)
# Make a time increment depending on the granularity and the current time
# (the length of years and months vary over time)
span = ""
while True:
yield t
if granularity == "year":
span = (calendar.isleap(t.year) and ["days=366"] or ["days=365"])[0]
elif granularity == "month":
span = "days=" + str(calendar.monthrange(t.year, t.month)[1])
elif granularity == "day":
span = "days=1"
elif granularity == "hour":
span = "hours=1"
elif granularity == "minute":
span = "minutes=1"
elif granularity == "second":
span = "seconds=1"
else:
# Default just in case
span = "days=1"
t += eval("datetime.timedelta(" + span + ")")
def _to_datetime(dt, format='%Y-%m-%d %H:%M:%S'):
return datetime.datetime(*time.strptime(dt, format)[:6])
def _run_cmd(command):
"""
Runs a certain command and returns the string output. If the command is
not found a string saying so will be returned. Use with caution!
@param command: The UNIX command to execute.
@type command: str
@return: The std-out from the command.
@type: str
"""
return commands.getoutput(command)
""" % (CFG_SITE_URL, colID, ln, pbxID, res[i - 1][0], tln, random.randint(0, 1000), CFG_SITE_URL)
else:
move += " "
move += "
""" % (CFG_SITE_URL, colID, ln, pbxID, res[i][0], tln, random.randint(0, 1000), CFG_SITE_URL)
move += """
""" % (CFG_SITE_URL, colID, ln, move_from, rtype, i, rtype, CFG_SITE_URL, col_dict[tree_from[int(move_from[1:len(move_from)])][0]], col_dict[tree_to[i][0]])
else:
try:
text += """
""" % (CFG_SITE_URL, colID, ln, rtype, i, rtype, tree[i][0], CFG_SITE_URL, col_dict[tree[i][0]])
except KeyError:
pass
else:
text += """
""" % CFG_SITE_URL
else:
text += """
""" % (CFG_SITE_URL, colID, ln, i, rtype, tree[i][0], CFG_SITE_URL)
except KeyError:
pass
elif i != 0:
text += """