)?[0-9]{9}\s001__\s\d+($|\n)', '', hmbuffer)
def compare_hmbuffers(hmbuffer1, hmbuffer2):
"""Compare two HM (HTML MARC) buffers by removing whitespaces
before testing.
"""
hmbuffer1 = hmbuffer1.strip()
hmbuffer2 = hmbuffer2.strip()
# remove eventual ...
formatting:
hmbuffer1 = re.sub(r'^', '', hmbuffer1)
hmbuffer2 = re.sub(r'^', '', hmbuffer2)
hmbuffer1 = re.sub(r'$', '', hmbuffer1)
hmbuffer2 = re.sub(r'$', '', hmbuffer2)
# remove 005 revision numbers:
hmbuffer1 = re.sub(r'(^|\n)[0-9]{9}\s005.*($|\n)', '\n', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)[0-9]{9}\s005.*($|\n)', '\n', hmbuffer2)
hmbuffer1 = hmbuffer1.strip()
hmbuffer2 = hmbuffer2.strip()
# remove leading recid, leaving only field values:
hmbuffer1 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer2)
# remove leading whitespace:
hmbuffer1 = re.sub(r'(^|\n)\s+', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)\s+', '', hmbuffer2)
compared_hmbuffers = hmbuffer1 == hmbuffer2
if not compared_hmbuffers:
return "\n=" + hmbuffer1 + "=\n" + '!=' + "\n=" + hmbuffer2 + "=\n"
return ''
def wipe_out_record_from_all_tables(recid):
"""
Wipe out completely the record and all its traces of RECID from
the database (bibrec, bibrec_bibxxx, bibxxx, bibfmt). Useful for
the time being for test cases.
"""
# delete all the linked bibdocs
try:
for bibdoc in BibRecDocs(recid).list_bibdocs():
bibdoc.expunge()
# delete from bibrec:
run_sql("DELETE FROM bibrec WHERE id=%s", (recid,))
# delete from bibrec_bibxxx:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE FROM %(bibrec_bibxxx)s WHERE id_bibrec=%%s" % # kwalitee: disable=sql
{'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)},
(recid,))
# delete all unused bibxxx values:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE %(bibxxx)s FROM %(bibxxx)s " \
" LEFT JOIN %(bibrec_bibxxx)s " \
" ON %(bibxxx)s.id=%(bibrec_bibxxx)s.id_bibxxx " \
" WHERE %(bibrec_bibxxx)s.id_bibrec IS NULL" % \
{'bibxxx': "bib%i%ix" % (i, j),
'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)})
# delete from bibfmt:
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s", (recid,))
# delete from bibrec_bibdoc:
run_sql("DELETE FROM bibrec_bibdoc WHERE id_bibrec=%s", (recid,))
# delete from holdingpen
run_sql("DELETE FROM bibHOLDINGPEN WHERE id_bibrec=%s", (recid,))
# delete from hstRECORD
run_sql("DELETE FROM hstRECORD WHERE id_bibrec=%s", (recid,))
except Exception, err:
print >> sys.stderr, "Exception captured while wiping records: %s" % err
def try_url_download(url):
"""Try to download a given URL"""
try:
open_url = urlopen(url)
open_url.read()
except Exception, e:
raise StandardError("Downloading %s is impossible because of %s"
% (url, str(e)))
return True
class GenericBibUploadTest(InvenioTestCase):
"""Generic BibUpload testing class with predefined
setUp and tearDown methods.
"""
def setUp(self):
self.verbose = 0
setup_loggers()
task_set_task_param('verbose', self.verbose)
self.last_recid = run_sql("SELECT MAX(id) FROM bibrec")[0][0]
self.tear_down = True ## For debugging, whether to call tearDown
self.webcolled_recids = [] ## List of record webcolled to be re-webcolled upon tearDown
def tearDown(self):
if self.tear_down:
for recid in run_sql("SELECT id FROM bibrec WHERE id>%s", (self.last_recid,)):
wipe_out_record_from_all_tables(recid[0])
for recid in list(self.webcolled_recids):
self.force_webcoll(recid)
def force_webcoll(self, recid):
self.webcolled_recids.append(recid)
from invenio.bibindex_engine_config import CFG_BIBINDEX_INDEX_TABLE_TYPE
from invenio import bibindex_engine
from invenio import websearch_webcoll
## Reset the collection global cache
websearch_webcoll.COLLECTION_HOUSE = {}
bibindex_engine.WordTable("collection",
table_type=CFG_BIBINDEX_INDEX_TABLE_TYPE["Words"]
).add_recIDs([[recid, recid]], 1)
#sleep 1s to make sure all tables are ready
time.sleep(1)
c = websearch_webcoll.Collection()
c.calculate_reclist()
c.update_reclist()
def check_record_consistency(self, recid):
rec_in_history = create_record(decompress(run_sql("SELECT marcxml FROM hstRECORD WHERE id_bibrec=%s ORDER BY job_date DESC LIMIT 1", (recid, ))[0][0]))[0]
rec_in_xm = create_record(decompress(run_sql("SELECT value FROM bibfmt WHERE id_bibrec=%s AND format='xm'", (recid, ))[0][0]))[0]
rec_in_bibxxx = create_record(get_record_from_bibxxx(recid))[0]
self.failUnless(records_identical(rec_in_xm, rec_in_history, skip_005=False), "\n%s\n!=\n%s\n" % (rec_in_xm, rec_in_history))
self.failUnless(records_identical(rec_in_xm, rec_in_bibxxx, skip_005=False, ignore_duplicate_subfields=True, ignore_duplicate_controlfields=True), "\n%s\n!=\n%s\n" % (rec_in_xm, rec_in_bibxxx))
if CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE:
rec_in_recstruct = loads(decompress(run_sql("SELECT value FROM bibfmt WHERE id_bibrec=%s AND format='recstruct'", (recid, ))[0][0]))
self.failUnless(records_identical(rec_in_xm, rec_in_recstruct, skip_005=False, ignore_subfield_order=True), "\n%s\n!=\n%s\n" % (rec_in_xm, rec_in_recstruct))
class BibUploadRealCaseRemovalDOIViaBibEdit(GenericBibUploadTest):
def test_removal_of_doi_via_bibedit(self):
test = """
HEP
Fiore, Gaetano
On quantum mechanics with a magnetic field on R**n and on a torus T**n, and their relation
Int.J.Theor.Phys.
52
877-896
2013
INSPIRE
General Physics
Published
20
2013
author
Bloch theory with magnetic field
author
Fiber bundles
author
Gauge symmetry
author
Quantization on manifolds
Springer
We show in elementary terms the equivalence in a general gauge of a U(1)-gauge theory of a scalar charged particle on a torus to the analogous theory on ℝ( )n( ) constrained by quasiperiodicity under translations in the lattice Λ. The latter theory provides a global description of the former: the quasiperiodic wavefunctions ψ defined on ℝ( )n( ) play the role of sections of the associated hermitean line bundle E on , since also E admits a global description as a quotient. The components of the covariant derivatives corresponding to a constant (necessarily integral) magnetic field B=dA generate a Lie algebra g ( )Q( ) and together with the periodic functions the algebra of observables . The non-abelian part of g ( )Q( ) is a Heisenberg Lie algebra with the electric charge operator Q as the central generator, the corresponding Lie group G ( )Q( ) acts on the Hilbert space as the translation group up to phase factors. Also the space of sections of E is mapped into itself by g∈G ( )Q( ). We identify the socalled magnetic translation group as a subgroup of the observables’ group Y ( )Q( ). We determine the unitary irreducible representations of corresponding to integer charges and for each of them an associated orthonormal basis explicitly in configuration space. We also clarify how in the n=2m case a holomorphic structure and Theta functions arise on the associated complex torus.
DOI
10.1007/s10773-012-1396-z
Fiore:2013nua
INSPIRETeX
Published
Citeable
"""
recs = create_record(test)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
new_rec = get_record(recid)
del new_rec['024'] ## let's delete DOI
_, recid2, _ = bibupload.bibupload(new_rec, opt_mode='replace')
self.assertEqual(recid, recid2)
self.check_record_consistency(recid2)
class BibUploadTypicalBibEditSessionTest(GenericBibUploadTest):
"""Testing a typical BibEdit session"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.test = """
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
recs = bibupload.xml_marc_to_records(self.test)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(self.recid)
# We retrieve the inserted xml
inserted_xm = print_record(self.recid, 'xm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.test), '')
self.history = run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, )) # kwalitee: disable=sql
self.timestamp = run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,))
self.tag005 = get_record(self.recid)['005'][0][3]
def test_simple_replace(self):
"""BibUpload - test a simple replace as in BibEdit"""
marc_to_replace1 = """
%(recid)s
%(tag005)s
SzGeCERN
Test, Foo
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
bla bla bla
""" % {'recid': self.recid, 'tag005': self.tag005}
recs = bibupload.xml_marc_to_records(marc_to_replace1)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(self.recid)
## The change should have been applied!
self.failUnless(records_identical(recs[0], get_record(self.recid)), "\n%s\n!=\n%s\n" % (recs[0], get_record(self.recid)))
marc_to_replace2 = """
%(recid)s
%(tag005)s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
Queen Elisabeth
Great Britain
""" % {'recid': self.recid, 'tag005': self.tag005}
expected_marc = """
%(recid)s
%(tag005)s
SzGeCERN
Test, Foo
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
bla bla bla
Queen Elisabeth
Great Britain
""" % {'recid': self.recid, 'tag005': self.tag005}
recs = bibupload.xml_marc_to_records(marc_to_replace2)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(self.recid)
## The change should have been merged with the previous without conflict
self.failUnless(records_identical(bibupload.xml_marc_to_records(expected_marc)[0], get_record(self.recid)))
def test_replace_with_conflict(self):
"""BibUpload - test a replace as in BibEdit that leads to conflicts"""
marc_to_replace1 = """
%(recid)s
%(tag005)s
SzGeCERN
Test, Foo
Test Institute2
Test, John
Test University
Cool
Test, Jim
Test Laboratory
bla bla bla
""" % {'recid': self.recid, 'tag005': self.tag005}
recs = bibupload.xml_marc_to_records(marc_to_replace1)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(self.recid)
## The change should have been applied!
self.failUnless(records_identical(recs[0], get_record(self.recid)), "\n%s\n!=\n%s" % (recs[0], get_record(self.recid)))
marc_to_replace2 = """
%(recid)s
%(tag005)s
SzGeCERN
Queen Elisabeth
Great Britain
Test, John
Test University
No more Cool
Test, Jim
Test Laboratory
bla bla bla
""" % {'recid': self.recid, 'tag005': self.tag005}
recs = bibupload.xml_marc_to_records(marc_to_replace2)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(self.recid)
## The change should have been merged with the previous without conflict
self.failUnless(records_identical(bibupload.xml_marc_to_records(marc_to_replace1)[0], get_record(self.recid)), "%s != %s" % (bibupload.xml_marc_to_records(marc_to_replace1)[0], get_record(self.recid)))
self.failUnless(records_identical(bibupload.xml_marc_to_records(marc_to_replace2)[0], bibupload.xml_marc_to_records(zlib.decompress(run_sql("SELECT changeset_xml FROM bibHOLDINGPEN WHERE id_bibrec=%s", (self.recid,))[0][0]))[0]))
class BibUploadNoUselessHistoryTest(GenericBibUploadTest):
"""Testing generation of history only when necessary"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.test = """
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
recs = bibupload.xml_marc_to_records(self.test)
# We call the main function with the record as a parameter
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(self.recid)
# We retrieve the inserted xml
inserted_xm = print_record(self.recid, 'xm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.test), '')
self.history = run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, )) # kwalitee: disable=sql
self.timestamp = run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,))
def test_replace_identical_record(self):
"""bibupload - replace with identical record does not touch history"""
xml_to_upload = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
""" % self.recid
recs = bibupload.xml_marc_to_records(xml_to_upload)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(recid)
self.assertEqual(self.recid, recid)
self.assertEqual(self.history, run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) # kwalitee: disable=sql
self.assertEqual(self.timestamp, run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,)))
def test_correct_identical_correction(self):
"""bibupload - correct with identical correction does not touch history"""
xml_to_upload = """
%s
SzGeCERN
""" % self.recid
recs = bibupload.xml_marc_to_records(xml_to_upload)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
self.assertEqual(self.recid, recid)
self.maxDiff = None
self.assertEqual(self.history, run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) # kwalitee: disable=sql
self.assertEqual(self.timestamp, run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,)))
def test_replace_different_record(self):
"""bibupload - replace with different records does indeed touch history"""
xml_to_upload = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Test, Jim
Test Laboratory
""" % self.recid
recs = bibupload.xml_marc_to_records(xml_to_upload)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(recid)
self.assertEqual(self.recid, recid)
self.assertNotEqual(self.history, run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) # kwalitee: disable=sql
self.failUnless(len(self.history) == 1 and len(run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) == 2) # kwalitee: disable=sql
self.assertNotEqual(self.timestamp, run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,)))
def test_correct_different_correction(self):
"""bibupload - correct with different correction does indeed touch history"""
xml_to_upload = """
%s
FooBar
""" % self.recid
recs = bibupload.xml_marc_to_records(xml_to_upload)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
self.assertEqual(self.recid, recid)
self.assertNotEqual(self.history, run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) # kwalitee: disable=sql
self.failUnless(len(self.history) == 1 and len(run_sql("SELECT * FROM hstRECORD WHERE id_bibrec=%s", (self.recid, ))) == 2) # kwalitee: disable=sql
self.assertNotEqual(self.timestamp, run_sql("SELECT modification_date FROM bibrec WHERE id=%s", (self.recid,)))
class BibUploadCallbackURLTest(GenericBibUploadTest):
"""Testing usage of CLI callback_url"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.test = """
something
Tester, J Y
MIT
Tester, K J
CERN2
Tester, G
CERN3
test11
test31
test12
test32
test13
test33
test21
test41
test22
test42
test14
test51
test52
Tester, T
CERN
"""
self.testfile_path = os.path.join(CFG_TMPDIR, 'bibupload_regression_test_input.xml')
open(self.testfile_path, "w").write(self.test)
self.resultfile_path = os.path.join(CFG_TMPDIR, 'bibupload_regression_test_result.json')
if CFG_DEVEL_SITE:
def test_simple_insert_callback_url(self):
"""bibupload - --callback-url with simple insert"""
#
taskid = task_low_level_submission('bibupload', 'test', '-i', self.testfile_path, '--callback-url', CFG_SITE_URL + '/httptest/post2?%s' % urlencode({"save": self.resultfile_path}), '-v0')
run_shell_command(CFG_BINDIR + '/bibupload %s', [str(taskid)])
results = json.loads(open(self.resultfile_path).read())
self.failUnless('results' in results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""Tester, J Y """ in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
class BibUploadBibRelationsTest(GenericBibUploadTest):
def setUp(self):
GenericBibUploadTest.setUp(self)
self.upload_xml = """
A very wise author
%(url_site)s/img/user-icon-1-20x20.gif
Main
docname
TMP:id_identifier1
TMP:ver_identifier1
%(url_site)s/record/8/files/9812226.pdf?version=1
Main
docname2
TMP:id_identifier2
TMP:ver_identifier2
TMP:id_identifier1
TMP:ver_identifier1
TMP:id_identifier2
TMP:ver_identifier2
is_extracted_from
""" % {'url_site' : CFG_SITE_URL}
def test_upload_with_tmpids(self):
"""bibupload - Trying to upload a relation between two new documents ... and then to delete"""
recs = bibupload.xml_marc_to_records(self.upload_xml)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
# ertrive document numbers and check if there exists a relation between them
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of documents attached to a record")
rels = docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from")
self.assertEqual(1, len(rels), "Incorrect number of relations retrieved from the first document")
rels = docs[1].get_incoming_relations("is_extracted_from") + docs[1].get_outgoing_relations("is_extracted_from")
self.assertEqual(1, len(rels), "Incorrect number of relations retrieved from the second document")
created_relation_id = rels[0].id
rels = docs[0].get_incoming_relations("different_type_of_relation") + docs[0].get_outgoing_relations("different_type_of_relation")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
upload_xml_2 = """
%(rec_id)s
%(rel_id)s
DELETE
""" % {'rel_id' : created_relation_id, 'rec_id' : recid}
recs = bibupload.xml_marc_to_records(upload_xml_2)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of documents attached to a record")
rels = docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
rels = docs[1].get_incoming_relations("is_extracted_from") + docs[1].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the second document")
rels = docs[0].get_incoming_relations("different_type_of_relation") + docs[0].get_outgoing_relations("different_type_of_relation")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
def test_delete_by_docids(self):
"""bibupload - delete relation entry by the docid inside the currently modified record
Uploading a sample relation and trying to modify it by refering to other parameters than
the relation number"""
recs = bibupload.xml_marc_to_records(self.upload_xml)
dummyerr, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of attached documents")
rel = (docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from"))[0]
upload_xml_2 = """
%(rec_id)s
%(first_docid)s
%(first_docver)s
%(second_docid)s
%(second_docver)s
is_extracted_from
DELETE
""" % { 'rec_id' : recid,
'first_docid': rel.bibdoc1_id,
'first_docver' : rel.bibdoc1_ver,
'second_docid': rel.bibdoc2_id,
'second_docver' : rel.bibdoc2_ver}
recs = bibupload.xml_marc_to_records(upload_xml_2)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of documents attached to a record")
rels = docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
rels = docs[1].get_incoming_relations("is_extracted_from") + docs[1].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the second document")
rels = docs[0].get_incoming_relations("different_type_of_relation") + docs[0].get_outgoing_relations("different_type_of_relation")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
def test_remove_by_name(self):
"""bibupload - trying removing relation by providing bibdoc names rather than relation numbers"""
recs = bibupload.xml_marc_to_records(self.upload_xml)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of attached documents")
rel = (docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from"))[0]
upload_xml_2 = """
%(rec_id)s
docname
%(first_docver)s
docname2
%(second_docver)s
is_extracted_from
DELETE
""" % {'rec_id' : recid,
'first_docver' : rel.bibdoc1_ver,
'second_docver' : rel.bibdoc2_ver}
# the above is incorrect ! we assert that nothing has been removed
recs = bibupload.xml_marc_to_records(upload_xml_2)
_ = bibupload.bibupload_records(recs, opt_mode='correct')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of documents attached to a record")
rels = docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
rels = docs[1].get_incoming_relations("is_extracted_from") + docs[1].get_outgoing_relations("is_extracted_from")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the second document")
rels = docs[0].get_incoming_relations("different_type_of_relation") + docs[0].get_outgoing_relations("different_type_of_relation")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
def test_remove_by_name_incorrect(self):
"""bibupload - trying removing relation by providing bibdoc names rather than relation numbers, but providing incorrect name"""
recs = bibupload.xml_marc_to_records(self.upload_xml)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of attached documents")
rel = (docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from"))[0]
upload_xml_2 = """
%(rec_id)s
docname1
%(first_docver)s
docname2
%(second_docver)s
is_extracted_from
DELETE
""" % { 'rec_id' : recid,
'first_docver' : rel.bibdoc1_ver,
'second_docver' : rel.bibdoc2_ver}
# the above is incorrect ! we assert that nothing has been removed
recs = bibupload.xml_marc_to_records(upload_xml_2)
_ = bibupload.bibupload_records(recs, opt_mode='correct')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of documents attached to a record")
rels = docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from")
self.assertEqual(1, len(rels), "Incorrect number of relations retrieved from the first document")
rels = docs[1].get_incoming_relations("is_extracted_from") + docs[1].get_outgoing_relations("is_extracted_from")
self.assertEqual(1, len(rels), "Incorrect number of relations retrieved from the second document")
rels = docs[0].get_incoming_relations("different_type_of_relation") + docs[0].get_outgoing_relations("different_type_of_relation")
self.assertEqual(0, len(rels), "Incorrect number of relations retrieved from the first document")
def _upload_initial_moreinfo_key(self):
"""Prepare MoreInfo with sample keys and check it has been correctly uploaded
uploaded dic: {"ns1" : {"k1":"val1", "k2":[1,2,3,"something"], "k3" : (1,3,2,"something else"), "k4" : {"a":"b", 1:2}}}
... after encoding gives KGRwMQpTJ25zMScKcDIKKGRwMwpTJ2szJwpwNAooSTEKSTMKSTIKUydzb21ldGhpbmcgZWxzZScKdHA1CnNTJ2syJwpwNgoobHA3CkkxCmFJMgphSTMKYVMnc29tZXRoaW5nJwpwOAphc1MnazEnCnA5ClMndmFsMScKcDEwCnNTJ2s0JwpwMTEKKGRwMTIKUydhJwpTJ2InCnNJMQpJMgpzc3Mu
"""
moreinfo_str = "KGRwMQpTJ25zMScKcDIKKGRwMwpTJ2szJwpwNAooSTEKSTMKSTIKUydzb21ldGhpbmcgZWxzZScKdHA1CnNTJ2syJwpwNgoobHA3CkkxCmFJMgphSTMKYVMnc29tZXRoaW5nJwpwOAphc1MnazEnCnA5ClMndmFsMScKcDEwCnNTJ2s0JwpwMTEKKGRwMTIKUydhJwpTJ2InCnNJMQpJMgpzc3Mu"
xml_to_upload = """
A very wise author
%(url_site)s/img/user-icon-1-20x20.gif
Main
docname
TMP:id_identifier1
TMP:ver_identifier1
%(url_site)s/record/8/files/9812226.pdf?version=1
Main
docname2
TMP:id_identifier2
TMP:ver_identifier2
TMP:id_identifier1
TMP:ver_identifier1
TMP:id_identifier2
TMP:ver_identifier2
is_extracted_from
%(moreinfo_str)s
""" % {'url_site' : CFG_SITE_URL, 'moreinfo_str' : moreinfo_str}
recs = bibupload.xml_marc_to_records(xml_to_upload)
dummyerr, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
brd = BibRecDocs(recid)
docs = brd.list_bibdocs()
self.assertEqual(2, len(docs), "Incorrect number of attached documents")
return ((docs[0].get_incoming_relations("is_extracted_from") + docs[0].get_outgoing_relations("is_extracted_from"))[0], recid)
def test_add_relation_moreinfo_key(self):
"""bibupload - upload new MoreInfo key into the dictionary related to a relation"""
rel, _ = self._upload_initial_moreinfo_key()
# asserting correctness of data
self.assertEqual(rel.more_info.get_data("ns1", "k1"), "val1", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k1)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[0], 1, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[2], 3, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[3], "something", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k3"), (1,3,2,"something else") , "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k3)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")["a"], "b", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
def test_modify_relation_moreinfo_key(self):
"""bibupload - modify existing MoreInfo key """
#the update : {"ns1":{"k1": "different value"}}
rel, recid = self._upload_initial_moreinfo_key()
moreinfo_str = "KGRwMQpTJ25zMScKcDIKKGRwMwpTJ2sxJwpwNApTJ2RpZmZlcmVudCB2YWx1ZScKcDUKc3Mu"
upload_xml = """
%(rec_id)s
docname
docname2
1
1
is_extracted_from
%(moreinfo_str)s
""" % {"rec_id" : recid, "moreinfo_str": moreinfo_str}
recs = bibupload.xml_marc_to_records(upload_xml)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
rel = BibRelation(rel_id = rel.id)
self.assertEqual(rel.more_info.get_data("ns1", "k1"), "different value", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k1)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[0], 1, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[2], 3, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[3], "something", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k3"), (1,3,2,"something else") , "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k3)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")["a"], "b", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
self.assertEqual(rel.more_info.get_data("ns2", "k4"), None, "Retrieved not none value for nonexisting namespace !")
def test_remove_relation_moreinfo_key(self):
"""bibupload - remove existing MoreInfo key """
#the update : {"ns1":{"k3": None}}
rel, recid = self._upload_initial_moreinfo_key()
moreinfo_str = "KGRwMQpTJ25zMScKcDIKKGRwMwpTJ2szJwpwNApOc3Mu"
upload_xml = """
%(rec_id)s
docname
docname2
1
1
is_extracted_from
%(moreinfo_str)s
""" % {"rec_id" : recid, "moreinfo_str": moreinfo_str}
recs = bibupload.xml_marc_to_records(upload_xml)
bibupload.bibupload_records(recs, opt_mode='correct')
rel = BibRelation(rel_id = rel.id)
self.assertEqual(rel.more_info.get_data("ns1", "k1"), "val1", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k1)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[0], 1, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[2], 3, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k2")[3], "something", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k2)")
self.assertEqual(rel.more_info.get_data("ns1", "k3"), None , "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k3)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")[1], 2, "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
self.assertEqual(rel.more_info.get_data("ns1", "k4")["a"], "b", "Retrieved incorrect data from the MoreInfo Dictionary (namespace : ns1 key: k4)")
class BibUploadMoreInfoTest(GenericBibUploadTest):
"""bibupload - Testing upload of different types of MoreInfo """
def _dict_checker(self, dic, more_info, equal = True):
""" Check the more_info for being conform with the dictionary
@param equal - The mode of conformity. True means that the dictionary
has to be equal with the MoreInfo. False means that dictionary
has to be contained in the MoreInfo
"""
for namespace in dic:
for key in dic[namespace]:
self.assertEqual(cPickle.dumps(dic[namespace][key]),
cPickle.dumps(more_info.get_data(namespace, key)),
"Different values for the value of key %s in the namespace %s inside of the MoreInfo object" % \
(namespace, key))
if equal:
for namespace in more_info.get_namespaces():
for key in more_info.get_keys(namespace):
self.assertTrue(namespace in dic,
"namespace %s present in the MoreInfo, but not present in the dictionary" % \
(namespace, ))
self.assertTrue(key in dic[namespace],
"key %s present in the namespace %s of the MoreInfo but not present in the dictionary" % \
(namespace, key))
self.assertEqual(cPickle.dumps(more_info.get_data(namespace, key)),
cPickle.dumps(dic[namespace][key]),
"Value for namespace '%s' and key '%s' varies between MoreInfo and the dictionary. moreinfo value: '%s' dictionary value: '%s'" % \
(namespace, key, repr(more_info.get_data(namespace, key)), repr(dic[namespace][key])))
def test_relation_moreinfo_insert(self):
"""bibupload - Testing the upload of BibRelation and corresponding MoreInfo field"""
# Cleaning existing data
rels = BibRelation.get_relations(bibdoc1_id = 70, bibdoc2_id = 71, rel_type = "is_extracted_from")
for rel in rels:
rel.delete()
# Uploading
relation_upload_template = """
70
71
is_extracted_from
%s
Some author
"""
data_to_insert = {"first namespace": {"k1" : "val1", "k2" : "val2"},
"second" : {"k1" : "#@$#$@###!!!", "k123": {1:2, 9: (6,2,7)}}}
serialised = base64.b64encode(cPickle.dumps(data_to_insert))
recs = bibupload.xml_marc_to_records(relation_upload_template % (serialised, ))
bibupload.bibupload_records(recs, opt_mode='insert')[0]
# Verifying the correctness of the uploaded data
rels = BibRelation.get_relations(bibdoc1_id = 70, bibdoc2_id = 71, rel_type = "is_extracted_from")
self.assertEqual(len(rels), 1)
rel = rels[0]
self.assertEqual(rel.bibdoc1_id, 70)
self.assertEqual(rel.bibdoc2_id, 71)
self.assertEqual(rel.get_data("first namespace", "k1"), "val1")
self.assertEqual(rel.get_data("first namespace", "k2"), "val2")
self.assertEqual(rel.get_data("second", "k1"), "#@$#$@###!!!")
self.assertEqual(rel.get_data("second", "k123")[1], 2)
self.assertEqual(rel.get_data("second", "k123")[9], (6,2,7))
self._dict_checker(data_to_insert, rel.more_info)
# Cleaning after the upload ... just in case we have selected more
for rel in rels:
rel.delete()
def _serialise_data(self, data):
return base64.b64encode(cPickle.dumps(data))
# Subfield tags used to upload particular types of MoreInfo
_mi_bibdoc = "w"
_mi_bibdoc_version = "p"
_mi_bibdoc_version_format = "b"
_mi_bibdoc_format = "u"
def _generate_moreinfo_tag(self, mi_type, data):
"""
"""
serialised = self._serialise_data(data)
return """%s """ % (mi_type, serialised)
def test_document_moreinfo_insert(self):
"""bibupload - Inserting new MoreInfo to the document
1) Inserting new MoreInfo to new document
2) Inserting new MoreInfo keys existing document version
3) Removing keys from MoreInfo
4) Removing document and asserting, MoreInfo gets removed as well
5) Overriding MoreInfo keys
"""
moreinfo_upload_template = """
%(siteurl)s/img/site_logo.gif
0106015_01
.jpg
restricted_picture
%%(additional_content)s
Some author
""" % {"siteurl": CFG_SITE_URL}
sfs = []
sfs.append(self._generate_moreinfo_tag(BibUploadMoreInfoTest._mi_bibdoc,
{"first namespace" :
{"type": "document moreinfo"}}))
sfs.append(self._generate_moreinfo_tag(BibUploadMoreInfoTest._mi_bibdoc_version,
{"first namespace" :
{"type": "Bibdoc - version moreinfo"}}))
sfs.append(self._generate_moreinfo_tag(BibUploadMoreInfoTest._mi_bibdoc_version_format,
{"first namespace" :
{"type": "Bibdoc - version, format moreinfo"}}))
sfs.append(self._generate_moreinfo_tag(BibUploadMoreInfoTest._mi_bibdoc_format,
{"first namespace" :
{"type": "Bibdoc - format moreinfo"}}))
marcxml_1 = moreinfo_upload_template % {"additional_content" : "\n".join(sfs)}
recs = bibupload.xml_marc_to_records(marcxml_1)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
# now checking if all the data has been uploaded correctly
bdr = BibRecDocs(recid)
doc = bdr.list_bibdocs()[0]
docid = doc.get_id()
mi_doc = MoreInfo(docid = docid)
mi_doc_ver = MoreInfo(docid = docid, version = 1)
mi_doc_ver_fmt = MoreInfo(docid = docid, version = 1, docformat=".jpg")
mi_doc_fmt = MoreInfo(docid = docid, docformat=".jpg")
self._dict_checker({"first namespace" : {"type": "document moreinfo"}},
mi_doc, equal=False) # in case of the document only inclusive check
self._dict_checker({"first namespace" : {"type": "Bibdoc - version moreinfo"}},
mi_doc_ver)
self._dict_checker({"first namespace" : {
"type": "Bibdoc - version, format moreinfo"}},
mi_doc_ver_fmt)
self._dict_checker({"first namespace" : {"type": "Bibdoc - format moreinfo"}},
mi_doc_fmt)
#now appending to a particular version of MoreInfo
# uplad new key to an existing dictionary of a version
def _get_mit_template(recid, bibdocid=None, bibdocname=None,
version=None, docformat=None, relation=None, data=None):
if data is None:
ser = None
else:
ser = base64.b64encode(cPickle.dumps(data))
subfields = []
for s_code, val in (("r", relation), ("i", bibdocid),
("n", bibdocname), ("v", version),
("f", docformat) , ("m", ser)):
if not val is None:
subfields.append("""%s """ % \
(s_code, val))
return """
%s
%s
""" % (str(recid), ("\n".join(subfields)))
marcxml_2 = _get_mit_template(recid, version = 1, bibdocid = docid,
data= {"first namespace" :
{"new key": {1:2, 987:678}}})
recs = bibupload.xml_marc_to_records(marcxml_2)
bibupload.bibupload_records(recs, opt_mode='append')
mi = MoreInfo(docid = docid, version = 1)
self._dict_checker({
"first namespace" : {"type": "Bibdoc - version moreinfo",
"new key": {1:2, 987:678}
}
}, mi)
#removing the entire old content of the MoreInfo and uploading new
data = {"ns1" : {"nk1": 12, "mk1": "this is new content"},
"namespace two" : {"ddd" : "bbb"}}
marcxml_3 = _get_mit_template(recid, version = 1, bibdocid = docid,
data= data)
recs = bibupload.xml_marc_to_records(marcxml_3)
bibupload.bibupload_records(recs, opt_mode='correct')
mi = MoreInfo(docid = docid, version = 1)
self._dict_checker(data, mi)
# removing a particular key
marcxml_4 = _get_mit_template(recid, version = 1, bibdocid = docid,
data= {"ns1": {"nk1" : None}})
recs = bibupload.xml_marc_to_records(marcxml_4)
bibupload.bibupload_records(recs, opt_mode='append')
mi = MoreInfo(docid = docid, version = 1)
self._dict_checker( {"ns1" : { "mk1": "this is new content"},
"namespace two" : {"ddd" : "bbb"}}, mi)
# adding new key
marcxml_5 = _get_mit_template(recid, version = 1, bibdocid = docid,
data= {"ns1": {"newkey" : "newvalue"}})
recs = bibupload.xml_marc_to_records(marcxml_5)
bibupload.bibupload_records(recs, opt_mode='append')
mi = MoreInfo(docid = docid, version = 1)
self._dict_checker( {"ns1" : { "mk1": "this is new content", "newkey" : "newvalue"},
"namespace two" : {"ddd" : "bbb"}}, mi)
class BibUploadInsertModeTest(GenericBibUploadTest):
"""Testing insert mode."""
def setUp(self):
# pylint: disable=C0103
"""Initialise the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test = """
something
Tester, J Y
MIT
Tester, K J
CERN2
Tester, G
CERN3
test11
test31
test12
test32
test13
test33
test21
test41
test22
test42
test14
test51
test52
Tester, T
CERN
"""
self.test_hm = """
100__ $$aTester, T$$uCERN
111__ $$atest11$$ctest31
111__ $$atest12$$ctest32
111__ $$atest13$$ctest33
111__ $$btest21$$dtest41
111__ $$btest22$$dtest42
111__ $$atest14
111__ $$etest51
111__ $$etest52
245__ $$asomething
700__ $$aTester, J Y$$uMIT
700__ $$aTester, K J$$uCERN2
700__ $$aTester, G$$uCERN3
"""
def test_create_record_id(self):
"""bibupload - insert mode, trying to create a new record ID in the database"""
rec_id = bibupload.create_new_record()
self.assertNotEqual(None, rec_id)
def test_create_specific_record_id(self):
"""bibupload - insert mode, trying to create a new specifc record ID in the database"""
expected_rec_id = run_sql("SELECT MAX(id) FROM bibrec")[0][0] + 1
rec_id = bibupload.create_new_record(expected_rec_id)
self.assertEqual(rec_id, expected_rec_id)
def test_no_retrieve_record_id(self):
"""bibupload - insert mode, detection of record ID in the input file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'insert')
# We compare the value found with None
self.assertEqual(None, rec_id)
def test_insert_complete_xmlmarc(self):
"""bibupload - insert mode, trying to insert complete MARCXML file"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# We retrieve the inserted xml
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
inserted_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.test), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.test_hm), '')
def test_retrieve_005_tag(self):
"""bibupload - insert mode, verifying insertion of 005 control field for record """
# Convert marc xml into record structure
recs = bibupload.xml_marc_to_records(self.test)
dummy, recid, dummy = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
# Retrive the inserted record based on the record id
rec = get_record(recid)
# We retrieve the creationdate date from the database
query = """SELECT DATE_FORMAT(last_updated,'%%Y%%m%%d%%H%%i%%s') FROM bibfmt where id_bibrec=%s AND format='xm'"""
res = run_sql(query, (recid, ))
self.assertEqual(record_has_field(rec, '005'), True)
self.assertEqual(str(res[0][0]) + '.0', record_get_field_value(rec, '005', '', ''))
class BibUploadAppendModeTest(GenericBibUploadTest):
"""Testing append mode."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_existing = """
123456789
Tester, T
DESY
0003719PHOPHO
"""
self.test_to_append = """
123456789
Tester, U
CERN
0003719PHOPHO
"""
self.test_expected_xm = """
123456789
Tester, T
DESY
Tester, U
CERN
0003719PHOPHO
"""
self.test_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uDESY
100__ $$aTester, U$$uCERN
970__ $$a0003719PHOPHO
"""
# insert test record:
test_to_upload = self.test_existing.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_to_upload)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
self.test_recid = recid
# replace test buffers with real recid of inserted test record:
self.test_existing = self.test_existing.replace('123456789',
str(self.test_recid))
self.test_to_append = self.test_to_append.replace('123456789',
str(self.test_recid))
self.test_expected_xm = self.test_expected_xm.replace('123456789',
str(self.test_recid))
self.test_expected_hm = self.test_expected_hm.replace('123456789',
str(self.test_recid))
def test_retrieve_record_id(self):
"""bibupload - append mode, the input file should contain a record ID"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'append')
# We compare the value found with None
self.assertEqual(self.test_recid, rec_id)
# clean up after ourselves:
def test_update_modification_record_date(self):
"""bibupload - append mode, checking the update of the modification date"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_existing)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], opt_mode='append')
# Retrieve current localtime
record_modification_date = time.localtime()
# We update the modification date
bibupload.update_bibrec_date(convert_datestruct_to_datetext(record_modification_date), rec_id, False)
# We retrieve the modification date from the database
query = """SELECT DATE_FORMAT(modification_date,'%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec where id = %s"""
res = run_sql(query, (str(rec_id), ))
# We compare the two results
self.assertEqual(res[0][0], convert_datestruct_to_datetext(record_modification_date))
# clean up after ourselves:
def test_append_complete_xml_marc(self):
"""bibupload - append mode, appending complete MARCXML file"""
# Now we append a datafield
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the main function with the record as a parameter
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='append')[0]
self.check_record_consistency(recid)
# We retrieve the inserted xm
after_append_xm = print_record(recid, 'xm')
after_append_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(after_append_xm, self.test_expected_xm), '')
self.assertEqual(compare_hmbuffers(after_append_hm, self.test_expected_hm), '')
def test_retrieve_updated_005_tag(self):
"""bibupload - append mode, updating 005 control tag after modifiction """
recs = bibupload.xml_marc_to_records(self.test_to_append)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='append')
self.check_record_consistency(recid)
rec = get_record(recid)
query = """SELECT DATE_FORMAT(MAX(job_date),'%%Y%%m%%d%%H%%i%%s') FROM hstRECORD where id_bibrec = %s"""
res = run_sql(query, (str(recid), ))
self.assertEqual(str(res[0][0])+'.0',record_get_field_value(rec,'005','',''))
class BibUploadCorrectModeTest(GenericBibUploadTest):
"""
Testing correcting a record containing similar tags (identical
tag, different indicators). Currently Invenio replaces only
those tags that have matching indicators too, unlike ALEPH500 that
does not pay attention to indicators, it corrects all fields with
the same tag, regardless of the indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
self.testrec1_xm_to_correct = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Cool
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10048 $$aCool
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_record_correction(self):
"""bibupload - correct mode, similar MARCXML tags/indicators"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
_, self.recid, _ = bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(self.recid)
corrected_xm = print_record(self.recid, 'xm')
corrected_hm = print_record(self.recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# clean up after ourselves:
return
class BibUploadDeleteModeTest(GenericBibUploadTest):
"""
Testing deleting specific tags from a record while keeping anything else
untouched. Currently Invenio deletes only those tags that have
matching indicators too, unlike ALEPH500 that does not pay attention to
indicators, it corrects all fields with the same tag, regardless of the
indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
dumb text
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
888__ $$adumb text
"""
self.testrec1_xm_to_delete = """
123456789
Test, Jane
Test Institute
Test, Johnson
Test University
Cool
dumb text
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, John
Test University
Test, Jim
Test Laboratory
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
10047 $$aTest, John$$uTest University
10047 $$aTest, Jim$$uTest Laboratory
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_delete = self.testrec1_xm_to_delete.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
# Checking dumb text is in bibxxx
self.failUnless(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
def test_record_tags_deletion(self):
"""bibupload - delete mode, deleting specific tags"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_delete)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='delete')[0]
self.check_record_consistency(recid)
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# Checking dumb text is no more in bibxxx
self.failIf(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
# clean up after ourselves:
class BibUploadReplaceModeTest(GenericBibUploadTest):
"""Testing replace mode."""
def test_record_replace(self):
"""bibupload - replace mode, similar MARCXML tags/indicators"""
# replace some tags:
testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
testrec1_xm_to_replace = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
testrec1_replaced_xm = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
testrec1_replaced_hm = """
001__ 123456789
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
testrec1_xm = testrec1_xm.replace('123456789', str(recid))
testrec1_hm = testrec1_hm.replace('123456789', str(recid))
testrec1_xm_to_replace = testrec1_xm_to_replace.replace('123456789', str(recid))
testrec1_replaced_xm = testrec1_replaced_xm.replace('123456789', str(recid))
testrec1_replaced_hm = testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, testrec1_hm), '')
recs = bibupload.xml_marc_to_records(testrec1_xm_to_replace)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(recid)
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, testrec1_replaced_hm), '')
def test_record_replace_force_non_existing(self):
"""bibupload - replace mode, force non existing recid"""
# replace some tags:
the_recid = self.last_recid + 1
testrec1_xm = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
""" % the_recid
testrec1_hm = """
001__ %s
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
""" % the_recid
recs = bibupload.xml_marc_to_records(testrec1_xm)
task_set_option('force', True)
try:
err, recid, msg = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(recid)
finally:
task_set_option('force', False)
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, testrec1_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, testrec1_hm), '')
self.assertEqual(recid, the_recid)
def test_record_replace_non_existing(self):
"""bibupload - replace mode, non existing recid"""
# replace some tags:
the_recid = self.last_recid + 1
testrec1_xm = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
""" % the_recid
recs = bibupload.xml_marc_to_records(testrec1_xm)
err, recid, _ = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual((err, recid), (1, -1))
def test_record_replace_two_recids(self):
"""bibupload - replace mode, two recids"""
# replace some tags:
testrec1_xm = """
300
305
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
recs = bibupload.xml_marc_to_records(testrec1_xm)
err, recid, _ = bibupload.bibupload(recs[0], opt_mode='replace')
# did it work?
self.assertEqual((err, recid), (1, -1))
class BibUploadReferencesModeTest(GenericBibUploadTest):
"""Testing references mode.
NOTE: in the past this was done by calling bibupload --reference|-z
which is now simply implying bibupload --correct.
"""
def setUp(self):
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_insert = """
123456789
Tester, T
CERN
"""
self.test_reference = """
123456789
M. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,
J. High Energy Phys. 07 (2004) 014
"""
self.test_reference_expected_xm = """
123456789
Tester, T
CERN
M. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,
J. High Energy Phys. 07 (2004) 014
"""
self.test_insert_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
"""
self.test_reference_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
%(reference_tag)sC5 $$mM. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,$$sJ. High Energy Phys. 07 (2004) 014
""" % {'reference_tag': CFG_BIBUPLOAD_REFERENCE_TAG}
# insert test record:
test_insert = self.test_insert.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_insert)
_, recid, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
self.test_insert = self.test_insert.replace('123456789', str(recid))
self.test_insert_hm = self.test_insert_hm.replace('123456789', str(recid))
self.test_reference = self.test_reference.replace('123456789', str(recid))
self.test_reference_expected_xm = self.test_reference_expected_xm.replace('123456789', str(recid))
self.test_reference_expected_hm = self.test_reference_expected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.test_insert), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.test_insert_hm), '')
self.test_recid = recid
def test_reference_complete_xml_marc(self):
"""bibupload - reference mode, inserting references MARCXML file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_reference)
# We call the main function with the record as a parameter
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='reference')[0]
self.check_record_consistency(recid)
# We retrieve the inserted xml
reference_xm = print_record(recid, 'xm')
reference_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(reference_xm, self.test_reference_expected_xm), '')
self.assertEqual(compare_hmbuffers(reference_hm, self.test_reference_expected_hm), '')
class BibUploadRecordsWithSYSNOTest(GenericBibUploadTest):
"""Testing uploading of records that have external SYSNO present."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that SYSNO fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
Bar, Baz
Foo
On the quux and huux 2
sysno2
sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
def test_insert_the_same_sysno_record(self):
"""bibupload - SYSNO tag, refuse to insert the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr2, recid2, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid2)
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_sysno_record() finished"
def test_insert_or_replace_the_same_sysno_record(self):
"""bibupload - SYSNO tag, allow to insert or replace the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs,
opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1_updated)
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() finished"
def test_replace_nonexisting_sysno_record(self):
"""bibupload - SYSNO tag, refuse to replace non-existing SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummy, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummy, recid2, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_sysno_record() finished"
class BibUploadRecordsWithEXTOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have external EXTOAIID present."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that EXTOAIID fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
extoaiid2
extoaisrc1
extoaiid1
Bar, Baz
Foo
On the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid2
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid1
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
def test_insert_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to insert the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr2, recid2, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid2)
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_extoaiid_record() finished"
def test_insert_or_replace_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, allow to insert or replace the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1_updated)
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() finished"
def test_replace_nonexisting_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to replace non-existing EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr2, recid2, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() finished"
class BibUploadRecordsWithOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have OAI ID present."""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that OAI fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
Bar, Baz
Foo
On the quux and huux 2
oai:foo:2
oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
def test_insert_the_same_oai_record(self):
"""bibupload - OAIID tag, refuse to insert the same OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr2, recid2, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid2)
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.assertEqual(-1, recid1_updated)
def test_insert_or_replace_the_same_oai_record(self):
"""bibupload - OAIID tag, allow to insert or replace the same OAI record"""
# initialize bibupload mode:
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
dummyerr1_updated, recid1_updated, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1_updated)
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
def test_replace_nonexisting_oai_record(self):
"""bibupload - OAIID tag, refuse to replace non-existing OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
dummyerr2, recid2, _ = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.assertEqual(-1, recid2)
class BibUploadRecordsWithDOITest(GenericBibUploadTest):
"""Testing uploading of records with DOI."""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
self.xm_testrec1 = """
123456789
SzGeCERN
doi
10.5170/123-456-789
nondoi
10.5170/123-456-789-0
Bar, Baz
Foo
On the quux and huux 1
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)sdoi$$%(doisubfieldcodevalue)s10.5170/123-456-789
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)snondoi$$%(doisubfieldcodevalue)s10.5170/123-456-789-0
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': '_',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec1_to_update = """
SzGeCERN
doi
10.5170/123-456-789
nondoi
10.5170/123-456-789-0
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
doi
10.5170/123-456-789
nondoi
10.5170/123-456-789-0
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)sdoi$$%(doisubfieldcodevalue)s10.5170/123-456-789
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)snondoi$$%(doisubfieldcodevalue)s10.5170/123-456-789-0
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': '_',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec2 = """
987654321
SzGeCERN
doi
10.5170/987-654-321
Bar, Baz
Foo
On the quux and huux 2
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)sdoi$$%(doisubfieldcodevalue)s10.5170/987-654-321
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': '_',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec2_to_update = """
987654321
SzGeCERN
doi
10.5170/123-456-789
Bar, Baz
Foo
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec3 = """
192837645
SzGeCERN
doi
10.5170/123-456-789-0
Bar, Baz
Foo
On the quux and huux 4
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.hm_testrec3 = """
001__ 192837645
003__ SzGeCERN
%(doitag)s%(doiind1)s%(doiind2)s $$%(doisubfieldcodesource)sdoi$$%(doisubfieldcodevalue)s10.5170/123-456-789-0
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 4
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': '_',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec4 = """
SzGeCERN
doi
10.5170/123-456-789-non-existing
Bar, Baz
Foo
On the quux and huux 5
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
self.xm_testrec5 = """
123456789
SzGeCERN
doi
10.5170/123-456-789
doi
10.5170/987-654-321
Bar, Baz
Foo
On the quux and huux 6
""" % {'doitag': '024',
'doiind1': '7',
'doiind2': ' ',
'doisubfieldcodevalue': 'a',
'doisubfieldcodesource': '2'
}
def test_insert_the_same_doi_matching_on_doi(self):
"""bibupload - DOI tag, refuse to "insert" twice same DOI (matching on DOI)"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid2)
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert again record 1 (without recid, matching on DOI)
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
# if we try to update, append or correct, the same record is matched
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid1_updated)
self.assertEqual(recid1, recid1_updated)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='append')
self.check_record_consistency(recid1_updated)
self.assertEqual(recid1, recid1_updated)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(recid1_updated)
self.assertEqual(recid1, recid1_updated)
def test_insert_the_same_doi_matching_on_recid(self):
"""bibupload - DOI tag, refuse to "insert" twice same DOI (matching on recid)"""
# First upload 2 test records
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid1)
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid2)
# try to update record 2 with DOI already in record 1. It must fail:
testrec_to_update = self.xm_testrec2_to_update.replace('987654321 ',
'%s ' % recid2)
recs = bibupload.xml_marc_to_records(testrec_to_update)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(recid)
self.assertEqual(1, err)
# Ditto in correct and append mode
recs = bibupload.xml_marc_to_records(testrec_to_update)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid)
self.assertEqual(1, err)
recs = bibupload.xml_marc_to_records(testrec_to_update)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
self.check_record_consistency(recid)
self.assertEqual(1, err)
def test_insert_or_replace_the_same_doi_record(self):
"""bibupload - DOI tag, allow to insert or replace matching on DOI"""
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
self.check_record_consistency(recid1_updated)
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
def test_correct_the_same_doi_record(self):
"""bibupload - DOI tag, allow to correct matching on DOI"""
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to correct updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid1_updated)
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
def test_replace_nonexisting_doi_record(self):
"""bibupload - DOI tag, refuse to replace non-existing DOI record (matching on DOI)"""
testrec_to_insert_first = self.xm_testrec4
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err4, recid4, msg4 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid4)
def test_matching_on_doi_source_field(self):
"""bibupload - DOI tag, test matching records using DOI value AND source field ($2)"""
# insert test record 1, with a "fake" doi (not "doi" in source field):
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid1)
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 3, which matches record 1 "fake" doi, so it
# should work.
testrec_to_insert_first = self.xm_testrec3.replace('192837645 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err3, recid3, msg3 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid3)
inserted_xm = print_record(recid3, 'xm')
inserted_hm = print_record(recid3, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec3 = self.xm_testrec3.replace('192837645', str(recid3))
self.hm_testrec3 = self.hm_testrec3.replace('192837645', str(recid3))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec3), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec3), '')
def test_replace_or_update_record__with_ambiguous_doi(self):
"""bibupload - DOI tag, refuse to replace/correct/append on the basis of ambiguous DOI"""
# First upload 2 test records with two different DOIs:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid1)
self.assertEqual(0, err1)
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid2)
self.assertEqual(0, err2)
# Now try to insert record with DOIs matching the records
# previously uploaded. It must fail.
testrec = self.xm_testrec5.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(1, err5)
# Ditto for other modes:
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='correct')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='append')
self.assertEqual(1, err5)
# The same is true if a recid exists in the input MARCXML (as
# long as DOIs are ambiguous):
testrec = self.xm_testrec5.replace('123456789 ',
'%s ' % recid1)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='correct')
self.assertEqual(1, err5)
recs = bibupload.xml_marc_to_records(testrec)
err5, recid5, msg5 = bibupload.bibupload(recs[0], opt_mode='append')
self.assertEqual(1, err5)
class BibUploadIndicatorsTest(GenericBibUploadTest):
"""
Testing uploading of a MARCXML record with indicators having
either blank space (as per MARC schema) or empty string value (old
behaviour).
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
def test_record_with_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec1_hm), '')
def test_record_with_no_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with no spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec2_hm), '')
class BibUploadUpperLowerCaseTest(GenericBibUploadTest):
"""
Testing treatment of similar records with only upper and lower
case value differences in the bibxxx table.
"""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERN
TeSt, JoHn
Test UniVeRsity
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTeSt, JoHn$$uTest UniVeRsity
"""
def test_record_with_upper_lower_case_letters(self):
"""bibupload - inserting similar MARCXML records with upper/lower case"""
# insert test record #1:
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
dummyerr1, recid1, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid1)
recid1_inserted_xm = print_record(recid1, 'xm')
recid1_inserted_hm = print_record(recid1, 'hm')
# insert test record #2:
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
dummyerr1, recid2, _ = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid2)
recid2_inserted_xm = print_record(recid2, 'xm')
recid2_inserted_hm = print_record(recid2, 'hm')
# let us compare stuff now:
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid1_inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid1_inserted_hm),
self.testrec1_hm), '')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid2_inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid2_inserted_hm),
self.testrec2_hm), '')
class BibUploadControlledProvenanceTest(GenericBibUploadTest):
"""Testing treatment of tags under controlled provenance in the correct mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
blabla
sam
blublu
sim
human
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$9sim$$ablublu
6531_ $$ahuman
"""
self.testrec1_xm_to_correct = """
123456789
bleble
sim
bloblo
som
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
blabla
sam
human
bleble
sim
bloblo
som
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$ahuman
6531_ $$9sim$$ableble
6531_ $$9som$$abloblo
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_controlled_provenance_persistence(self):
"""bibupload - correct mode, tags with controlled provenance"""
# correct metadata tags; will the protected tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
class BibUploadStrongTagsTest(GenericBibUploadTest):
"""Testing treatment of strong tags and the replace mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
A value
Another value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_xm_to_replace = """
123456789
Test, Joseph
Test Academy
"""
self.testrec1_replaced_xm = """
123456789
Test, Joseph
Test Academy
A value
Another value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_replaced_hm = """
001__ 123456789
100__ $$aTest, Joseph$$uTest Academy
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_replace = self.testrec1_xm_to_replace.replace('123456789', str(recid))
self.testrec1_replaced_xm = self.testrec1_replaced_xm.replace('123456789', str(recid))
self.testrec1_replaced_hm = self.testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_strong_tags_persistence(self):
"""bibupload - strong tags, persistence in replace mode"""
# replace all metadata tags; will the strong tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_replace)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(recid)
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, self.testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, self.testrec1_replaced_hm), '')
class BibUploadPretendTest(GenericBibUploadTest):
"""
Testing bibupload --pretend correctness.
"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.demo_data = bibupload.xml_marc_to_records(open(os.path.join(CFG_TMPDIR, 'demobibdata.xml')).read())[0]
self.before = self._get_tables_fingerprint()
task_set_task_param('pretend', True)
def tearDown(self):
GenericBibUploadTest.tearDown(self)
task_set_task_param('pretend', False)
@staticmethod
def _get_tables_fingerprint():
"""
Take lenght and last modification time of all the tables that
might be touched by bibupload and return them in a nice structure.
"""
fingerprint = {}
tables = ['bibrec', 'bibdoc', 'bibrec_bibdoc', 'bibdoc_bibdoc', 'bibfmt', 'hstDOCUMENT', 'hstRECORD', 'bibHOLDINGPEN', 'bibdocmoreinfo', 'bibdocfsinfo']
for i in xrange(100):
tables.append('bib%02dx' % i)
tables.append('bibrec_bib%02dx' % i)
for table in tables:
fingerprint[table] = get_table_status_info(table)
return fingerprint
@staticmethod
def _checks_tables_fingerprints(before, after):
"""
Checks differences in table_fingerprints.
"""
for table in before.keys():
if before[table] != after[table]:
raise StandardError("Table %s has been modified: before was [%s], after was [%s]" % (table, pprint.pformat(before[table]), pprint.pformat(after[table])))
return True
def test_pretend_insert(self):
"""bibupload - pretend insert"""
bibupload.bibupload_records([self.demo_data], opt_mode='insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_correct(self):
"""bibupload - pretend correct"""
bibupload.bibupload_records([self.demo_data], opt_mode='correct', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace(self):
"""bibupload - pretend replace"""
bibupload.bibupload_records([self.demo_data], opt_mode='replace', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_append(self):
"""bibupload - pretend append"""
bibupload.bibupload_records([self.demo_data], opt_mode='append', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace_or_insert(self):
"""bibupload - pretend replace or insert"""
bibupload.bibupload_records([self.demo_data], opt_mode='replace_or_insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_holdingpen(self):
"""bibupload - pretend holdingpen"""
bibupload.bibupload_records([self.demo_data], opt_mode='holdingpen', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_delete(self):
"""bibupload - pretend delete"""
bibupload.bibupload_records([self.demo_data], opt_mode='delete', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_reference(self):
"""bibupload - pretend reference"""
bibupload.bibupload_records([self.demo_data], opt_mode='reference', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
class BibUploadHoldingPenTest(GenericBibUploadTest):
"""
Testing the Holding Pen usage.
"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.verbose = 9
setup_loggers()
task_set_task_param('verbose', self.verbose)
self.recid = 10
self.oai_id = "oai:cds.cern.ch:CERN-EP-2001-094"
def test_holding_pen_upload_with_recid(self):
"""bibupload - holding pen upload with recid"""
test_to_upload = """
%s
Kleefeld, F
Newcomer, Y
Rupp, G
Scadron, M D
""" % self.recid
recs = bibupload.xml_marc_to_records(test_to_upload)
bibupload.insert_record_into_holding_pen(recs[0], "")
res = run_sql("SELECT changeset_xml FROM bibHOLDINGPEN WHERE id_bibrec=%s", (self.recid, ))
self.failUnless("Rupp, G" in zlib.decompress(res[0][0]))
def test_holding_pen_upload_with_oai_id(self):
"""bibupload - holding pen upload with oai_id"""
test_to_upload = """
Kleefeld, F
Newcomer, Y
Rupp, G
Scadron, M D
%(value)s
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'value': self.oai_id
}
recs = bibupload.xml_marc_to_records(test_to_upload)
bibupload.insert_record_into_holding_pen(recs[0], self.oai_id)
res = run_sql("SELECT changeset_xml FROM bibHOLDINGPEN WHERE id_bibrec=%s AND oai_id=%s", (self.recid, self.oai_id))
self.failUnless("Rupp, G" in zlib.decompress(res[0][0]))
def tearDown(self):
GenericBibUploadTest.tearDown(self)
run_sql("DELETE FROM bibHOLDINGPEN WHERE id_bibrec=%s", (self.recid, ))
class BibUploadFFTModeTest(GenericBibUploadTest):
"""
Testing treatment of fulltext file transfer import mode.
"""
def _test_bibdoc_status(self, recid, docname, status):
res = run_sql('SELECT bd.status FROM bibrec_bibdoc as bb JOIN bibdoc as bd ON bb.id_bibdoc = bd.id WHERE bb.id_bibrec = %s AND bb.docname = %s', (recid, docname))
self.failUnless(res)
self.assertEqual(status, res[0][0])
def test_writing_rights(self):
"""bibupload - FFT has writing rights"""
self.failUnless(bibupload.writing_rights_p())
def test_simple_fft_insert(self):
"""bibupload - simple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.failUnless(try_url_download(testrec_expected_url))
def test_fft_insert_with_valid_embargo(self):
"""bibupload - FFT insert with valid embargo"""
# define the test case:
future_date = time.strftime('%Y-%m-%d', time.gmtime(time.time() + 24 * 3600 * 2))
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
firerole: deny until '%(future_date)s'
allow any
""" % {
'future_date': future_date,
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
result = urlopen(testrec_expected_url).read()
self.failUnless("This file is restricted." in result, result)
def test_fft_insert_with_expired_embargo(self):
"""bibupload - FFT insert with expired embargo"""
# define the test case:
past_date = time.strftime('%Y-%m-%d', time.gmtime(time.time() - 24 * 3600 * 2))
test_to_upload = """
SzGeCERN
Test, John
Test University
ARTICLE
%(siteurl)s/img/site_logo.gif
firerole: deny until '%(past_date)s'
allow any
""" % {
'past_date': past_date,
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
ARTICLE
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
result = urlopen(testrec_expected_url).read()
self.failIf("If you already have an account, please login using the form below." in result, result)
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
self.force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text=urlopen("%(siteurl)s/img/site_logo.gif" % {
'siteurl': CFG_SITE_URL
}).read()), [])
def test_exotic_format_fft_append(self):
"""bibupload - exotic format FFT append"""
# define the test case:
testfile = os.path.join(CFG_TMPDIR, 'test.ps.Z')
open(testfile, 'w').write('TEST')
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
""" % {
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_to_append = """
123456789
%s
""" % testfile
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
4
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test?format=ps.Z" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_to_append = testrec_to_append.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url.replace('123456789',
str(recid))
recs = bibupload.xml_marc_to_records(testrec_to_append)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='append')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.assertEqual(test_web_page_content(testrec_expected_url, 'jekyll', 'j123ekyll', expected_text='TEST'), [])
self.assertEqual(test_web_page_content(testrec_expected_url2, 'jekyll', 'j123ekyll', expected_text='TEST'), [])
def test_fft_check_md5_through_bibrecdoc_str(self):
"""bibupload - simple FFT insert, check md5 through BibRecDocs.str()"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%s/img/head.gif
""" % CFG_SITE_URL
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
original_md5 = md5(urlopen('%s/img/head.gif' % CFG_SITE_URL).read()).hexdigest()
bibrec_str = str(BibRecDocs(int(recid)))
md5_found = False
for row in bibrec_str.split('\n'):
if 'checksum' in row:
if original_md5 in row:
md5_found = True
self.failUnless(md5_found)
def test_detailed_fft_insert(self):
"""bibupload - detailed FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
SuperMain
This is a description
This is a comment
CIDIESSE
%(siteurl)s/img/rss.png
SuperMain
.jpeg
This is a description
This is a second comment
CIDIESSE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif
This is a description
This is a comment
530
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg
This is a description
This is a second comment
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url1 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url1 = testrec_expected_url1.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url1.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.failUnless(try_url_download(testrec_expected_url1))
self.failUnless(try_url_download(testrec_expected_url2))
def test_simple_fft_insert_with_restriction(self):
"""bibupload - simple FFT insert with restriction"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
ARTICLE
%(siteurl)s/img/site_logo.gif
thesis
%(siteurl)s/img/sb.gif
""" % {'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code,
'siteurl': CFG_SITE_URL}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
ARTICLE
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_icon = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.assertEqual(test_web_page_content(testrec_expected_icon, 'jekyll', 'j123ekyll', expected_text=urlopen('%(siteurl)s/img/sb.gif' % {
'siteurl': CFG_SITE_URL
}).read()), [])
self.assertEqual(test_web_page_content(testrec_expected_icon, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
self.force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_icon, 'hyde', 'h123yde', expected_text=urlopen('%(siteurl)s/img/restricted.gif' % {'siteurl': CFG_SITE_URL}).read()), [])
self.failUnless("HTTP Error 401: Unauthorized" in test_web_page_content(testrec_expected_url, 'hyde', 'h123yde')[0])
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
def test_simple_fft_insert_with_icon(self):
"""bibupload - simple FFT insert with icon"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/sb.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_icon = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_icon))
def test_multiple_fft_insert(self):
"""bibupload - multiple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/head.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/95/files/9809057.pdf
%(prefix)s/var/tmp/demobibdata.xml
""" % {
'prefix': CFG_PREFIX,
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
295078
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/9809057.pdf
%(sizeofdemobibdata)s
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/demobibdata.xml
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'sizeofdemobibdata': os.path.getsize(os.path.join(CFG_TMPDIR, "demobibdata.xml"))}
# insert test record:
testrec_expected_urls = []
for files in ('site_logo.gif', 'head.gif', '9809057.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'files' : files})
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_urls = []
for files in ('site_logo.gif', 'head.gif', '9809057.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/%(CFG_SITE_RECORD)s/%(recid)s/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'files' : files, 'recid' : recid})
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
for url in testrec_expected_urls:
self.failUnless(try_url_download(url))
self._test_bibdoc_status(recid, 'head', '')
self._test_bibdoc_status(recid, '9809057', '')
self._test_bibdoc_status(recid, 'site_logo', '')
self._test_bibdoc_status(recid, 'demobibdata', '')
def test_simple_fft_correct(self):
"""bibupload - simple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/sb.gif
site_logo
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
79
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'site_logo', '')
def test_fft_correct_already_exists(self):
"""bibupload - FFT correct with already identical existing file"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
a description
%(siteurl)s/img/help.png
site_logo
another description
%(siteurl)s/img/rss.png
%(siteurl)s/img/line.gif
%(siteurl)s/img/merge.png
line
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/site_logo.gif
a second description
%(siteurl)s/img/help.png
site_logo
another second description
%(siteurl)s/img/refresh.png
rss
%(siteurl)s/img/line.gif
%(siteurl)s/img/merge-small.png
line
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
35
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif
626
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png
432
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a second description
786
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url3 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url4 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url5 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url2.replace('123456789',
str(recid))
testrec_expected_url3 = testrec_expected_url3.replace('123456789',
str(recid))
testrec_expected_url4 = testrec_expected_url4.replace('123456789',
str(recid))
testrec_expected_url5 = testrec_expected_url5.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_url2))
self.failUnless(try_url_download(testrec_expected_url3))
self.failUnless(try_url_download(testrec_expected_url4))
self.failUnless(try_url_download(testrec_expected_url5))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
bibrecdocs = BibRecDocs(recid)
self.failUnless(bibrecdocs.get_bibdoc('rss').list_versions(), [1, 2])
self.failUnless(bibrecdocs.get_bibdoc('site_logo').list_versions(), [1])
self.failUnless(bibrecdocs.get_bibdoc('line').list_versions(), [1, 2])
def test_fft_correct_modify_doctype(self):
"""bibupload - FFT correct with different doctype"""
test_to_upload = """
SzGeCERN
%(siteurl)s/img/site_logo.gif
a description
TEST1
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
site_logo
TEST2
"""
testrec_expected_xm = """
123456789
SzGeCERN
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
bibrecdocs = BibRecDocs(recid)
self.failUnless(bibrecdocs.get_bibdoc('site_logo').doctype, 'TEST1')
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
bibrecdocs = BibRecDocs(recid)
self.failUnless(bibrecdocs.get_bibdoc('site_logo').doctype, 'TEST2')
def test_fft_append_already_exists(self):
"""bibupload - FFT append with already identical existing file"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
a description
""" % {
'siteurl': CFG_SITE_URL
}
test_to_append = """
123456789
%(siteurl)s/img/site_logo.gif
a second description
%(siteurl)s/img/help.png
site_logo
another second description
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a description
786
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
_, recid, _ = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_append = test_to_append.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_append)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_url2))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
def test_fft_implicit_fix_marc(self):
"""bibupload - FFT implicit FIX-MARC"""
test_to_upload = """
SzGeCERN
Test, John
Test University
foo@bar.com
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
foo@bar.com
%(siteurl)s/img/site_logo.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
foo@bar.com
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
test_to_correct = test_to_correct.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
# correct test record with implicit FIX-MARC:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
def test_fft_vs_bibedit(self):
"""bibupload - FFT Vs. BibEdit compatibility"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_replace = """
123456789
SzGeCERN
Test, John
Test University
http://www.google.com/
2032
BibEdit Comment
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
BibEdit Description
01
http://cern.ch/
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_xm = str(test_to_replace)
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload_records(recs, opt_mode='replace')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'site_logo', '')
bibrecdocs = BibRecDocs(recid)
bibdoc = bibrecdocs.get_bibdoc('site_logo')
self.assertEqual(bibdoc.get_description('.gif'), 'BibEdit Description')
def test_detailed_fft_correct(self):
"""bibupload - detailed FFT correct
"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/head.gif
site_logo
patata
Next Try
KEEP-OLD-VALUE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Next Try
Comment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'patata', '')
def test_no_url_fft_correct(self):
"""bibupload - no_url FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
site_logo
patata
.gif
KEEP-OLD-VALUE
Next Comment
"""
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Try
Next Comment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'patata', '')
def test_new_icon_fft_append(self):
"""bibupload - new icon FFT append"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
"""
test_to_correct = """
123456789
site_logo
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='append')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.failUnless(try_url_download(testrec_expected_url))
self._test_bibdoc_status(recid, 'site_logo', '')
def test_multiple_fft_correct(self):
"""bibupload - multiple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
Restricted
%(siteurl)s/img/okay.gif
site_logo
.jpeg
Try jpeg
Comment jpeg
Restricted
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/loading.gif
site_logo
patata
.gif
New restricted
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
9427
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'patata', 'New restricted')
def test_purge_fft_correct(self):
"""bibupload - purge FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/head.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_purge = """
123456789
%(siteurl)s/img/site_logo.gif
PURGE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_purge = test_to_purge.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')[0]
self.check_record_consistency(recid)
# purge test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_purge)
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'site_logo', '')
self._test_bibdoc_status(recid, 'head', '')
def test_revert_fft_correct(self):
"""bibupload - revert FFT correct"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/iconpen.gif
site_logo
""" % {
'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
test_to_correct = """
123456789
%s/img/head.gif
site_logo
""" % CFG_SITE_URL
test_to_revert = """
123456789
site_logo
REVERT
1
"""
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
171
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_revert = test_to_revert.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
# revert test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_revert)
bibupload.bibupload_records(recs, opt_mode='correct')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self._test_bibdoc_status(recid, 'site_logo', '')
expected_content_version1 = urlopen('%s/img/iconpen.gif' % CFG_SITE_URL).read()
expected_content_version2 = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
expected_content_version3 = expected_content_version1
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=1' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version1), [])
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=2' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version2), [])
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=3' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version3), [])
def test_simple_fft_replace(self):
"""bibupload - simple FFT replace"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/iconpen.gif
site_logo
""" % {'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
test_to_replace = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/head.gif
""" % {'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
208
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
""" % {
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# replace test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload_records(recs, opt_mode='replace')
self.check_record_consistency(recid)
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
expected_content_version = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
self.assertEqual(test_web_page_content(testrec_expected_url, 'jekyll', 'j123ekyll', expected_text=expected_content_version), [])
def test_simple_fft_replace_or_insert(self):
"""bibupload - simple FFT replace_or_insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/iconpen.gif
site_logo
""" % {'siteurl': CFG_SITE_URL,}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='replace_or_insert')[0]
self.check_record_consistency(recid)
## When insert_or_replace a record for the first time, it should be like
## a simple insert, hence affected_fields should be empty.
## This also for the special case of FFT.
affected_fields = run_sql("SELECT affected_fields FROM hstRECORD where id_bibrec=%s", (recid,))
self.assertEqual(len(affected_fields), 1)
self.failIf(affected_fields[0][0])
def test_simple_fft_insert_with_modification_time(self):
"""bibupload - simple FFT insert with modification time"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
ARTICLE
%(siteurl)s/img/site_logo.gif
2006-05-04 03:02:01
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
2032
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
ARTICLE
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload_records(recs, opt_mode='insert')[0]
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url2.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
self.failUnless(records_identical(create_record(inserted_xm)[0], create_record(testrec_expected_xm)[0], ignore_subfield_order=True, ignore_field_order=True))
self.failUnless(try_url_download(testrec_expected_url))
self.force_webcoll(recid)
self.tear_down = True
self.assertEqual(test_web_page_content(testrec_expected_url2, expected_text='04 May 2006, 03:02'), [])
def test_multiple_fft_insert_with_modification_time(self):
"""bibupload - multiple FFT insert with modification time"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
ARTICLE
%(siteurl)s/img/site_logo.gif
2006-05-04 03:02:01
%(siteurl)s/img/head.gif
2007-05-04 03:02:01
%(siteurl)s/%(CFG_SITE_RECORD)s/95/files/9809057.pdf
2008-05-04 03:02:01
%(prefix)s/var/tmp/demobibdata.xml
2009-05-04 03:02:01
""" % {
'prefix': CFG_PREFIX,
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
self.force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_url, expected_text=['04 May 2006, 03:02', '04 May 2007, 03:02', '04 May 2008, 03:02', '04 May 2009, 03:02']), [])
def test_simple_fft_correct_with_modification_time(self):
"""bibupload - simple FFT correct with modification time"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
ARTICLE
%(siteurl)s/img/site_logo.gif
2007-05-04 03:02:01
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/sb.gif
site_logo
2008-05-04 03:02:01
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
dummy, recid, dummy = bibupload.bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(recid)
# replace test buffers with real recid of inserted test record:
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='correct')
self.check_record_consistency(recid)
self.force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_url, expected_text=['04 May 2008, 03:02']), [])
TEST_SUITE = make_test_suite(BibUploadNoUselessHistoryTest,
BibUploadHoldingPenTest,
BibUploadInsertModeTest,
BibUploadAppendModeTest,
BibUploadCorrectModeTest,
BibUploadDeleteModeTest,
BibUploadReplaceModeTest,
BibUploadReferencesModeTest,
BibUploadRecordsWithSYSNOTest,
BibUploadRecordsWithEXTOAIIDTest,
BibUploadRecordsWithOAIIDTest,
BibUploadIndicatorsTest,
BibUploadUpperLowerCaseTest,
BibUploadControlledProvenanceTest,
BibUploadStrongTagsTest,
BibUploadFFTModeTest,
BibUploadPretendTest,
BibUploadCallbackURLTest,
BibUploadMoreInfoTest,
BibUploadBibRelationsTest,
BibUploadRecordsWithDOITest,
BibUploadTypicalBibEditSessionTest,
BibUploadRealCaseRemovalDOIViaBibEdit,
)
if __name__ == "__main__":
run_test_suite(TEST_SUITE, warn_user=True)
diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py
index 78c1caa2d..3f1bee5f1 100644
--- a/modules/miscutil/lib/inveniocfg.py
+++ b/modules/miscutil/lib/inveniocfg.py
@@ -1,1785 +1,1785 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Invenio configuration and administration CLI tool.
Usage: inveniocfg [options]
General options:
-h, --help print this help
-V, --version print version number
Options to finish your installation:
--create-apache-conf create Apache configuration files
--create-tables create DB tables for Invenio
--load-bibfield-conf load the BibField configuration
--load-webstat-conf load the WebStat configuration
--drop-tables drop DB tables of Invenio
--check-openoffice check for correctly set up of openoffice temporary directory
Options to set up and test a demo site:
--create-demo-site create demo site
--load-demo-records load demo records
--remove-demo-records remove demo records, keeping demo site
--drop-demo-site drop demo site configurations too
--run-unit-tests run unit test suite (needs demo site)
--run-regression-tests run regression test suite (needs demo site)
--run-web-tests run web tests in a browser (needs demo site, Firefox, Selenium IDE)
Options to update config files in situ:
--update-all perform all the update options
--update-config-py update config.py file from invenio.conf file
--update-dbquery-py update dbquery.py with DB credentials from invenio.conf
--update-dbexec update dbexec with DB credentials from invenio.conf
--update-bibconvert-tpl update bibconvert templates with CFG_SITE_URL from invenio.conf
--update-web-tests update web test cases with CFG_SITE_URL from invenio.conf
Options to update DB tables:
--reset-all perform all the reset options
--reset-sitename reset tables to take account of new CFG_SITE_NAME*
--reset-siteadminemail reset tables to take account of new CFG_SITE_ADMIN_EMAIL
--reset-fieldnames reset tables to take account of new I18N names from PO files
--reset-recstruct-cache reset record structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE
--reset-recjson-cache reset record json cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE
Options to upgrade your installation:
--upgrade apply all pending upgrades
--upgrade-check run pre-upgrade checks for all pending upgrades
--upgrade-show-pending show pending upgrades ready to be applied
--upgrade-show-applied show history of applied upgrades
--upgrade-create-standard-recipe create a new upgrade recipe (for developers)
--upgrade-create-release-recipe create a new release upgrade recipe (for developers)
Options to help the work:
--list print names and values of all options from conf files
--get get value of a given option from conf files
--conf-dir path to directory where invenio*.conf files are [optional]
--detect-system-details print system details such as Apache/Python/MySQL versions
"""
__revision__ = "$Id$"
from ConfigParser import ConfigParser
from optparse import OptionParser, OptionGroup, IndentedHelpFormatter, Option, \
OptionError
import os
import re
import shutil
import socket
import sys
import zlib
def print_usage():
"""Print help."""
print __doc__
def get_version():
""" Get running version of Invenio """
from invenio.config import CFG_VERSION
return CFG_VERSION
def print_version():
"""Print version information."""
print get_version()
def convert_conf_option(option_name, option_value):
"""
Convert conf option into Python config.py line, converting
values to ints or strings as appropriate.
"""
## 1) convert option name to uppercase:
option_name = option_name.upper()
## 1a) adjust renamed variables:
if option_name in ['CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES',
'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS',
'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC',
'CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT',
'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS',
'CFG_WEBSUBMIT_DESIRED_CONVERSIONS']:
new_option_name = option_name.replace('WEBSUBMIT', 'BIBDOCFILE')
print >> sys.stderr, ("""WARNING: %s has been renamed to %s.
Please, update your invenio-local.conf file accordingly.""" % (option_name, new_option_name))
option_name = new_option_name
## 2) convert option value to int or string:
if option_name in ['CFG_BIBUPLOAD_REFERENCE_TAG',
'CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS']:
# some options are supposed be string even when they look like
# numeric
option_value = '"' + option_value + '"'
else:
try:
option_value = int(option_value)
except ValueError:
option_value = '"' + option_value + '"'
## 3a) special cases: chars regexps
if option_name in ['CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS',
'CFG_BIBINDEX_CHARS_PUNCTUATION']:
option_value = 'r"[' + option_value[1:-1] + ']"'
## 3abis) special cases: real regexps
if option_name in ['CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES',
'CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS',
'CFG_BIBUPLOAD_INTERNAL_DOI_PATTERN']:
option_value = 'r"' + option_value[1:-1] + '"'
## 3b) special cases: True, False, None
if option_value in ['"True"', '"False"', '"None"']:
option_value = option_value[1:-1]
## 3c) special cases: dicts and real pythonic lists
if option_name in ['CFG_WEBSEARCH_FIELDS_CONVERT',
'CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS',
'CFG_WEBSEARCH_FULLTEXT_SNIPPETS',
'CFG_WEBSEARCH_FULLTEXT_SNIPPETS_CHARS',
'CFG_SITE_EMERGENCY_EMAIL_ADDRESSES',
'CFG_BIBMATCH_FUZZY_WORDLIMITS',
'CFG_BIBMATCH_QUERY_TEMPLATES',
'CFG_WEBSEARCH_SYNONYM_KBRS',
'CFG_BIBINDEX_SYNONYM_KBRS',
'CFG_WEBCOMMENT_EMAIL_REPLIES_TO',
'CFG_WEBCOMMENT_RESTRICTION_DATAFIELD',
'CFG_WEBCOMMENT_ROUND_DATAFIELD',
'CFG_BIBUPLOAD_FFT_ALLOWED_EXTERNAL_URLS',
'CFG_BIBSCHED_NODE_TASKS',
'CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE',
'CFG_OAI_METADATA_FORMATS',
'CFG_BIBDOCFILE_DESIRED_CONVERSIONS',
'CFG_BIBDOCFILE_BEST_FORMATS_TO_EXTRACT_TEXT_FROM',
'CFG_WEB_API_KEY_ALLOWED_URL',
'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_MISC',
'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_DOCTYPES',
'CFG_BIBDOCFILE_DOCUMENT_FILE_MANAGER_RESTRICTIONS',
'CFG_REFEXTRACT_KBS_OVERRIDE',
'CFG_OPENID_CONFIGURATIONS',
'CFG_OAUTH1_CONFIGURATIONS',
'CFG_OAUTH2_CONFIGURATIONS',
'CFG_BIBDOCFILE_ADDITIONAL_KNOWN_MIMETYPES',
'CFG_BIBDOCFILE_PREFERRED_MIMETYPES_MAPPING',
'CFG_BIBSCHED_NON_CONCURRENT_TASKS',
'CFG_REDIS_HOSTS',
'CFG_BIBSCHED_INCOMPATIBLE_TASKS',
'CFG_ICON_CREATION_FORMAT_MAPPINGS',
'CFG_BIBEDIT_AUTOCOMPLETE']:
try:
option_value = option_value[1:-1]
if option_name == "CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE" and option_value.strip().startswith("{"):
print >> sys.stderr, ("""ERROR: CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE
now accepts only a list of tuples, not a dictionary. Check invenio.conf for an example.
Please, update your invenio-local.conf file accordingly.""")
sys.exit(1)
except TypeError:
if option_name in ('CFG_WEBSEARCH_FULLTEXT_SNIPPETS',):
print >> sys.stderr, """WARNING: CFG_WEBSEARCH_FULLTEXT_SNIPPETS
has changed syntax: it can be customised to display different snippets for
different document types. See the corresponding documentation in invenio.conf.
You may want to customise your invenio-local.conf configuration accordingly."""
option_value = """{'': %s}""" % option_value
else:
print >> sys.stderr, "ERROR: type error in %s value %s." % \
(option_name, option_value)
sys.exit(1)
## 3cbis) very special cases: dicts with backward compatible string
if option_name in ['CFG_BIBINDEX_SPLASH_PAGES']:
if option_value.startswith('"{') and option_value.endswith('}"'):
option_value = option_value[1:-1]
else:
option_value = """{%s: ".*"}""" % option_value
## 3d) special cases: comma-separated lists
if option_name in ['CFG_SITE_LANGS',
'CFG_BIBDOCFILE_ADDITIONAL_KNOWN_FILE_EXTENSIONS',
'CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS',
'CFG_BIBFORMAT_HIDDEN_RECJSON_FIELDS',
'CFG_BIBSCHED_GC_TASKS_TO_REMOVE',
'CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE',
'CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS',
'CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS',
'CFG_BIBUPLOAD_DELETE_FORMATS',
'CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST',
'CFG_WEBSEARCH_RSS_I18N_COLLECTIONS',
'CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY',
'CFG_BIBAUTHORID_EXTERNAL_CLAIMED_RECORDS_KEY',
'CFG_BIBCIRCULATION_ITEM_STATUS_OPTIONAL',
'CFG_PLOTEXTRACTOR_DISALLOWED_TEX',
'CFG_OAI_FRIENDS',
'CFG_WEBSTYLE_REVERSE_PROXY_IPS',
'CFG_BIBEDIT_AUTOCOMPLETE_INSTITUTIONS_FIELDS',
'CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS',
'CFG_BIBFORMAT_HIDDEN_FILE_FORMATS',
'CFG_BIBFIELD_MASTER_FORMATS',
'CFG_OPENID_PROVIDERS',
'CFG_OAUTH1_PROVIDERS',
'CFG_OAUTH2_PROVIDERS',
'CFG_BIBFORMAT_CACHED_FORMATS',
'CFG_BIBEDIT_ADD_TICKET_RT_QUEUES',
'CFG_BIBAUTHORID_ENABLED_REMOTE_LOGIN_SYSTEMS',]:
out = "["
for elem in option_value[1:-1].split(","):
if elem:
elem = elem.strip()
# string values
out += "'%s', " % elem
out += "]"
option_value = out
## 3e) special cases: multiline
if option_name == 'CFG_OAI_IDENTIFY_DESCRIPTION':
# make triple quotes
option_value = '""' + option_value + '""'
## 3f) ignore some options:
if option_name.startswith('CFG_SITE_NAME_INTL'):
# treated elsewhere
return
## 3g) special cases: float
if option_name in ['CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY',
'CFG_BIBMATCH_LOCAL_SLEEPTIME',
'CFG_BIBMATCH_REMOTE_SLEEPTIME',
'CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT',
'CFG_BIBMATCH_FUZZY_MATCH_VALIDATION_LIMIT']:
option_value = float(option_value[1:-1])
## 3h) special cases: bibmatch validation list
if option_name in ['CFG_BIBMATCH_MATCH_VALIDATION_RULESETS']:
option_value = option_value[1:-1]
## 4a) dropped variables
if option_name in ['CFG_BATCHUPLOADER_WEB_ROBOT_AGENT']:
print >> sys.stderr, ("""ERROR: CFG_BATCHUPLOADER_WEB_ROBOT_AGENT has been dropped in favour of
CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS.
Please, update your invenio-local.conf file accordingly.""")
sys.exit(1)
option_value = option_value[1:-1]
elif option_name in ['CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES',
'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS',
'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC',
'CFG_WEBSUBMIT_FILESYSTEM_BIBDOC_GROUP_LIMIT',
'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS',
'CFG_WEBSUBMIT_DESIRED_CONVERSIONS']:
new_option_name = option_name.replace('WEBSUBMIT', 'BIBDOCFILE')
print >> sys.stderr, ("""ERROR: %s has been renamed to %s.
Please, update your invenio-local.conf file accordingly.""" % (option_name, new_option_name))
option_name = new_option_name
## 5) finally, return output line:
return '%s = %s' % (option_name, option_value)
def cli_cmd_update_config_py(conf):
"""
Update new config.py from conf options, keeping previous
config.py in a backup copy.
"""
## NOTE: the following function exists also in urlutils.py
## However we can't import urlutils here, as it depends on config.py
## to already exist, while we are in the process of creating it.
def get_relative_url(url):
"""
Returns the relative URL from a URL. For example:
'http://web.net' -> ''
'http://web.net/' -> ''
'http://web.net/1222' -> '/1222'
'http://web.net/wsadas/asd' -> '/wsadas/asd'
It will never return a trailing "/".
@param url: A url to transform
@type url: str
@return: relative URL
"""
# remove any protocol info before
stripped_site_url = url.replace("://", "")
baseurl = "/" + "/".join(stripped_site_url.split("/")[1:])
# remove any trailing slash ("/")
if baseurl[-1] == "/":
return baseurl[:-1]
else:
return baseurl
print ">>> Going to update config.py..."
## location where config.py is:
configpyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'config.py'
## backup current config.py file:
if os.path.exists(configpyfile):
shutil.copy(configpyfile, configpyfile + '.OLD')
## here we go:
fdesc = open(configpyfile, 'w')
## generate preamble:
fdesc.write("# -*- coding: utf-8 -*-\n")
fdesc.write("# DO NOT EDIT THIS FILE! IT WAS AUTOMATICALLY GENERATED\n")
fdesc.write("# FROM INVENIO.CONF BY EXECUTING:\n")
fdesc.write("# " + " ".join(sys.argv) + "\n")
## special treatment for CFG_SITE_NAME_INTL options:
fdesc.write("CFG_SITE_NAME_INTL = {}\n")
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
fdesc.write("CFG_SITE_NAME_INTL['%s'] = \"%s\"\n" % (lang, conf.get("Invenio",
"CFG_SITE_NAME_INTL_" + lang)))
## special treatment for CFG_SITE_SECURE_URL that may be empty, in
## which case it should be put equal to CFG_SITE_URL:
if not conf.get("Invenio", "CFG_SITE_SECURE_URL"):
conf.set("Invenio", "CFG_SITE_SECURE_URL",
conf.get("Invenio", "CFG_SITE_URL"))
## Special treatment of base URL, adding CFG_BASE_URL
base_url = get_relative_url(conf.get("Invenio", "CFG_SITE_URL"))
fdesc.write("CFG_BASE_URL = \"%s\"\n" % (base_url,))
## process all the options normally:
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
if not option.upper().startswith('CFG_DATABASE_'):
# put all options except for db credentials into config.py
line_out = convert_conf_option(option, conf.get(section, option))
if line_out:
fdesc.write(line_out + "\n")
## generate postamble:
fdesc.write("")
fdesc.write("# END OF GENERATED FILE")
## we are done:
fdesc.close()
print "You may want to restart Apache now."
print ">>> config.py updated successfully."
def cli_cmd_update_dbquery_py(conf):
"""
Update lib/dbquery.py file with DB parameters read from conf file.
Note: this edits dbquery.py in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbquery.py..."
## location where dbquery.py is:
dbqueryconfigpyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'dbquery_config.py'
## backup current dbquery.py file:
if os.path.exists(dbqueryconfigpyfile + 'c'):
shutil.copy(dbqueryconfigpyfile + 'c', dbqueryconfigpyfile + 'c.OLD')
out = ["%s = '%s'\n" % (item.upper(), value) \
for item, value in conf.items('Invenio') \
if item.upper().startswith('CFG_DATABASE_')]
fdesc = open(dbqueryconfigpyfile, 'w')
fdesc.write("# -*- coding: utf-8 -*-\n")
fdesc.writelines(out)
fdesc.close()
print "You may want to restart Apache now."
print ">>> dbquery.py updated successfully."
def cli_cmd_update_dbexec(conf):
"""
Update bin/dbexec file with DB parameters read from conf file.
Note: this edits dbexec in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbexec..."
## location where dbexec is:
dbexecfile = conf.get("Invenio", "CFG_BINDIR") + \
os.sep + 'dbexec'
## backup current dbexec file:
if os.path.exists(dbexecfile):
shutil.copy(dbexecfile, dbexecfile + '.OLD')
## replace db parameters via sed:
out = ''
for line in open(dbexecfile, 'r').readlines():
match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS|SLAVE)(\s*=\s*)\'.*\'$', line)
if match:
dbparam = 'CFG_DATABASE_' + match.group(1)
out += "%s%s'%s'\n" % (dbparam, match.group(2),
conf.get("Invenio", dbparam))
else:
out += line
fdesc = open(dbexecfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> dbexec updated successfully."
def cli_cmd_update_bibconvert_tpl(conf):
"""
Update bibconvert/config/*.tpl files looking for 856
http://.../CFG_SITE_RECORD lines, replacing URL with CFG_SITE_URL taken
from conf file. Note: this edits tpl files in situ, taking a
backup first. Use only when you know what you are doing.
"""
print ">>> Going to update bibconvert templates..."
## location where bibconvert/config/*.tpl are:
tpldir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'bibconvert' + os.sep + 'config'
## find all *.tpl files:
for tplfilename in os.listdir(tpldir):
if tplfilename.endswith(".tpl"):
## change tpl file:
tplfile = tpldir + os.sep + tplfilename
shutil.copy(tplfile, tplfile + '.OLD')
out = ''
for line in open(tplfile, 'r').readlines():
match = re.search(r'^(.*)http://.*?/%s/(.*)$' % conf.get("Invenio", 'CFG_SITE_RECORD'), line)
if match:
out += "%s%s/%s/%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_SITE_URL'),
conf.get("Invenio", 'CFG_SITE_RECORD'),
match.group(2))
else:
out += line
fdesc = open(tplfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> bibconvert templates updated successfully."
def cli_cmd_update_web_tests(conf):
"""
Update web test cases lib/webtest/test_*.html looking for
http://.+?[>> Going to update web tests..."
## location where test_*.html files are:
testdir = conf.get("Invenio", 'CFG_PREFIX') + os.sep + \
'lib' + os.sep + 'webtest' + os.sep + 'invenio'
## find all test_*.html files:
for testfilename in os.listdir(testdir):
if testfilename.startswith("test_") and \
testfilename.endswith(".html"):
## change test file:
testfile = testdir + os.sep + testfilename
shutil.copy(testfile, testfile + '.OLD')
out = ''
for line in open(testfile, 'r').readlines():
match = re.search(r'^(.* )http://.+?([)/opt/invenio(.*)$', line)
if match:
out += "%s%s%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_PREFIX'),
match.group(2))
else:
out += line
fdesc = open(testfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> web tests updated successfully."
def cli_cmd_reset_sitename(conf):
"""
Reset collection-related tables with new CFG_SITE_NAME and
CFG_SITE_NAME_INTL* read from conf files.
"""
print ">>> Going to reset CFG_SITE_NAME and CFG_SITE_NAME_INTL..."
from invenio.dbquery import run_sql, IntegrityError
# reset CFG_SITE_NAME:
sitename = conf.get("Invenio", "CFG_SITE_NAME")
try:
run_sql("""INSERT INTO collection (id, name, dbquery, reclist) VALUES
(1,%s,NULL,NULL)""", (sitename,))
except IntegrityError:
run_sql("""UPDATE collection SET name=%s WHERE id=1""", (sitename,))
# reset CFG_SITE_NAME_INTL:
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
sitename_lang = conf.get("Invenio", "CFG_SITE_NAME_INTL_" + lang)
try:
run_sql("""INSERT INTO collectionname (id_collection, ln, type, value) VALUES
(%s,%s,%s,%s)""", (1, lang, 'ln', sitename_lang))
except IntegrityError:
run_sql("""UPDATE collectionname SET value=%s
WHERE ln=%s AND id_collection=1 AND type='ln'""",
(sitename_lang, lang))
print "You may want to restart Apache now."
print ">>> CFG_SITE_NAME and CFG_SITE_NAME_INTL* reset successfully."
def cli_cmd_reset_recstruct_cache(conf):
"""If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the recstruct
format."""
from invenio.intbitset import intbitset
from invenio.dbquery import run_sql, serialize_via_marshal
from invenio.search_engine import get_record, print_record
from invenio.bibsched import server_pid, pidfile
enable_recstruct_cache = conf.get("Invenio", "CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
enable_recstruct_cache = enable_recstruct_cache in ('True', '1')
pid = server_pid(ping_the_process=False)
if pid:
print >> sys.stderr, "ERROR: bibsched seems to run with pid %d, according to %s." % (pid, pidfile)
print >> sys.stderr, " Please stop bibsched before running this procedure."
sys.exit(1)
if enable_recstruct_cache:
print ">>> Searching records which need recstruct cache resetting; this may take a while..."
all_recids = intbitset(run_sql("SELECT id FROM bibrec"))
good_recids = intbitset(run_sql("SELECT bibrec.id FROM bibrec JOIN bibfmt ON bibrec.id = bibfmt.id_bibrec WHERE format='recstruct' AND modification_date < last_updated"))
recids = all_recids - good_recids
print ">>> Generating recstruct cache..."
tot = len(recids)
count = 0
for recid in recids:
try:
value = serialize_via_marshal(get_record(recid))
except zlib.error, err:
print >> sys.stderr, "Looks like XM is corrupted for record %s. Let's recover it from bibxxx" % recid
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='xm'", (recid, ))
xm_value = zlib.compress(print_record(recid, 'xm'))
run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'xm', NOW(), %s)", (recid, xm_value))
value = serialize_via_marshal(get_record(recid))
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='recstruct'", (recid, ))
- run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'recstruct', NOW(), %s)", (recid, value))
+ run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'recstruct', NOW(), _binary %s)", (recid, value))
count += 1
if count % 1000 == 0:
print " ... done records %s/%s" % (count, tot)
if count % 1000 != 0:
print " ... done records %s/%s" % (count, tot)
print ">>> recstruct cache generated successfully."
else:
print ">>> Cleaning recstruct cache..."
run_sql("DELETE FROM bibfmt WHERE format='recstruct'")
def cli_cmd_reset_recjson_cache(conf):
"""If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the recjson
format."""
try:
import cPickle as pickle
except:
import pickle
from invenio.intbitset import intbitset
from invenio.dbquery import run_sql
from invenio.bibfield import get_record
from invenio.bibsched import server_pid, pidfile
enable_recjson_cache = conf.get("Invenio", "CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
enable_recjson_cache = enable_recjson_cache in ('True', '1')
pid = server_pid(ping_the_process=False)
if pid:
print >> sys.stderr, "ERROR: bibsched seems to run with pid %d, according to %s." % (pid, pidfile)
print >> sys.stderr, " Please stop bibsched before running this procedure."
sys.exit(1)
if enable_recjson_cache:
print ">>> Searching records which need recjson cache resetting; this may take a while..."
all_recids = intbitset(run_sql("SELECT id FROM bibrec"))
#TODO: prevent doing all records?
recids = all_recids
print ">>> Generating recjson cache..."
tot = len(recids)
count = 0
cli_cmd_load_bibfield_config(conf)
for recid in recids:
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='recjson'", (recid,))
#TODO: Update the cache or wait for the first access
get_record(recid)
count += 1
if count % 1000 == 0:
print " ... done records %s/%s" % (count, tot)
if count % 1000 != 0:
print " ... done records %s/%s" % (count, tot)
print ">>> recjson cache generated successfully."
def cli_cmd_reset_siteadminemail(conf):
"""
Reset user-related tables with new CFG_SITE_ADMIN_EMAIL read from conf files.
"""
print ">>> Going to reset CFG_SITE_ADMIN_EMAIL..."
from invenio.dbquery import run_sql
siteadminemail = conf.get("Invenio", "CFG_SITE_ADMIN_EMAIL")
run_sql("DELETE FROM user WHERE id=1")
run_sql("""INSERT INTO user (id, email, password, note, nickname) VALUES
(1, %s, AES_ENCRYPT(email, ''), 1, 'admin')""",
(siteadminemail,))
print "You may want to restart Apache now."
print ">>> CFG_SITE_ADMIN_EMAIL reset successfully."
def cli_cmd_reset_fieldnames(conf):
"""
Reset I18N field names such as author, title, etc and other I18N
ranking method names such as word similarity. Their translations
are taken from the PO files.
"""
print ">>> Going to reset I18N field names..."
from invenio.messages import gettext_set_language, language_list_long
from invenio.dbquery import run_sql, IntegrityError
## get field id and name list:
field_id_name_list = run_sql("SELECT id, name FROM field")
## get rankmethod id and name list:
rankmethod_id_name_list = run_sql("SELECT id, name FROM rnkMETHOD")
## update names for every language:
for lang, dummy in language_list_long():
_ = gettext_set_language(lang)
## this list is put here in order for PO system to pick names
## suitable for translation
field_name_names = {"any field": _("any field"),
"title": _("title"),
"author": _("author"),
"abstract": _("abstract"),
"keyword": _("keyword"),
"report number": _("report number"),
"subject": _("subject"),
"reference": _("reference"),
"fulltext": _("fulltext"),
"collection": _("collection"),
"division": _("division"),
"year": _("year"),
"journal": _("journal"),
"experiment": _("experiment"),
"record ID": _("record ID")}
## update I18N names for every language:
for (field_id, field_name) in field_id_name_list:
if field_name_names.has_key(field_name):
try:
run_sql("""INSERT INTO fieldname (id_field,ln,type,value) VALUES
(%s,%s,%s,%s)""", (field_id, lang, 'ln',
field_name_names[field_name]))
except IntegrityError:
run_sql("""UPDATE fieldname SET value=%s
WHERE id_field=%s AND ln=%s AND type=%s""",
(field_name_names[field_name], field_id, lang, 'ln',))
## ditto for rank methods:
rankmethod_name_names = {"wrd": _("word similarity"),
"demo_jif": _("journal impact factor"),
"citation": _("times cited"),
"citerank_citation_t": _("time-decay cite count"),
"citerank_pagerank_c": _("all-time-best cite rank"),
"citerank_pagerank_t": _("time-decay cite rank"),}
for (rankmethod_id, rankmethod_name) in rankmethod_id_name_list:
if rankmethod_name_names.has_key(rankmethod_name):
try:
run_sql("""INSERT INTO rnkMETHODNAME (id_rnkMETHOD,ln,type,value) VALUES
(%s,%s,%s,%s)""", (rankmethod_id, lang, 'ln',
rankmethod_name_names[rankmethod_name]))
except IntegrityError:
run_sql("""UPDATE rnkMETHODNAME SET value=%s
WHERE id_rnkMETHOD=%s AND ln=%s AND type=%s""",
(rankmethod_name_names[rankmethod_name], rankmethod_id, lang, 'ln',))
print ">>> I18N field names reset successfully."
def cli_check_openoffice(conf):
"""
If OpenOffice.org integration is enabled, checks whether the system is
properly configured.
"""
from invenio.bibtask import check_running_process_user
from invenio.websubmit_file_converter import can_unoconv, get_file_converter_logger
logger = get_file_converter_logger()
for handler in logger.handlers:
logger.removeHandler(handler)
check_running_process_user()
print ">>> Checking if Libre/OpenOffice.org is correctly integrated...",
sys.stdout.flush()
if can_unoconv(True):
print "ok"
else:
sys.exit(1)
def test_db_connection():
"""
Test DB connection, and if fails, advise user how to set it up.
Useful to be called during table creation.
"""
print "Testing DB connection...",
from invenio.textutils import wrap_text_in_a_box
from invenio.dbquery import run_sql, Error
## first, test connection to the DB server:
try:
run_sql("SHOW TABLES")
except Error, err:
from invenio.dbquery import CFG_DATABASE_HOST, CFG_DATABASE_PORT, \
CFG_DATABASE_NAME, CFG_DATABASE_USER, CFG_DATABASE_PASS
print wrap_text_in_a_box("""\
DATABASE CONNECTIVITY ERROR %(errno)d: %(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please login as MySQL admin user and run the
following commands now:
$ mysql -h %(dbhost)s -P %(dbport)s -u root -p mysql
mysql> CREATE DATABASE %(dbname)s DEFAULT CHARACTER SET utf8;
mysql> GRANT ALL PRIVILEGES ON %(dbname)s.*
TO %(dbuser)s@%(webhost)s IDENTIFIED BY '%(dbpass)s';
mysql> QUIT
The values printed above were detected from your
configuration. If they are not right, then please edit your
invenio-local.conf file and rerun 'inveniocfg --update-all' first.
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing.""" % \
{'errno': err.args[0],
'errmsg': err.args[1],
'dbname': CFG_DATABASE_NAME,
'dbhost': CFG_DATABASE_HOST,
'dbport': CFG_DATABASE_PORT,
'dbuser': CFG_DATABASE_USER,
'dbpass': CFG_DATABASE_PASS,
'webhost': CFG_DATABASE_HOST == 'localhost' and 'localhost' or os.popen('hostname -f', 'r').read().strip(),
})
sys.exit(1)
print "ok"
## second, test insert/select of a Unicode string to detect
## possible Python/MySQL/MySQLdb mis-setup:
print "Testing Python/MySQL/MySQLdb UTF-8 chain...",
try:
try:
beta_in_utf8 = "β" # Greek beta in UTF-8 is 0xCEB2
run_sql("CREATE TABLE test__invenio__utf8 (x char(1), y varbinary(2)) DEFAULT CHARACTER SET utf8 ENGINE=MyISAM;")
run_sql("INSERT INTO test__invenio__utf8 (x, y) VALUES (%s, %s)", (beta_in_utf8, beta_in_utf8))
res = run_sql("SELECT x,y,HEX(x),HEX(y),LENGTH(x),LENGTH(y),CHAR_LENGTH(x),CHAR_LENGTH(y) FROM test__invenio__utf8")
assert res[0] == ('\xce\xb2', '\xce\xb2', 'CEB2', 'CEB2', 2L, 2L, 1L, 2L)
run_sql("DROP TABLE test__invenio__utf8")
except Exception, err:
print wrap_text_in_a_box("""\
DATABASE RELATED ERROR %s\n
A problem was detected with the UTF-8 treatment in the chain
between the Python application, the MySQLdb connector, and
the MySQL database. You may perhaps have installed older
versions of some prerequisite packages?\n
Please check the INSTALL file and please fix this problem
before continuing.""" % err)
sys.exit(1)
finally:
run_sql("DROP TABLE IF EXISTS test__invenio__utf8")
print "ok"
def cli_cmd_create_tables(conf):
"""Create and fill Invenio DB tables. Useful for the installation process."""
print ">>> Going to create and fill tables..."
from invenio.config import CFG_PREFIX
test_db_connection()
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabcreate.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/dbexec < %s/lib/sql/invenio/tabfill.sql" % (CFG_PREFIX, CFG_PREFIX)]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
for cmd in ["%s/bin/webaccessadmin -u admin -c -a" % CFG_PREFIX]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables created and filled successfully."
def cli_cmd_load_webstat_conf(conf):
print ">>> Going to load WebStat config..."
from invenio.config import CFG_PREFIX
cmd = "%s/bin/webstatadmin --load-config" % CFG_PREFIX
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> WebStat config load successfully."
def cli_cmd_load_bibfield_config(conf):
print ">>> Going to load BibField config..."
from invenio.bibfield_config_engine import BibFieldParser
BibFieldParser.reparse()
print ">>> BibField config load successfully."
def cli_cmd_drop_tables(conf):
"""Drop Invenio DB tables. Useful for the uninstallation process."""
print ">>> Going to drop tables..."
from invenio.config import CFG_PREFIX
from invenio.textutils import wrap_text_in_a_box, wait_for_user
from invenio.webstat import destroy_customevents
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your database tables!"""))
msg = destroy_customevents()
if msg:
print msg
cmd = "%s/bin/dbexec < %s/lib/sql/invenio/tabdrop.sql" % (CFG_PREFIX, CFG_PREFIX)
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables dropped successfully."
def cli_cmd_create_demo_site(conf):
"""Create demo site. Useful for testing purposes."""
print ">>> Going to create demo site..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
run_sql("TRUNCATE schTASK")
run_sql("TRUNCATE session")
run_sql("DELETE FROM user WHERE email=''")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/democfgdata.sql" % \
(CFG_PREFIX, CFG_PREFIX),]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_fieldnames(conf) # needed for I18N demo ranking method names
for cmd in ["%s/bin/webaccessadmin -u admin -c -r -D" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,
"%s/bin/bibsort -u admin --load-config" % CFG_PREFIX,
"%s/bin/bibsort 2" % CFG_PREFIX, ]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo site created successfully."
def cli_cmd_load_demo_records(conf):
"""Load demo records. Useful for testing purposes."""
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
print ">>> Going to load demo records..."
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/bibupload -u admin -i %s/var/tmp/demobibdata.xml" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/bibupload 1" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --with-ocr --recid 97" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --all" % CFG_PREFIX,
"%s/bin/bibindex -u admin" % CFG_PREFIX,
"%s/bin/bibindex 2" % CFG_PREFIX,
"%s/bin/bibindex -u admin -w global" % CFG_PREFIX,
"%s/bin/bibindex 3" % CFG_PREFIX,
"%s/bin/bibreformat -u admin -o HB" % CFG_PREFIX,
"%s/bin/bibreformat 4" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 5" % CFG_PREFIX,
"%s/bin/bibrank -u admin" % CFG_PREFIX,
"%s/bin/bibrank 6" % CFG_PREFIX,
"%s/bin/bibsort -u admin -R" % CFG_PREFIX,
"%s/bin/bibsort 7" % CFG_PREFIX,
"%s/bin/oairepositoryupdater -u admin" % CFG_PREFIX,
"%s/bin/oairepositoryupdater 8" % CFG_PREFIX,
"%s/bin/bibupload 9" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records loaded successfully."
def cli_cmd_remove_demo_records(conf):
"""Remove demo records. Useful when you are finished testing."""
print ">>> Going to remove demo records..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your records and documents!"""))
if os.path.exists(CFG_PREFIX + os.sep + 'var' + os.sep + 'data'):
shutil.rmtree(CFG_PREFIX + os.sep + 'var' + os.sep + 'data')
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabbibclean.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records removed successfully."
def cli_cmd_drop_demo_site(conf):
"""Drop demo site completely. Useful when you are finished testing."""
print ">>> Going to drop demo site..."
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your site and documents!"""))
cli_cmd_drop_tables(conf)
cli_cmd_create_tables(conf)
cli_cmd_remove_demo_records(conf)
print ">>> Demo site dropped successfully."
def cli_cmd_run_unit_tests(conf):
"""Run unit tests, usually on the working demo site."""
from invenio.testutils import build_and_run_unit_test_suite
if not build_and_run_unit_test_suite():
sys.exit(1)
def cli_cmd_run_js_unit_tests(conf):
"""Run JavaScript unit tests, usually on the working demo site."""
from invenio.testutils import build_and_run_js_unit_test_suite
if not build_and_run_js_unit_test_suite():
sys.exit(1)
def cli_cmd_run_regression_tests(conf):
"""Run regression tests, usually on the working demo site."""
from invenio.testutils import build_and_run_regression_test_suite
if not build_and_run_regression_test_suite():
sys.exit(1)
def cli_cmd_run_web_tests(conf):
"""Run web tests in a browser. Requires Firefox with Selenium."""
from invenio.testutils import build_and_run_web_test_suite
if not build_and_run_web_test_suite():
sys.exit(1)
def _detect_ip_address(conf):
"""Detect IP address of this computer. Useful for creating Apache
vhost conf snippet on RHEL like machines. However, if wanted site
is 0.0.0.0, then use that, since we are running inside Docker.
@return: IP address, or '*' if cannot detect
@rtype: string
@note: creates socket for real in order to detect real IP address,
not the loopback one.
"""
if '0.0.0.0' in conf.get('Invenio', 'CFG_SITE_URL'):
return '0.0.0.0'
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('invenio-software.org', 0))
return s.getsockname()[0]
except:
return '*'
def cli_cmd_create_apache_conf(conf):
"""
Create Apache conf files for this site, keeping previous
files in a backup copy.
"""
print ">>> Going to create Apache conf files..."
from invenio.textutils import wrap_text_in_a_box
from invenio.access_control_config import CFG_EXTERNAL_AUTH_USING_SSO
apache_conf_dir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'apache'
if guess_apache_24():
directory_www_directive = """
# Uncomment the following on Apache < 2.4
#
# Options FollowSymLinks MultiViews
# AllowOverride None
# Order allow,deny
# Allow from all
#
# Comment the following on Apache < 2.4
Options FollowSymLinks MultiViews
AllowOverride None
Require all granted
""" % {'webdir': conf.get('Invenio', 'CFG_WEBDIR')}
directory_wsgi_directive = """
# Uncomment the following on Apache < 2.4
#
# WSGIProcessGroup invenio
# WSGIApplicationGroup %%{GLOBAL}
# Options FollowSymLinks MultiViews
# AllowOverride None
# Order allow,deny
# Allow from all
#
# Comment the following on Apache < 2.4
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Require all granted
""" % {'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi')}
else:
directory_www_directive = """
# Comment the following on Apache >= 2.4
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
# Uncomment the following on Apache >= 2.4
#
# Options FollowSymLinks MultiViews
# AllowOverride None
# Require all granted
# """ % {'webdir': conf.get('Invenio', 'CFG_WEBDIR')}
directory_wsgi_directive = """
# Comment the following on Apache >= 2.4
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
# Uncomment the following on Apache >= 2.4
#
# WSGIProcessGroup invenio
# WSGIApplicationGroup %%{GLOBAL}
# Options FollowSymLinks MultiViews
# AllowOverride None
# Require all granted
# """ % {'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi')}
## Preparation of XSendFile directive
xsendfile_directive_needed = int(conf.get("Invenio", 'CFG_BIBDOCFILE_USE_XSENDFILE')) != 0
if xsendfile_directive_needed:
xsendfile_directive = "XSendFile On\n"
else:
xsendfile_directive = "#XSendFile On\n"
for path in (conf.get('Invenio', 'CFG_BIBDOCFILE_FILEDIR'), # BibDocFile
conf.get('Invenio', 'CFG_WEBDIR'),
conf.get('Invenio', 'CFG_WEBSUBMIT_STORAGEDIR'), # WebSubmit
conf.get('Invenio', 'CFG_TMPDIR'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'tmp', 'attachfile'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'comments'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'baskets', 'comments'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'lib', 'webdoc', 'invenio', 'info'),
'/tmp'): # BibExport
if xsendfile_directive_needed:
xsendfile_directive += ' XSendFilePath %s\n' % path
else:
xsendfile_directive += ' #XSendFilePath %s\n' % path
xsendfile_directive = xsendfile_directive.strip()
## Preparation of deflate directive
deflate_directive_needed = int(conf.get("Invenio", 'CFG_WEBSTYLE_HTTP_USE_COMPRESSION')) != 0
if deflate_directive_needed:
deflate_directive = r"""
## Configuration snippet taken from:
##
SetOutputFilter DEFLATE
# Netscape 4.x has some problems...
BrowserMatch ^Mozilla/4 gzip-only-text/html
# Netscape 4.06-4.08 have some more problems
BrowserMatch ^Mozilla/4\.0[678] no-gzip
# MSIE masquerades as Netscape, but it is fine
# BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
# NOTE: Due to a bug in mod_setenvif up to Apache 2.0.48
# the above regex won't work. You can use the following
# workaround to get the desired effect:
BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
# Don't compress images
SetEnvIfNoCase Request_URI \
\.(?:gif|jpe?g|png)$ no-gzip dont-vary
# Make sure proxies don't deliver the wrong content
Header append Vary User-Agent env=!dont-vary
"""
else:
deflate_directive = ""
if CFG_EXTERNAL_AUTH_USING_SSO:
shibboleth_directive = r"""
SSLRequireSSL # The modules only work using HTTPS
AuthType shibboleth
ShibRequireSession On
ShibRequireAll On
ShibExportAssertion Off
require valid-user
"""
else:
shibboleth_directive = ""
## Apache vhost conf file is distro specific, so analyze needs:
# Gentoo (and generic defaults):
listen_directive_needed = True
ssl_pem_directive_needed = False
ssl_pem_path = '/etc/apache2/ssl/apache.pem'
ssl_crt_path = '/etc/apache2/ssl/server.crt'
ssl_key_path = '/etc/apache2/ssl/server.key'
vhost_ip_address_needed = False
wsgi_socket_directive_needed = False
# Debian:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'debian_version'):
listen_directive_needed = False
ssl_pem_directive_needed = True
# RHEL/SLC:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'redhat-release'):
listen_directive_needed = False
ssl_crt_path = '/etc/pki/tls/certs/localhost.crt'
ssl_key_path = '/etc/pki/tls/private/localhost.key'
vhost_ip_address_needed = True
if os.popen('grep -c "CentOS.*[67]\." /etc/redhat-release').read().strip() == '1':
vhost_ip_address_needed = False
wsgi_socket_directive_needed = True
# maybe we are using non-standard ports?
vhost_site_url = conf.get('Invenio', 'CFG_SITE_URL').replace("http://", "")
if vhost_site_url.startswith("https://"):
## The installation is configured to require HTTPS for any connection
vhost_site_url = vhost_site_url.replace("https://", "")
vhost_site_url_port = '80'
vhost_site_secure_url = conf.get('Invenio', 'CFG_SITE_SECURE_URL').replace("https://", "")
vhost_site_secure_url_port = '443'
if ':' in vhost_site_url:
vhost_site_url, vhost_site_url_port = vhost_site_url.split(':', 1)
if ':' in vhost_site_secure_url:
vhost_site_secure_url, vhost_site_secure_url_port = vhost_site_secure_url.split(':', 1)
if vhost_site_url_port != '80' or vhost_site_secure_url_port != '443':
listen_directive_needed = True
## OK, let's create Apache vhost files:
if not os.path.exists(apache_conf_dir):
os.mkdir(apache_conf_dir)
apache_vhost_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost.conf'
apache_vhost_ssl_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost-ssl.conf'
apache_vhost_body = """\
AddDefaultCharset UTF-8
ServerSignature Off
ServerTokens Prod
NameVirtualHost %(vhost_ip_address)s:%(vhost_site_url_port)s
%(listen_directive)s
%(wsgi_socket_directive)s
WSGIPythonHome %(wsgi_python_home)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
DocumentRoot %(webdir)s
%(directory_www_directive)s
ErrorLog %(logdir)s/apache.err
LogLevel warn
LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing
CustomLog %(logdir)s/apache.log combined_with_timing
DirectoryIndex index.en.html index.html
Alias /static/ %(webdir)s/static/
Alias /img/ %(webdir)s/img/
Alias /css/ %(webdir)s/css/
Alias /js/ %(webdir)s/js/
Alias /flash/ %(webdir)s/flash/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /ckeditor/ %(webdir)s/ckeditor/
Alias /mediaelement/ %(webdir)s/mediaelement/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
WSGIDaemonProcess invenio processes=5 threads=1 display-name=%%{GROUP} inactivity-timeout=3600 maximum-requests=10000 %(wsgiuser)s
WSGIImportScript %(wsgidir)s/invenio.wsgi process-group=invenio application-group=%%{GLOBAL}
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
%(xsendfile_directive)s
%(directory_wsgi_directive)s
%(deflate_directive)s
""" % {'vhost_site_url_port': vhost_site_url_port,
'servername': vhost_site_url,
'serveralias': vhost_site_url.split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'wsgiuser': conf.get('Invenio', 'CFG_BIBSCHED_PROCESS_USER') and
'user='+conf.get('Invenio', 'CFG_BIBSCHED_PROCESS_USER'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address(conf) or '*',
'listen_directive': listen_directive_needed and 'Listen ' + vhost_site_url_port or \
'#Listen ' + vhost_site_url_port,
'wsgi_python_home': sys.prefix,
'wsgi_socket_directive': (wsgi_socket_directive_needed and \
'WSGISocketPrefix ' or '#WSGISocketPrefix ') + \
conf.get('Invenio', 'CFG_PREFIX') + os.sep + 'var' + os.sep + 'run',
'xsendfile_directive': xsendfile_directive,
'directory_www_directive': directory_www_directive,
'directory_wsgi_directive': directory_wsgi_directive,
'deflate_directive': deflate_directive,
}
apache_vhost_ssl_body = """\
ServerSignature Off
ServerTokens Prod
%(listen_directive)s
NameVirtualHost %(vhost_ip_address)s:%(vhost_site_secure_url_port)s
%(ssl_pem_directive)s
%(ssl_crt_directive)s
%(ssl_key_directive)s
%(ssl_protocol_directive)s
%(ssl_cipher_directive)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
SSLEngine on
DocumentRoot %(webdir)s
%(directory_www_directive)s
ErrorLog %(logdir)s/apache-ssl.err
LogLevel warn
LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing
CustomLog %(logdir)s/apache-ssl.log combined_with_timing
DirectoryIndex index.en.html index.html
Alias /static/ %(webdir)s/static/
Alias /img/ %(webdir)s/img/
Alias /css/ %(webdir)s/css/
Alias /js/ %(webdir)s/js/
Alias /flash/ %(webdir)s/flash/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /ckeditor/ %(webdir)s/ckeditor/
Alias /mediaelement/ %(webdir)s/mediaelement/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
RedirectMatch /sslredirect/(.*) http://$1
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
%(xsendfile_directive)s
%(directory_wsgi_directive)s
%(deflate_directive)s
%(shibboleth_directive)s
""" % {'vhost_site_secure_url_port': vhost_site_secure_url_port,
'servername': vhost_site_secure_url,
'serveralias': vhost_site_secure_url.split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'wsgidir' : os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address(conf) or '*',
'listen_directive' : listen_directive_needed and 'Listen ' + vhost_site_secure_url_port or \
'#Listen ' + vhost_site_secure_url_port,
'ssl_pem_directive': ssl_pem_directive_needed and \
'SSLCertificateFile %s' % ssl_pem_path or \
'#SSLCertificateFile %s' % ssl_pem_path,
'ssl_crt_directive': ssl_pem_directive_needed and \
'#SSLCertificateFile %s' % ssl_crt_path or \
'SSLCertificateFile %s' % ssl_crt_path,
'ssl_key_directive': ssl_pem_directive_needed and \
'#SSLCertificateKeyFile %s' % ssl_key_path or \
'SSLCertificateKeyFile %s' % ssl_key_path,
'ssl_protocol_directive': ssl_pem_directive_needed and \
'SSLProtocol all -SSLv2 -SSLv3' or \
'#SSLProtocol all -SSLv2 -SSLv3',
'ssl_cipher_directive': ssl_pem_directive_needed and \
'SSLCipherSuite HIGH:MEDIUM:!ADH' or \
'#SSLCipherSuite HIGH:MEDIUM:!ADH',
'xsendfile_directive': xsendfile_directive,
'directory_www_directive': directory_www_directive,
'directory_wsgi_directive': directory_wsgi_directive,
'deflate_directive': deflate_directive,
'shibboleth_directive': shibboleth_directive,
}
# write HTTP vhost snippet:
if os.path.exists(apache_vhost_file):
shutil.copy(apache_vhost_file,
apache_vhost_file + '.OLD')
fdesc = open(apache_vhost_file, 'w')
fdesc.write(apache_vhost_body)
fdesc.close()
print
print "Created file", apache_vhost_file
# write HTTPS vhost snippet:
vhost_ssl_created = False
if conf.get('Invenio', 'CFG_SITE_SECURE_URL').startswith("https://"):
if os.path.exists(apache_vhost_ssl_file):
shutil.copy(apache_vhost_ssl_file,
apache_vhost_ssl_file + '.OLD')
fdesc = open(apache_vhost_ssl_file, 'w')
fdesc.write(apache_vhost_ssl_body)
fdesc.close()
vhost_ssl_created = True
print "Created file", apache_vhost_ssl_file
print wrap_text_in_a_box("""\
Apache virtual host configuration file(s) for your Invenio site
was(were) created. Please check created file(s) and activate virtual
host(s). For example, you can put the following include statements in
your httpd.conf:\n
Include %s
%s
Please see the INSTALL file for more details.
""" % (apache_vhost_file, (vhost_ssl_created and 'Include ' or '#Include ') + apache_vhost_ssl_file))
print ">>> Apache conf files created."
def cli_cmd_get(conf, varname):
"""
Return value of VARNAME read from CONF files. Useful for
third-party programs to access values of conf options such as
CFG_PREFIX. Return None if VARNAME is not found.
"""
try:
if not varname:
raise Exception("ERROR: Please specify a configuration variable.")
varname = varname.lower()
# do not pay attention to section names yet:
all_options = {}
for section in conf.sections():
for option in conf.options(section):
all_options[option] = conf.get(section, option)
varvalue = all_options.get(varname, None)
if varvalue is None:
raise Exception()
print varvalue
except Exception, e:
if e.message:
print e.message
sys.exit(1)
def cli_cmd_list(conf):
"""
Print a list of all conf options and values from CONF.
"""
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
print option.upper(), '=', conf.get(section, option)
def _grep_version_from_executable(path_to_exec, version_regexp):
"""
Try to detect a program version by digging into its binary
PATH_TO_EXEC and looking for VERSION_REGEXP. Return program
version as a string. Return empty string if not succeeded.
"""
from invenio.shellutils import run_shell_command
exec_version = ""
if os.path.exists(path_to_exec):
dummy1, cmd2_out, dummy2 = run_shell_command("strings %s | grep %s",
(path_to_exec, version_regexp))
if cmd2_out:
for cmd2_out_line in cmd2_out.split("\n"):
if len(cmd2_out_line) > len(exec_version):
# the longest the better
exec_version = cmd2_out_line
return exec_version
_RE_APACHE_MAJOR_VERSION = re.compile(r"Apache/(\d+\.\d+)")
def guess_apache_24(apache_versions=None):
"""
Returns True if it looks like the system is running Apache 2.4 or later.
"""
if apache_versions is None:
apache_versions = detect_apache_version()
for apache_version in apache_versions:
g = _RE_APACHE_MAJOR_VERSION.search(apache_version)
if g:
try:
version = float(g.group(1))
except ValueError:
continue
if version >= 2.4:
return True
return False
def detect_apache_version():
"""
Try to detect Apache version by localizing httpd or apache
executables and grepping inside binaries. Return list of all
found Apache versions and paths. (For a given executable, the
returned format is 'apache_version [apache_path]'.) Return empty
list if no success.
"""
from invenio.shellutils import run_shell_command
out = []
dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache")
for apache in cmd_out.split("\n"):
apache_version = _grep_version_from_executable(apache, '^Apache\/')
if apache_version:
out.append("%s [%s]" % (apache_version, apache))
return out
def cli_cmd_detect_system_details(conf):
"""
Detect and print system details such as Apache/Python/MySQL
versions etc. Useful for debugging problems on various OS.
"""
import MySQLdb
print ">>> Going to detect system details..."
print "* Hostname: " + socket.gethostname()
print "* Invenio version: " + conf.get("Invenio", "CFG_VERSION")
print "* Python version: " + sys.version.replace("\n", " ")
print "* Apache version: " + ";\n ".join(detect_apache_version())
print "* MySQLdb version: " + MySQLdb.__version__
try:
from invenio.dbquery import run_sql
print "* MySQL version:"
for key, val in run_sql("SHOW VARIABLES LIKE 'version%'") + \
run_sql("SHOW VARIABLES LIKE 'charact%'") + \
run_sql("SHOW VARIABLES LIKE 'collat%'"):
if False:
print " - %s: %s" % (key, val)
elif key in ['version',
'character_set_client',
'character_set_connection',
'character_set_database',
'character_set_results',
'character_set_server',
'character_set_system',
'collation_connection',
'collation_database',
'collation_server']:
print " - %s: %s" % (key, val)
except ImportError:
print "* ERROR: cannot import dbquery"
print ">>> System details detected successfully."
def cli_cmd_upgrade(conf):
"""
Command for applying upgrades
"""
from invenio.inveniocfg_upgrader import cmd_upgrade
cmd_upgrade(conf)
def cli_cmd_upgrade_check(conf):
"""
Command for running pre-upgrade checks
"""
from invenio.inveniocfg_upgrader import cmd_upgrade_check
cmd_upgrade_check(conf)
def cli_cmd_upgrade_show_pending(conf):
"""
Command for showing upgrades ready to be applied
"""
from invenio.inveniocfg_upgrader import cmd_upgrade_show_pending
cmd_upgrade_show_pending(conf)
def cli_cmd_upgrade_show_applied(conf):
"""
Command for showing all upgrades already applied.
"""
from invenio.inveniocfg_upgrader import cmd_upgrade_show_applied
cmd_upgrade_show_applied(conf)
def cli_cmd_upgrade_create_release_recipe(conf, path):
"""
Create a new release upgrade recipe (for developers).
"""
from invenio.inveniocfg_upgrader import cmd_upgrade_create_release_recipe
cmd_upgrade_create_release_recipe(conf, path)
def cli_cmd_upgrade_create_standard_recipe(conf, path, depends_on=None,
release=False):
"""
Create a new upgrade recipe (for developers).
"""
from invenio.inveniocfg_upgrader import cmd_upgrade_create_standard_recipe
cmd_upgrade_create_standard_recipe(conf, path, depends_on=depends_on,
release=release)
def prepare_option_parser():
"""Parse the command line options."""
class InvenioOption(Option):
"""
Option class that implements the action 'store_append_const' which will
1) append to list in options.
2) take a value and store in options.
Useful for e.g. appending a const to an actions list, while also taking
an option value and storing it.
This ensures that we can run actions in the order they are given on the
command-line.
Python 2.4 compatibility note: *append_const* action is not available in
Python 2.4, so it is implemented here, together with the new action
*store_append_const*.
"""
ACTIONS = Option.ACTIONS + ("store_append_const", "append_const")
STORE_ACTIONS = Option.STORE_ACTIONS + ("store_append_const", "append_const")
TYPED_ACTIONS = Option.TYPED_ACTIONS + ("store_append_const", )
ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("store_append_const", )
CONST_ACTIONS = getattr(Option, 'CONST_ACTIONS', ()) + ("store_append_const", "append_const")
def take_action(self, action, dest, opt, value, values, parser):
if action == "store_append_const":
# Combination of 'store' and 'append_const' actions
values.ensure_value(dest, []).append(self.const)
value_dest = self.const.replace('-', '_')
setattr(values, value_dest, value)
elif action == "append_const" and not hasattr(Option, 'CONST_ACTIONS'):
values.ensure_value(dest, []).append(self.const)
else:
Option.take_action(self, action, dest, opt, value, values, parser)
def _check_const(self):
if self.action not in self.CONST_ACTIONS and self.const is not None:
raise OptionError(
"'const' must not be supplied for action %r" % self.action,
self)
CHECK_METHODS = [
Option._check_action,
Option._check_type,
Option._check_choice,
Option._check_dest,
_check_const,
Option._check_nargs,
Option._check_callback,
]
parser = OptionParser(option_class=InvenioOption, description="Invenio configuration and administration CLI tool", formatter=IndentedHelpFormatter(max_help_position=31))
parser.add_option("-V", "--version", action="store_true", help="print version number")
finish_options = OptionGroup(parser, "Options to finish your installation")
finish_options.add_option("", "--create-apache-conf", dest='actions', const='create-apache-conf', action="append_const", help="create Apache configuration files")
finish_options.add_option("", "--create-tables", dest='actions', const='create-tables', action="append_const", help="create DB tables for Invenio")
finish_options.add_option("", "--load-bibfield-conf", dest='actions', const='load-bibfield-conf', action="append_const", help="load bibfield configuration file")
finish_options.add_option("", "--load-webstat-conf", dest='actions', const='load-webstat-conf', action="append_const", help="load the WebStat configuration")
finish_options.add_option("", "--drop-tables", dest='actions', const='drop-tables', action="append_const", help="drop DB tables of Invenio")
finish_options.add_option("", "--check-openoffice", dest='actions', const='check-openoffice', action="append_const", help="check for correctly set up of openoffice temporary directory")
parser.add_option_group(finish_options)
demotest_options = OptionGroup(parser, "Options to set up and test a demo site")
demotest_options.add_option("", "--create-demo-site", dest='actions', const='create-demo-site', action="append_const", help="create demo site")
demotest_options.add_option("", "--load-demo-records", dest='actions', const='load-demo-records', action="append_const", help="load demo records")
demotest_options.add_option("", "--remove-demo-records", dest='actions', const='remove-demo-records', action="append_const", help="remove demo records, keeping demo site")
demotest_options.add_option("", "--drop-demo-site", dest='actions', const='drop-demo-site', action="append_const", help="drop demo site configurations too")
demotest_options.add_option("", "--run-unit-tests", dest='actions', const='run-unit-tests', action="append_const", help="run unit test suite (needs demo site)")
demotest_options.add_option("", "--run-js-unit-tests", dest='actions', const='run-js-unit-tests', action="append_const", help="run JS unit test suite (needs demo site)")
demotest_options.add_option("", "--run-regression-tests", dest='actions', const='run-regression-tests', action="append_const", help="run regression test suite (needs demo site)")
demotest_options.add_option("", "--run-web-tests", dest='actions', const='run-web-tests', action="append_const", help="run web tests in a browser (needs demo site, Firefox, Selenium IDE)")
parser.add_option_group(demotest_options)
config_options = OptionGroup(parser, "Options to update config files in situ")
config_options.add_option("", "--update-all", dest='actions', const='update-all', action="append_const", help="perform all the update options")
config_options.add_option("", "--update-config-py", dest='actions', const='update-config-py', action="append_const", help="update config.py file from invenio.conf file")
config_options.add_option("", "--update-dbquery-py", dest='actions', const='update-dbquery-py', action="append_const", help="update dbquery.py with DB credentials from invenio.conf")
config_options.add_option("", "--update-dbexec", dest='actions', const='update-dbexec', action="append_const", help="update dbexec with DB credentials from invenio.conf")
config_options.add_option("", "--update-bibconvert-tpl", dest='actions', const='update-bibconvert-tpl', action="append_const", help="update bibconvert templates with CFG_SITE_URL from invenio.conf")
config_options.add_option("", "--update-web-tests", dest='actions', const='update-web-tests', action="append_const", help="update web test cases with CFG_SITE_URL from invenio.conf")
parser.add_option_group(config_options)
reset_options = OptionGroup(parser, "Options to update DB tables")
reset_options.add_option("", "--reset-all", dest='actions', const='reset-all', action="append_const", help="perform all the reset options")
reset_options.add_option("", "--reset-sitename", dest='actions', const='reset-sitename', action="append_const", help="reset tables to take account of new CFG_SITE_NAME*")
reset_options.add_option("", "--reset-siteadminemail", dest='actions', const='reset-siteadminemail', action="append_const", help="reset tables to take account of new CFG_SITE_ADMIN_EMAIL")
reset_options.add_option("", "--reset-fieldnames", dest='actions', const='reset-fieldnames', action="append_const", help="reset tables to take account of new I18N names from PO files")
reset_options.add_option("", "--reset-recstruct-cache", dest='actions', const='reset-recstruct-cache', action="append_const", help="reset record structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
reset_options.add_option("", "--reset-recjson-cache", dest='actions', const='reset-recjson-cache', action="append_const", help="reset record json structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
parser.add_option_group(reset_options)
upgrade_options = OptionGroup(parser, "Options to upgrade your installation")
upgrade_options.add_option("", "--upgrade", dest='actions', const='upgrade', action="append_const", help="apply all pending upgrades")
upgrade_options.add_option("", "--upgrade-check", dest='actions', const='upgrade-check', action="append_const", help="run pre-upgrade checks for pending upgrades")
upgrade_options.add_option("", "--upgrade-show-pending", dest='actions', const='upgrade-show-pending', action="append_const", help="show pending upgrades")
upgrade_options.add_option("", "--upgrade-show-applied", dest='actions', const='upgrade-show-applied', action="append_const", help="show history of applied upgrades")
upgrade_options.add_option("", "--upgrade-create-standard-recipe", dest='actions', metavar='REPOSITORY[,DIR]', const='upgrade-create-standard-recipe', action="store_append_const", help="create a new standard upgrade recipe (for developers)")
upgrade_options.add_option("", "--upgrade-create-release-recipe", dest='actions', metavar='REPOSITORY[,DIR]', const='upgrade-create-release-recipe', action="store_append_const", help="create a new release upgrade recipe (for developers)")
parser.add_option_group(upgrade_options)
helper_options = OptionGroup(parser, "Options to help the work")
helper_options.add_option("", "--list", dest='actions', const='list', action="append_const", help="print names and values of all options from conf files")
helper_options.add_option("", "--get", dest='actions', const='get', action="store_append_const", metavar="OPTION", help="get value of a given option from conf files")
helper_options.add_option("", "--conf-dir", action="store", metavar="PATH", help="path to directory where invenio*.conf files are [optional]")
helper_options.add_option("", "--detect-system-details", dest='actions', const='detect-system-details', action="append_const", help="print system details such as Apache/Python/MySQL versions")
parser.add_option_group(helper_options)
parser.add_option('--yes-i-know', action='store_true', dest='yes-i-know', help='use with care!')
parser.add_option('-x', '--stop', action='store_true', dest='stop_on_error', help='When running tests, stop at first error')
return parser
def prepare_conf(options):
""" Read configuration files """
conf = ConfigParser()
confdir = getattr(options, 'conf_dir', None)
if confdir is None:
## try to detect path to conf dir (relative to this bin dir):
confdir = re.sub(r'/bin$', '/etc', sys.path[0])
if confdir and not os.path.exists(confdir):
raise Exception("ERROR: bad --conf-dir option value - directory does not exists.")
sys.exit(1)
## read conf files:
for conffile in [confdir + os.sep + 'invenio.conf',
confdir + os.sep + 'invenio-autotools.conf',
confdir + os.sep + 'invenio-local.conf', ]:
if os.path.exists(conffile):
conf.read(conffile)
else:
if not conffile.endswith("invenio-local.conf"):
# invenio-local.conf is optional, otherwise stop
raise Exception("ERROR: Badly guessed conf file location %s (Please use --conf-dir option.)" % conffile)
return conf
def main(*cmd_args):
"""Main entry point."""
# Allow easier testing
if not cmd_args:
cmd_args = sys.argv[1:]
# Parse arguments
parser = prepare_option_parser()
(options, dummy_args) = parser.parse_args(list(cmd_args))
if getattr(options, 'stop_on_error', False):
from invenio.testutils import wrap_failfast
wrap_failfast()
if getattr(options, 'version', False):
print_version()
else:
# Read configuration
try:
conf = prepare_conf(options)
except Exception, e:
print e
sys.exit(1)
## Decide what to do
actions = getattr(options, 'actions', None)
if not actions:
print """ERROR: Please specify a command. Please see '--help'."""
sys.exit(1)
for action in actions:
if action == 'get':
cli_cmd_get(conf, getattr(options, 'get', None))
elif action == 'list':
cli_cmd_list(conf)
elif action == 'detect-system-details':
cli_cmd_detect_system_details(conf)
elif action == 'create-tables':
cli_cmd_create_tables(conf)
elif action == 'load-webstat-conf':
cli_cmd_load_webstat_conf(conf)
elif action == 'drop-tables':
cli_cmd_drop_tables(conf)
elif action == 'check-openoffice':
cli_check_openoffice(conf)
elif action == 'load-bibfield-conf':
cli_cmd_load_bibfield_config(conf)
elif action == 'create-demo-site':
cli_cmd_create_demo_site(conf)
elif action == 'load-demo-records':
cli_cmd_load_demo_records(conf)
elif action == 'remove-demo-records':
cli_cmd_remove_demo_records(conf)
elif action == 'drop-demo-site':
cli_cmd_drop_demo_site(conf)
elif action == 'run-unit-tests':
cli_cmd_run_unit_tests(conf)
elif action == 'run-js-unit-tests':
cli_cmd_run_js_unit_tests(conf)
elif action == 'run-regression-tests':
cli_cmd_run_regression_tests(conf)
elif action == 'run-web-tests':
cli_cmd_run_web_tests(conf)
elif action == 'update-all':
cli_cmd_update_config_py(conf)
cli_cmd_update_dbquery_py(conf)
cli_cmd_update_dbexec(conf)
cli_cmd_update_bibconvert_tpl(conf)
cli_cmd_update_web_tests(conf)
elif action == 'update-config-py':
cli_cmd_update_config_py(conf)
elif action == 'update-dbquery-py':
cli_cmd_update_dbquery_py(conf)
elif action == 'update-dbexec':
cli_cmd_update_dbexec(conf)
elif action == 'update-bibconvert-tpl':
cli_cmd_update_bibconvert_tpl(conf)
elif action == 'update-web-tests':
cli_cmd_update_web_tests(conf)
elif action == 'reset-all':
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
cli_cmd_reset_recstruct_cache(conf)
elif action == 'reset-sitename':
cli_cmd_reset_sitename(conf)
elif action == 'reset-siteadminemail':
cli_cmd_reset_siteadminemail(conf)
elif action == 'reset-fieldnames':
cli_cmd_reset_fieldnames(conf)
elif action == 'reset-recstruct-cache':
cli_cmd_reset_recstruct_cache(conf)
elif action == 'reset-recjson-cache':
cli_cmd_reset_recjson_cache(conf)
elif action == 'create-apache-conf':
cli_cmd_create_apache_conf(conf)
elif action == 'upgrade':
cli_cmd_upgrade(conf)
elif action == 'upgrade-check':
cli_cmd_upgrade_check(conf)
elif action == 'upgrade-show-pending':
cli_cmd_upgrade_show_pending(conf)
elif action == 'upgrade-show-applied':
cli_cmd_upgrade_show_applied(conf)
elif action == 'upgrade-create-standard-recipe':
cli_cmd_upgrade_create_standard_recipe(conf, getattr(options, 'upgrade_create_standard_recipe', None))
elif action == 'upgrade-create-release-recipe':
cli_cmd_upgrade_create_release_recipe(conf, getattr(options, 'upgrade_create_release_recipe', None))
else:
print "ERROR: Unknown command", action
sys.exit(1)
if __name__ == '__main__':
main()
diff --git a/modules/webaccess/lib/access_control_firerole.py b/modules/webaccess/lib/access_control_firerole.py
index 01e1fb756..efd51a505 100644
--- a/modules/webaccess/lib/access_control_firerole.py
+++ b/modules/webaccess/lib/access_control_firerole.py
@@ -1,337 +1,344 @@
# This file is part of Invenio.
-# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2013 CERN.
+# Copyright (C) 2007, 2008, 2009, 2010, 2011, 2013, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Invenio Access Control FireRole."""
__revision__ = "$Id$"
__lastupdated__ = """$Date$"""
"""These functions are for realizing a firewall like role definition for extending
webaccess to connect user to roles using every infos about users.
"""
import re
import cPickle
from zlib import compress, decompress
import sys
import time
if sys.hexversion < 0x2040000:
# pylint: disable=W0622
from sets import Set as set
# pylint: enable=W0622
from invenio.access_control_config import InvenioWebAccessFireroleError
from invenio.dbquery import run_sql, blob_to_string
from invenio.config import CFG_CERN_SITE
from invenio.access_control_config import CFG_ACC_EMPTY_ROLE_DEFINITION_SRC, \
CFG_ACC_EMPTY_ROLE_DEFINITION_SER, CFG_ACC_EMPTY_ROLE_DEFINITION_OBJ
from invenio.errorlib import register_exception
# INTERFACE
def compile_role_definition(firerole_def_src):
""" Given a text in which every row contains a rule it returns the compiled
object definition.
Rules have the following syntax:
allow|deny [not] field {list of one or more (double)quoted string or regexp}
or allow|deny any
Every row may contain a # sign followed by a comment which are discarded.
Field could be any key contained in a user_info dictionary. If the key does
not exist in the dictionary, the rule is skipped.
The first rule which matches return.
"""
line = 0
ret = []
default_allow_p = False
if not firerole_def_src or not firerole_def_src.strip():
firerole_def_src = CFG_ACC_EMPTY_ROLE_DEFINITION_SRC
for row in firerole_def_src.split('\n'):
line += 1
row = row.strip()
if not row:
continue
clean_row = _no_comment_re.sub('', row)
if clean_row:
g = _any_rule_re.match(clean_row)
if g:
default_allow_p = g.group('command').lower() == 'allow'
break
g = _rule_re.match(clean_row)
if g:
allow_p = g.group('command').lower() == 'allow'
not_p = g.group('not') != None
field = g.group('field').lower()
# Renaming groups to group
for alias_item in _aliasTable:
if field in alias_item:
field = alias_item[0]
break
if field.startswith('precached_'):
raise InvenioWebAccessFireroleError("Error while compiling rule %s (line %s): %s is a reserved key and can not be used in FireRole rules!" % (row, line, field))
expressions = g.group('expression')+g.group('more_expressions')
expressions_list = []
for expr in _expressions_re.finditer(expressions):
expr = expr.group()
if field in ('from', 'until'):
try:
expressions_list.append((False, time.mktime(time.strptime(expr[1:-1], '%Y-%m-%d'))))
except Exception, msg:
raise InvenioWebAccessFireroleError("Syntax error while compiling rule %s (line %s): %s is not a valid date with format YYYY-MM-DD because %s!" % (row, line, expr, msg))
elif expr[0] == '/':
try:
expressions_list.append((True, re.compile(expr[1:-1], re.I)))
except Exception, msg:
raise InvenioWebAccessFireroleError("Syntax error while compiling rule %s (line %s): %s is not a valid re because %s!" % (row, line, expr, msg))
else:
if field == 'remote_ip' and '/' in expr[1:-1]:
try:
expressions_list.append((False, _ip_matcher_builder(expr[1:-1])))
except Exception, msg:
raise InvenioWebAccessFireroleError("Syntax error while compiling rule %s (line %s): %s is not a valid ip group because %s!" % (row, line, expr, msg))
else:
expressions_list.append((False, expr[1:-1]))
expressions_list = tuple(expressions_list)
if field in ('from', 'until'):
if len(expressions_list) != 1:
raise InvenioWebAccessFireroleError("Error when compiling rule %s (line %s): exactly one date is expected when using 'from' or 'until', but %s were found" % (row, line, len(expressions_list)))
if not_p:
raise InvenioWebAccessFireroleError("Error when compiling rule %s (line %s): 'not' is not allowed when using 'from' or 'until'" % (row, line))
ret.append((allow_p, not_p, field, expressions_list))
else:
raise InvenioWebAccessFireroleError("Syntax error while compiling rule %s (line %s): not a valid rule!" % (row, line))
return (default_allow_p, tuple(ret))
def repair_role_definitions():
""" Try to rebuild compiled serialized definitions from their respectives
sources. This is needed in case Python break back compatibility.
"""
definitions = run_sql("SELECT id, firerole_def_src FROM accROLE")
for role_id, firerole_def_src in definitions:
- run_sql("UPDATE accROLE SET firerole_def_ser=%s WHERE id=%s", (serialize(compile_role_definition(firerole_def_src)), role_id))
+ firerole_def_ser = serialize(compile_role_definition(firerole_def_src))
+ if firerole_def_ser:
+ run_sql("UPDATE accROLE SET firerole_def_ser=_binary %s WHERE id=%s", (firerole_def_ser, role_id))
+ else:
+ run_sql("UPDATE accROLE SET firerole_def_ser=%s WHERE id=%s", (firerole_def_ser, role_id))
def store_role_definition(role_id, firerole_def_ser, firerole_def_src):
""" Store a compiled serialized definition and its source in the database
alongside the role to which it belong.
@param role_id: the role_id
@param firerole_def_ser: the serialized compiled definition
@param firerole_def_src: the sources from which the definition was taken
"""
- run_sql("UPDATE accROLE SET firerole_def_ser=%s, firerole_def_src=%s WHERE id=%s", (firerole_def_ser, firerole_def_src, role_id))
+ if firerole_def_ser:
+ run_sql("UPDATE accROLE SET firerole_def_ser=_binary %s, firerole_def_src=%s WHERE id=%s", (firerole_def_ser, firerole_def_src, role_id))
+ else:
+ run_sql("UPDATE accROLE SET firerole_def_ser=%s, firerole_def_src=%s WHERE id=%s", (firerole_def_ser, firerole_def_src, role_id))
def load_role_definition(role_id):
""" Load the definition corresponding to a role. If the compiled definition
is corrupted it try to repairs definitions from their sources and try again
to return the definition.
@param role_id:
@return: a deserialized compiled role definition
"""
res = run_sql("SELECT firerole_def_ser FROM accROLE WHERE id=%s", (role_id, ), 1, run_on_slave=True)
if res:
try:
return deserialize(res[0][0])
except Exception:
## Something bad might have happened? (Update of Python?)
repair_role_definitions()
res = run_sql("SELECT firerole_def_ser FROM accROLE WHERE id=%s", (role_id, ), 1, run_on_slave=True)
if res:
return deserialize(res[0][0])
return CFG_ACC_EMPTY_ROLE_DEFINITION_OBJ
def acc_firerole_extract_emails(firerole_def_obj):
"""
Best effort function to extract all the possible email addresses
authorized by the given firerole.
"""
authorized_emails = set()
try:
default_allow_p, rules = firerole_def_obj
for (allow_p, not_p, field, expressions_list) in rules: # for every rule
if not_p:
continue
if field == 'group':
for reg_p, expr in expressions_list:
if reg_p:
continue
if CFG_CERN_SITE and expr.endswith(' [CERN]'):
authorized_emails.add(expr[:-len(' [CERN]')].lower().strip() + '@cern.ch')
emails = run_sql("SELECT user.email FROM usergroup JOIN user_usergroup ON usergroup.id=user_usergroup.id_usergroup JOIN user ON user.id=user_usergroup.id_user WHERE usergroup.name=%s", (expr, ))
for email in emails:
authorized_emails.add(email[0].lower().strip())
elif field == 'email':
for reg_p, expr in expressions_list:
if reg_p:
continue
authorized_emails.add(expr.lower().strip())
elif field == 'uid':
for reg_p, expr in expressions_list:
if reg_p:
continue
email = run_sql("SELECT email FROM user WHERE id=%s", (expr, ))
if email:
authorized_emails.add(email[0][0].lower().strip())
return authorized_emails
except Exception, msg:
raise InvenioWebAccessFireroleError, msg
def acc_firerole_check_user(user_info, firerole_def_obj):
""" Given a user_info dictionary, it matches the rules inside the deserializez
compiled definition in order to discover if the current user match the roles
corresponding to this definition.
@param user_info: a dict produced by collect_user_info which contains every
info about a user
@param firerole_def_obj: a compiled deserialized definition produced by
compile_role_defintion
@return: True if the user match the definition, False otherwise.
"""
try:
default_allow_p, rules = firerole_def_obj
for (allow_p, not_p, field, expressions_list) in rules: # for every rule
group_p = field == 'group' # Is it related to group?
ip_p = field == 'remote_ip' # Is it related to Ips?
until_p = field == 'until' # Is it related to dates?
from_p = field == 'from' # Idem.
next_expr_p = False # Silly flag to break 2 for cycles
if not user_info.has_key(field) and not from_p and not until_p:
continue
for reg_p, expr in expressions_list: # For every element in the rule
if group_p: # Special case: groups
if reg_p: # When it is a regexp
for group in user_info[field]: # iterate over every group
if expr.match(group): # if it matches
if not_p: # if must not match
next_expr_p = True # let's skip to next expr
break
else: # Ok!
return allow_p
if next_expr_p:
break # I said: let's skip to next rule ;-)
elif expr.lower() in [group.lower() for group in user_info[field]]: # Simple expression then just check for expr in groups
if not_p: # If expr is in groups then if must not match
break # let's skip to next expr
else: # Ok!
return allow_p
elif reg_p: # Not a group, then easier. If it's a regexp
if expr.match(user_info[field]): # if it matches
if not_p: # If must not match
break # Let's skip to next expr
else:
return allow_p # Ok!
elif ip_p and type(expr) == type(()): # If it's just a simple expression but an IP!
if _ipmatch(user_info['remote_ip'], expr): # Then if Ip matches
if not_p: # If must not match
break # let's skip to next expr
else:
return allow_p # ok!
elif until_p:
if time.time() <= expr:
if allow_p:
break
else:
return False
elif allow_p:
return False
else:
break
elif from_p:
if time.time() >= expr:
if allow_p:
break
else:
return False
elif allow_p:
return False
else:
break
elif expr.lower() == str(user_info[field]).lower(): # Finally the easiest one!!
if not_p: # ...
break
else: # ...
return allow_p # ...
if not_p and not next_expr_p: # Nothing has matched and we got not
return allow_p # Then the whole rule matched!
except Exception, msg:
register_exception(alert_admin=True)
raise InvenioWebAccessFireroleError, msg
return default_allow_p # By default we allow ;-) it'an OpenAccess project
def serialize(firerole_def_obj):
""" Serialize and compress a definition."""
if firerole_def_obj == CFG_ACC_EMPTY_ROLE_DEFINITION_OBJ:
return CFG_ACC_EMPTY_ROLE_DEFINITION_SER
elif firerole_def_obj:
return compress(cPickle.dumps(firerole_def_obj, -1))
else:
return CFG_ACC_EMPTY_ROLE_DEFINITION_SER
def deserialize(firerole_def_ser):
""" Deserialize and decompress a definition."""
if firerole_def_ser:
return cPickle.loads(decompress(blob_to_string(firerole_def_ser)))
else:
return CFG_ACC_EMPTY_ROLE_DEFINITION_OBJ
# IMPLEMENTATION
# Comment finder
_no_comment_re = re.compile(r'[\s]*(?allow|deny)[\s]+(?:(?Pnot)[\s]+)?(?P[\w]+)[\s]+(?P(?([\s]*,[\s]*((?allow|deny)[\s]+(any|all)[\s]*', re.I)
# Sub expression finder
_expressions_re = re.compile(r'(? group member ?
query_group_baskets = """
SELECT share_level
FROM user_usergroup AS ug LEFT JOIN usergroup_bskBASKET AS ub
ON ug.id_usergroup=ub.id_usergroup
WHERE ug.id_user=%s AND ub.id_bskBASKET=%s AND NOT(ub.share_level='NO') AND ug.user_status!=%s
"""
params_group_baskets = (int(uid), int(bskid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query_group_baskets, params_group_baskets)
group_index = None
if res:
try:
group_index = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(res[0][0])
except:
return None
# public basket ?
query_public_baskets = """
SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0 AND id_bskBASKET=%s
"""
public_index = None
res = run_sql(query_public_baskets, (int(bskid),))
if res:
try:
public_index = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(res[0][0])
except:
return None
if group_index or public_index:
if group_index > public_index:
return CFG_WEBBASKET_SHARE_LEVELS_ORDERED[group_index]
else:
return CFG_WEBBASKET_SHARE_LEVELS_ORDERED[public_index]
return None
########################### Personal baskets ##################################
def get_personal_baskets_info_for_topic(uid, topic):
"""Return information about every basket that belongs to the given user and topic."""
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s')
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner=%s
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
WHERE ubsk.id_user=%s
AND ubsk.topic=%s
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (uid, uid, topic)
res = run_sql(query, params)
return res
def get_all_user_personal_basket_ids_by_topic(uid):
"""For a given user return all their personal basket ids grouped by topic."""
query = """ SELECT ubsk.topic,
GROUP_CONCAT(bsk.id)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
GROUP BY ubsk.topic
ORDER BY ubsk.topic"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_personal_baskets_names(uid):
""" for a given user, returns every basket he is owner of
returns list of tuples: (bskid, bsk_name, topic)
"""
query = """
SELECT bsk.id,
bsk.name,
ubsk.topic
FROM user_bskBASKET ubsk JOIN bskBASKET bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
ORDER BY ubsk.topic
"""
params = (int(uid),)
return run_sql(query, params)
def get_basket_name(bskid):
"""return the name of a given basket"""
query = 'SELECT name FROM bskBASKET where id=%s'
res = run_sql(query, (int(bskid), ))
if res:
return res[0][0]
else:
return ''
def is_personal_basket_valid(uid, bskid):
"""Check if the basked (bskid) belongs to user (uid) and is valid."""
query = """ SELECT id
FROM bskBASKET
WHERE id=%s
AND id_owner=%s"""
params = (bskid, uid)
res = run_sql(query, params)
return res
def is_topic_valid(uid, topic):
"""Check if the topic defined by user (uid) exists."""
query = """ SELECT distinct(topic)
FROM user_bskBASKET
WHERE topic=%s
AND id_user=%s"""
params = (topic, uid)
res = run_sql(query, params)
return res
def get_basket_topic(uid, bskid):
"""Return the name of the topic this basket (bskid) belongs to."""
query = """ SELECT topic
FROM user_bskBASKET
WHERE id_bskBASKET=%s
AND id_user=%s"""
params = (bskid,uid)
res = run_sql(query, params)
return res
def get_personal_topics_infos(uid):
"""
Get the list of every topic user has defined,
and the number of baskets in each topic
@param uid: user id (int)
@return: a list of tuples (topic name, nb of baskets)
"""
query = """SELECT topic, count(b.id)
FROM user_bskBASKET ub JOIN bskBASKET b
ON ub.id_bskBASKET=b.id AND
b.id_owner=ub.id_user
WHERE ub.id_user=%s
GROUP BY topic
ORDER BY topic"""
uid = int(uid)
res = run_sql(query, (uid,))
return res
def get_basket_ids_and_names(bskids, limit=0):
"""For the given basket ids, return their ids and names,
ordered by basket name.
If 'limit' is greater than 0, limit the number of results returned."""
if not((type(bskids) is list) or (type(bskids) is tuple)):
bskids = [bskids]
query = """ SELECT bsk.id,
bsk.name
FROM bskBASKET AS bsk
WHERE %s
ORDER BY bsk.name
%s"""
sep = ' OR '
query %= (sep.join(['id=%s'] * len(bskids)), limit and 'LIMIT %i' % limit or '')
params = tuple(bskids)
res = run_sql(query, params)
return res
def rename_basket(bskid, new_name):
"""Rename basket to new_name"""
run_sql("UPDATE bskBASKET SET name=%s WHERE id=%s", (new_name, bskid))
def rename_topic(uid, old_topic, new_topic):
"""Rename topic to new_topic """
res = run_sql("UPDATE user_bskBASKET SET topic=%s WHERE id_user=%s AND topic=%s",
(new_topic, uid, old_topic))
return res
def move_baskets_to_topic(uid, bskids, new_topic):
"""Move given baskets to another topic"""
if not((type(bskids) is list) or (type(bskids) is tuple)):
bskids = [bskids]
query = "UPDATE user_bskBASKET SET topic=%s WHERE id_user=%s AND ("
query += ' OR '.join(['id_bskBASKET=%s'] * len(bskids))
query += ")"
params = (new_topic, uid) + tuple(bskids)
res = run_sql(query, params)
return res
def delete_basket(bskid):
"""Delete given basket."""
# TODO: check if any alerts are automaticly adding items to the given basket.
bskid = int(bskid)
query1 = "DELETE FROM bskBASKET WHERE id=%s"
res = run_sql(query1, (bskid,))
query2A = "SELECT id_bibrec_or_bskEXTREC FROM bskREC WHERE id_bskBASKET=%s"
local_and_external_ids = run_sql(query2A, (bskid,))
external_ids = [local_and_external_id[0] for local_and_external_id in \
local_and_external_ids if local_and_external_id[0]<0]
for external_id in external_ids:
delete_item(bskid=bskid, recid=external_id, update_date_modification=False)
query2B = "DELETE FROM bskREC WHERE id_bskBASKET=%s"
run_sql(query2B, (bskid,))
query3 = "DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET=%s"
run_sql(query3, (bskid,))
query4 = "DELETE FROM user_bskBASKET WHERE id_bskBASKET=%s"
run_sql(query4, (bskid,))
query5 = "DELETE FROM usergroup_bskBASKET WHERE id_bskBASKET=%s"
run_sql(query5, (bskid,))
query6 = "DELETE FROM user_query_basket WHERE id_basket=%s"
run_sql(query6, (bskid,))
return int(res)
def create_basket(uid, basket_name, topic):
"""Create new basket for given user in given topic"""
now = convert_datestruct_to_datetext(localtime())
id_bsk = run_sql("""INSERT INTO bskBASKET (id_owner, name, date_modification)
VALUES (%s, %s, %s)""",
(uid, basket_name, now))
run_sql("""INSERT INTO user_bskBASKET (id_user, id_bskBASKET, topic)
VALUES (%s, %s, %s)""",
(uid, id_bsk, topic))
return id_bsk
def get_all_items_in_user_personal_baskets(uid,
topic="",
format='hb'):
"""For the specified user, return all the items in their personal baskets,
grouped by basket if local or as a list if external.
If topic is set, return only that topic's items."""
if topic:
topic_clause = """AND ubsk.topic=%s"""
params_local = (uid, uid, topic)
params_external = (uid, uid, topic, format)
else:
topic_clause = ""
params_local = (uid, uid)
params_external = (uid, uid, format)
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
ubsk.topic,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%%s
%s
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET""" % (topic_clause,)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
ubsk.topic,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%%s
%s
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET""" % (topic_clause,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_personal_baskets_by_matching_notes(uid,
topic="",
p=""):
"""For the specified user, return all the items in their personal baskets
matching their notes' titles and bodies, grouped by basket.
If topic is set, return only that topic's items."""
p = p and '%' + p + '%' or '%'
if topic:
topic_clause = """AND ubsk.topic=%s"""
params = (uid, uid, topic, p, p)
else:
topic_clause = ""
params = (uid, uid, p, p)
query = """ SELECT notes.id_bskBASKET,
bsk.name,
ubsk.topic,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
AND bsk.id_owner=%%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=notes.id_bskBASKET
AND ubsk.id_user=%%s
%s
WHERE notes.title like %%s
OR notes.body like %%s
GROUP BY notes.id_bskBASKET""" % (topic_clause,)
res = run_sql(query, params)
return res
def get_all_user_topics(uid):
"""Return a list of the user's topics."""
query = """ SELECT ubsk.topic
FROM bskBASKET AS bsk
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user=bsk.id_owner
WHERE bsk.id_owner=%s
GROUP BY ubsk.topic"""
params = (uid,)
res = run_sql(query, params)
return res
########################## Actions on baskets #################################
def get_basket_record(bskid, recid, format='hb'):
"""get record recid in basket bskid
"""
if recid < 0:
rec_table = 'bskEXTREC'
format_table = 'bskEXTFMT'
id_field = 'id_bskEXTREC'
sign = '-'
else:
rec_table = 'bibrec'
format_table = 'bibfmt'
id_field = 'id_bibrec'
sign = ''
query = """
SELECT DATE_FORMAT(record.creation_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
DATE_FORMAT(record.modification_date, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
DATE_FORMAT(bskREC.date_added, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
user.nickname,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
fmt.value
FROM bskREC LEFT JOIN user
ON bskREC.id_user_who_added_item=user.id
LEFT JOIN bskRECORDCOMMENT cmt
ON bskREC.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
LEFT JOIN %(rec_table)s record
ON (%(sign)sbskREC.id_bibrec_or_bskEXTREC=record.id)
LEFT JOIN %(format_table)s fmt
ON (record.id=fmt.%(id_field)s)
WHERE bskREC.id_bskBASKET=%%s AND
bskREC.id_bibrec_or_bskEXTREC=%%s AND
fmt.format=%%s
GROUP BY bskREC.id_bibrec_or_bskEXTREC
""" % {'rec_table': rec_table,
'sign': sign,
'format_table': format_table,
'id_field':id_field}
params = (int(bskid), int(recid), format)
res = run_sql(query, params)
if res:
return __decompress_last(res[0])
return ()
def get_basket_content(bskid, format='hb'):
"""Get all records for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
extrec.collection_id,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%Y-%%m-%%d %%H:%%i:%%s'),
extern.value as ext_val,
intern.value as int_val,
rec.score
FROM bskREC AS rec
LEFT JOIN bskRECORDCOMMENT AS cmt
ON (rec.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
AND rec.id_bskBASKET=cmt.id_bskBASKET)
LEFT JOIN bskEXTFMT AS extern
ON (-rec.id_bibrec_or_bskEXTREC=extern.id_bskEXTREC
AND extern.format=%s)
LEFT JOIN bibfmt AS intern
ON (rec.id_bibrec_or_bskEXTREC=intern.id_bibrec
AND intern.format=%s)
LEFT JOIN bskEXTREC AS extrec
ON extrec.id=-rec.id_bibrec_or_bskEXTREC
WHERE rec.id_bskBASKET=%s
GROUP BY rec.id_bibrec_or_bskEXTREC
ORDER BY rec.score"""
params = (format, format, int(bskid))
res = run_sql(query, params)
if res:
query2 = "UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"
run_sql(query2, (int(bskid),))
return res
return ()
def get_basket_item(bskid, recid, format='hb'):
"""Get item (recid) for a given basket."""
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
extrec.collection_id,
count(cmt.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(cmt.date_creation), '%%Y-%%m-%%d %%H:%%i:%%s'),
extern.value as ext_val,
intern.value as int_val,
rec.score
FROM bskREC rec
LEFT JOIN bskRECORDCOMMENT cmt
ON (rec.id_bibrec_or_bskEXTREC=cmt.id_bibrec_or_bskEXTREC
AND
rec.id_bskBASKET=cmt.id_bskBASKET)
LEFT JOIN bskEXTFMT extern
ON (-rec.id_bibrec_or_bskEXTREC=extern.id_bskEXTREC
AND
extern.format=%s)
LEFT JOIN bibfmt intern
ON (rec.id_bibrec_or_bskEXTREC=intern.id_bibrec
AND
intern.format=%s)
LEFT JOIN bskEXTREC AS extrec
ON extrec.id=-rec.id_bibrec_or_bskEXTREC
WHERE rec.id_bskBASKET=%s
AND rec.id_bibrec_or_bskEXTREC=%s
GROUP BY rec.id_bibrec_or_bskEXTREC
ORDER BY rec.score"""
params = (format, format, bskid, recid)
res = run_sql(query, params)
if res:
queryU = """UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"""
paramsU = (bskid,)
run_sql(queryU, paramsU)
score = res[0][6]
query_previous = """SELECT id_bibrec_or_bskEXTREC
FROM bskREC
WHERE id_bskBASKET=%s
AND score<%s
ORDER BY score DESC
LIMIT 1"""
params_previous = (bskid, score)
res_previous = run_sql(query_previous, params_previous)
query_next = """SELECT id_bibrec_or_bskEXTREC
FROM bskREC
WHERE id_bskBASKET=%s
AND score>%s
ORDER BY score ASC
LIMIT 1"""
params_next = (bskid, score)
res_next = run_sql(query_next, params_next)
query_index = """ SELECT COUNT(id_bibrec_or_bskEXTREC)
FROM bskREC
WHERE id_bskBASKET=%s
AND score<=%s
ORDER BY score"""
params_index = (bskid, score)
res_index = run_sql(query_index, params_index)
res_index = __wash_sql_count(res_index)
return (res[0], res_previous and res_previous[0][0] or 0, res_next and res_next[0][0] or 0, res_index)
else:
return ()
def get_basket_item_title_and_URL(recid):
"""
Retrieves the title and URL for the specified item in the specified basket.
@param bskid: The basked id
@type bskid: int
@param recid: The record (item) id
@type recid: int
@return: A tuple containing the title as a sting and the URL as a string.
"""
if recid > 0:
# This is a local record, we can easily retrieve the title using the
# search engine's get_fieldvalues function and the MARC field and tag.
title_list = get_fieldvalues(recid, '245___')
# Check if the main title is always the first element in the list
if title_list:
title = title_list[0]
else:
title = ""
url = '%s/record/%i' % (CFG_SITE_URL, recid)
elif recid < 0:
# This is an external record or item, use
title = "This is an external record or item."
url = '%s' % (CFG_SITE_URL,)
query = """ SELECT rec.collection_id,
rec.original_url,
fmt.value
FROM bskEXTREC as rec,
bskEXTFMT as fmt
WHERE rec.id=%s
AND fmt.id_bskEXTREC=%s
AND fmt.format='hb'"""
params = (-recid, -recid)
result = run_sql(query, params)
if result:
item = __decompress_last(result[0])
collection = item[0]
url = item[1]
hb = item[2]
if collection == 0:
# This is an external item
title = hb.split('\n',1)[0]
elif collection > 0:
# This is an external record from a hosted collection
title = hb.split('',1)[0].split('')[-1]
return (title, url)
def share_basket_with_group(bskid, group_id,
share_level=CFG_WEBBASKET_SHARE_LEVELS['READITM']):
""" Share basket bskid with group group_id with given share_level
@param share_level: see CFG_WEBBASKET_SHARE_LEVELS in webbasket_config
"""
now = convert_datestruct_to_datetext(localtime())
run_sql("""REPLACE INTO usergroup_bskBASKET
(id_usergroup, id_bskBASKET, date_shared, share_level)
VALUES (%s,%s,%s,%s)""",
(group_id, bskid, now, str(share_level)))
def update_rights(bskid, group_rights):
"""update rights (permissions) for groups.
@param bskid: basket id
@param group_rights: dictionary of {group id: new rights}
"""
now = convert_datestruct_to_datetext(localtime())
query1 = """REPLACE INTO usergroup_bskBASKET
(id_usergroup, id_bskBASKET, date_shared, share_level)
VALUES """ + \
', '.join(["(%s, %s, %s, %s)"] * len(group_rights.items()))
params = ()
for (group_id, share_level) in group_rights.items():
params += (int(group_id), int(bskid), now, str(share_level))
run_sql(query1, params)
query2 = """DELETE FROM usergroup_bskBASKET WHERE share_level='NO'"""
run_sql(query2)
def move_item(bskid, recid, direction):
"""Change score of an item in a basket"""
bskid = int(bskid)
query1 = """SELECT id_bibrec_or_bskEXTREC,
score
FROM bskREC
WHERE id_bskBASKET=%s
ORDER BY score, date_added"""
items = run_sql(query1, (bskid,))
(recids, scores) = zip(*items)
(recids, scores) = (list(recids), list(scores))
if len(recids) and recid in recids:
current_index = recids.index(recid)
if direction == CFG_WEBBASKET_ACTIONS['UP']:
switch_index = 0
if current_index != 0:
switch_index = current_index -1
else:
switch_index = len(recids) - 1
if current_index != len(recids)-1:
switch_index = current_index + 1
query2 = """UPDATE bskREC
SET score=%s
WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"""
res1 = run_sql(query2, (scores[switch_index], bskid, recids[current_index]))
res2 = run_sql(query2, (scores[current_index], bskid, recids[switch_index]))
if res1 and res2:
now = convert_datestruct_to_datetext(localtime())
query3 = "UPDATE bskBASKET SET date_modification=%s WHERE id=%s"
params3 = (now, int(bskid))
run_sql(query3, params3)
def delete_item(bskid, recid, update_date_modification=True):
"""Remove item recid from basket bskid"""
if recid < 0:
query0A = "select count(id_bskBASKET) from bskREC where id_bibrec_or_bskEXTREC=%s" % (int(recid))
ncopies = run_sql(query0A)
if ncopies and ncopies[0][0]<=1:
# uncomment the following 5 lines and comment the following 2 to delete cached records
# only for external sources and not for external records
#query0B = "SELECT collection_id FROM bskEXTREC WHERE id=%s" % (-int(recid))
#colid = run_sql(query0B)
#if colid and colid[0][0]==0:
#query0C = "DELETE from bskEXTFMT WHERE id_bskEXTREC=%s" % (-int(recid))
#run_sql(query0C)
# the following two lines delete cached external records. We could keep them if we find
# a way to reuse them in case the external records are added again in the future.
query0D = "DELETE from bskEXTFMT WHERE id_bskEXTREC=%s" % (-int(recid))
run_sql(query0D)
query0E = "DELETE from bskEXTREC WHERE id=%s" % (-int(recid))
run_sql(query0E)
query_notes = "DELETE FROM bskRECORDCOMMENT WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"
run_sql(query_notes, (bskid, recid,))
query1 = "DELETE from bskREC WHERE id_bskBASKET=%s AND id_bibrec_or_bskEXTREC=%s"
params1 = (int(bskid), int(recid))
res = run_sql(query1, params1)
if update_date_modification and res:
now = convert_datestruct_to_datetext(localtime())
query2 = "UPDATE bskBASKET SET date_modification=%s WHERE id=%s"
params2 = (now, int(bskid))
run_sql(query2, params2)
return res
def add_to_basket(uid,
recids=[],
colid=0,
bskid=0,
es_title="",
es_desc="",
es_url=""):
"""Add items (recids) basket (bskid)."""
if (recids or (colid == -1 and es_title and es_desc and es_url)) and bskid > 0:
query_max_score = """ SELECT MAX(score)
FROM bskREC
WHERE id_bskBASKET=%s"""
params_max_score = (bskid,)
res_max_score = run_sql(query_max_score, params_max_score)
max_score = __wash_sql_count(res_max_score)
if not max_score:
# max_score == None actually means that the basket doesn't exist.
# Maybe we should return 0 and inform the admin?
max_score = 1
if colid > 0:
query_existing = """ SELECT id,
external_id
FROM bskEXTREC
WHERE %s
AND collection_id=%s"""
sep_or = ' OR '
query_existing %= (sep_or.join(['external_id=%s'] * len(recids)), colid)
params_existing = tuple(recids)
res_existing = run_sql(query_existing, params_existing)
existing_recids = [int(external_ids_couple[1]) for external_ids_couple in res_existing]
existing_ids = [int(ids[0]) for ids in res_existing]
new_recids = [recid for recid in recids if int(recid) not in existing_recids]
# sets approach
#existing_recids = [ids[1] for ids in res_existing]
#new_recids = list(set(recids)-set(existing_recids))
if new_recids:
query_new = """ INSERT INTO bskEXTREC (external_id,
collection_id,
creation_date,
modification_date)
VALUES """
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s)"] * len(new_recids)
query_new += ', '.join(records)
params_new = ()
for new_recid in new_recids:
params_new += (int(new_recid), colid, now, now)
res_new = run_sql(query_new, params_new)
recids = [-int(recid) for recid in existing_ids]
recids.extend(range(-res_new,-(res_new+len(new_recids)),-1))
else:
recids = [-int(recid) for recid in existing_ids]
elif colid < 0:
query_external = """INSERT INTO bskEXTREC (collection_id,
original_url,
creation_date,
modification_date)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
params_external = (colid, es_url, now, now)
res_external = run_sql(query_external, params_external)
recids = [-res_external]
store_external_source(res_external, es_title, es_desc, es_url, 'xm')
store_external_source(res_external, es_title, es_desc, es_url, 'hb')
query_insert = """ INSERT IGNORE INTO bskREC
(id_bibrec_or_bskEXTREC,
id_bskBASKET,
id_user_who_added_item,
date_added,
score)
VALUES """
if colid == 0 or (colid > 0 and not new_recids):
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s, %s)"] * len(recids)
query_insert += ', '.join(records)
params_insert = ()
i = 1
for recid in recids:
params_insert += (recid, bskid, uid, now, max_score + i)
i += 1
run_sql(query_insert, params_insert)
query_update = """ UPDATE bskBASKET
SET date_modification=%s
WHERE id=%s"""
params_update = (now, bskid)
run_sql(query_update, params_update)
return recids
return 0
def move_to_basket(uid,
recids=None,
old_bskid=0,
new_bskid=0,
update_date_modification=True):
""" Move items (recids) from basket (old_bskid) to basket (new_bskid) """
if (recids is not None) and len(recids) > 0:
moved_recids = []
for recid in recids:
# Prevent duplication of items
query = """ SELECT '1'
FROM bskREC
WHERE id_bskBASKET=%s
AND
id_bibrec_or_bskEXTREC=%s
"""
params = (int(new_bskid), int(recid))
res = run_sql(query, params)
if len(res) == 0:
# Change the item's pointer to basket
query = """ UPDATE bskREC
SET id_bskBASKET=%s,
id_user_who_added_item=%s
WHERE id_bskBASKET=%s
AND id_bibrec_or_bskEXTREC=%s
"""
params = (int(new_bskid), int(uid), int(old_bskid), int(recid))
res = run_sql(query, params)
moved_recids.append(int(recid))
# Update 'modification date'
if len(moved_recids) > 0 and update_date_modification:
now = convert_datestruct_to_datetext(localtime())
query = "UPDATE bskBASKET SET date_modification=%s WHERE id=%s"
params = (now, int(old_bskid))
run_sql(query, params)
params = (now, int(new_bskid))
run_sql(query, params)
return moved_recids
def add_to_many_baskets(uid, recids=[], colid=0, bskids=[], es_title="", es_desc="", es_url=""):
"""Add items recids to every basket in bskids list."""
if (len(recids) or colid == -1) and len(bskids):
query1 = """SELECT id_bskBASKET,
max(score)
FROM bskREC
WHERE %s
GROUP BY id_bskBASKET"""
bskids = [bskid for bskid in bskids if int(bskid) >= 0]
sep_or = ' OR '
query1 %= sep_or.join(['id_bskBASKET=%s'] * len(bskids))
bsks = dict.fromkeys(bskids, 0)
params = tuple(bskids)
bsks.update(dict(run_sql(query1, params)))
if colid > 0:
query2A = """SELECT id,
external_id
FROM bskEXTREC
WHERE %s
AND collection_id=%s"""
query2A %= (sep_or.join(['external_id=%s'] * len(recids)), colid)
params2A = tuple(recids)
res2A = run_sql(query2A, params2A)
existing_recids = [int(external_ids_couple[1]) for external_ids_couple in res2A]
existing_ids = [int(ids[0]) for ids in res2A]
new_recids = [recid for recid in recids if int(recid) not in existing_recids]
# sets approach
#existing_recids = [ids[1] for ids in res2A]
#new_recids = list(set(recids)-set(existing_recids))
if new_recids:
query2B = """INSERT
INTO bskEXTREC
(external_id,
collection_id,
creation_date,
modification_date)
VALUES """
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s)"] * len(new_recids)
query2B += ', '.join(records)
params2B = ()
for new_recid in new_recids:
params2B += (int(new_recid), colid, now, now)
res = run_sql(query2B, params2B)
recids = [-int(recid) for recid in existing_ids]
recids.extend(range(-res,-(res+len(new_recids)),-1))
else:
recids = [-int(recid) for recid in existing_ids]
elif colid < 0:
query2C = """INSERT
INTO bskEXTREC
(collection_id,
original_url,
creation_date,
modification_date)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
params = (colid, es_url, now, now)
res = run_sql(query2C, params)
recids = [-res]
store_external_source(res, es_title, es_desc, es_url, 'xm')
store_external_source(res, es_title, es_desc, es_url, 'hb')
query2 = """INSERT IGNORE
INTO bskREC
(id_bibrec_or_bskEXTREC,
id_bskBASKET,
id_user_who_added_item,
date_added,
score)
VALUES """
if colid == 0 or (colid > 0 and not new_recids):
now = convert_datestruct_to_datetext(localtime())
records = ["(%s, %s, %s, %s, %s)"] * (len(recids) * len(bsks.items()))
query2 += ', '.join(records)
params = ()
for (bskid, max_score) in bsks.items():
i = 1
for recid in recids:
params += (int(recid), int(bskid), int(uid), now, int(max_score) + i)
i += 1
run_sql(query2, params)
query3 = """UPDATE bskBASKET
SET date_modification=%s
WHERE """
query3 += sep_or.join(["id=%s"] * len(bskids))
params = (now,) + tuple(bskids)
run_sql(query3, params)
return len(bskids)
return 0
def get_external_records_by_collection(recids):
"""Get the selected recids, both local and external, grouped by collection."""
if recids:
query = """ SELECT GROUP_CONCAT(id),
GROUP_CONCAT(external_id),
collection_id
FROM bskEXTREC
WHERE %s
GROUP BY collection_id"""
recids = [-recid for recid in recids]
sep_or = ' OR '
query %= sep_or.join(['id=%s'] * len(recids))
params = tuple(recids)
res = run_sql(query,params)
return res
return 0
def get_external_records(recids, of="hb"):
"""Get formatted external records from the database."""
if recids:
query = """ SELECT rec.collection_id,
fmt.id_bskEXTREC,
fmt.value
FROM bskEXTFMT AS fmt
JOIN bskEXTREC AS rec
ON rec.id=fmt.id_bskEXTREC
WHERE format=%%s
AND ( %s )"""
recids = [-recid for recid in recids]
sep_or = ' OR '
query %= sep_or.join(['id_bskEXTREC=%s'] * len(recids))
params = [of]
params.extend(recids)
params = tuple(params)
res = run_sql(query,params)
return res
return ()
def store_external_records(records, of="hb"):
"""Store formatted external records to the database."""
if records:
query = """INSERT
INTO bskEXTFMT
(id_bskEXTREC,
format,
last_updated,
value)
VALUES """
now = convert_datestruct_to_datetext(localtime())
formatted_records = ["(%s, %s, %s, %s)"] * len(records)
query += ', '.join(formatted_records)
params = ()
for record in records:
params += (record[0], of, now, compress(record[1]))
run_sql(query,params)
def store_external_urls(ids_urls):
"""Store original urls for external records to the database."""
#for id_url in ids_urls.iteritems():
for id_url in ids_urls:
query = """UPDATE
bskEXTREC
SET original_url=%s
WHERE id=%s"""
params = (id_url[1], id_url[0])
run_sql(query,params)
def store_external_source(es_id, es_title, es_desc, es_url, of="hb"):
"""Store formatted external sources to the database."""
if es_id and es_title and es_desc:
query = """INSERT INTO bskEXTFMT
(id_bskEXTREC,
format,
last_updated,
value)
VALUES (%s, %s, %s, %s)"""
now = convert_datestruct_to_datetext(localtime())
value = create_pseudo_record(es_title, es_desc, es_url, of)
params = (es_id, of, now, compress(value))
run_sql(query,params)
def get_external_colid_and_url(recid):
"""Get the collection id and original url for an external record."""
if recid:
query = """SELECT
collection_id,
original_url
FROM bskEXTREC
WHERE id=%s"""
params = (-recid,)
res = run_sql(query,params)
if res:
return res
else:
return 0
############################ Group baskets ####################################
def get_group_baskets_info_for_group(grpid):
"""Return information about every basket that belongs to the given group,
provided the user is its manager or a member of it."""
if not grpid:
return ()
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
COUNT(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level,
bsk.id_owner
FROM usergroup_bskBASKET AS ugbsk
JOIN bskBASKET AS bsk
ON bsk.id=ugbsk.id_bskBASKET
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
WHERE ugbsk.id_usergroup=%s
AND ugbsk.share_level!='NO'
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (grpid,)
res = run_sql(query, params)
return res
def get_group_name(gid):
"""Given its id return the group's name."""
query = """ SELECT name
FROM usergroup
WHERE id=%s"""
params = (gid,)
res = run_sql(query, params)
return res
def get_all_user_group_basket_ids_by_group(uid):
"""For a given user return all their group basket ids grouped by group."""
query = """ SELECT ug.id,
ug.name,
GROUP_CONCAT(ugbsk.id_bskBASKET)
FROM usergroup AS ug
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=ug.id
JOIN bskBASKET AS bsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON ug.id=uug.id_usergroup
AND uug.id_user=%s
GROUP BY ug.name
ORDER BY ug.name"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_user_group_basket_ids_by_group_with_add_rights(uid):
"""For a given user return all their group basket ids grouped by group.
Return only the basket ids to which it is allowed to add records."""
query = """ SELECT ug.name,
GROUP_CONCAT(ugbsk.id_bskBASKET)
FROM usergroup AS ug
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=ug.id
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
AND ugbsk.share_level!='RC'
AND ugbsk.share_level!='AC'
JOIN bskBASKET AS bsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON ug.id=uug.id_usergroup
AND uug.id_user=%s
GROUP BY ug.name
ORDER BY ug.name"""
params = (uid,)
res = run_sql(query, params)
return res
def get_all_group_baskets_names(uid,
min_rights=CFG_WEBBASKET_SHARE_LEVELS['ADDCMT']):
"""For a given user returns every group baskets in which he can
return a list of tuples: (bskid, bsk_name, group_name)."""
# TODO: This function is no longer used. Delete if necessary.
uid = int(uid)
try:
min_rights_num = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(min_rights)
except ValueError:
return ()
groups = get_groups_user_member_of(uid)
if groups:
where_clause = '('
where_clause += " OR ".join(["ugbsk.id_usergroup=%s"] * len(groups))
where_clause += ') AND ('
where_clause += " OR ".join(["ugbsk.share_level=%s"] * len(CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:]))
where_clause += ")"
query = """
SELECT bsk.id,
bsk.name,
ug.name
FROM usergroup ug JOIN usergroup_bskBASKET ugbsk
ON ug.id=ugbsk.id_usergroup
JOIN bskBASKET bsk
ON bsk.id=ugbsk.id_bskBASKET
WHERE %s AND NOT(ugbsk.share_level='NO')
ORDER BY ug.name""" % where_clause
params = tuple([group_id for (group_id, dummy) in groups])
params += tuple(CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:])
return run_sql(query, params)
return ()
def is_shared_to(bskids):
"""For each bskid in bskids get id of one of its group. Used to
make distinction between private basket (no group), 'world' basket
(0) or group basket (any int > 0)
"""
if not((type(bskids) == list) or (type(bskids) == tuple)):
bskids = [bskids]
query = """SELECT b.id,
min(u.id_usergroup)
FROM
bskBASKET b LEFT JOIN usergroup_bskBASKET u
ON (b.id=u.id_bskBASKET) """
if len(bskids) != 0:
query += " WHERE "
query += " OR ".join(['b.id=%s'] * len(bskids))
query += " GROUP BY b.id"
params = tuple(bskids)
res = run_sql(query, params)
if res:
return res
return ()
def get_basket_share_level(bskid):
"""Get the minimum share level of the basket (bskid).
Returns:
None for personal baskets
positive integet for group baskets
0 for public baskets
Will return 0 if the basket is both group and publicly shared."""
query = """ SELECT MIN(ugbsk.id_usergroup)
FROM bskBASKET AS bsk
LEFT JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=bsk.id
WHERE bsk.id=%s
GROUP BY bsk.id"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_all_items_in_user_group_baskets(uid,
group=0,
format='hb'):
"""For the specified user, return all the items in their group baskets,
grouped by basket if local or as a list if external.
If group is set, return only that group's items."""
if group:
group_clause = """AND ugbsk.id_usergroup=%s"""
params_local = (group, uid)
params_external = (group, uid, format)
else:
group_clause = ""
params_local = (uid,)
params_external = (uid, format)
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
ugbsk.share_level,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET""" % (group_clause,)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
ugbsk.share_level,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET""" % (group_clause,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_group_baskets_by_matching_notes(uid,
group=0,
p=""):
"""For the specified user, return all the items in group personal baskets
matching their notes' titles and bodies, grouped by basket.
If topic is set, return only that topic's items."""
p = p and '%' + p + '%' or '%'
if group:
group_clause = """AND ugbsk.id_usergroup=%s"""
params = (group, uid, p, p)
else:
group_clause = ""
params = (uid, p, p)
query = """ SELECT notes.id_bskBASKET,
bsk.name,
uug.id_usergroup,
ug.name,
ugbsk.share_level,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.share_level IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
%s
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%%s
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
WHERE notes.title like %%s
OR notes.body like %%s
GROUP BY notes.id_bskBASKET""" % (group_clause,)
res = run_sql(query, params)
return res
def is_group_basket_valid(uid, bskid):
"""Check if the basked (bskid) belongs to one of the groups the user (uid)
is a member of and is valid."""
query = """ SELECT id
FROM bskBASKET AS bsk
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=bsk.id
JOIN user_usergroup AS uug
ON uug.id_usergroup=ugbsk.id_usergroup
AND uug.id_user=%s
WHERE id=%s"""
params = (uid, bskid)
res = run_sql(query, params)
return res
def is_group_valid(uid, group):
"""Check if the group exists and the user is a member or manager."""
query = """ SELECT id_usergroup
FROM user_usergroup
WHERE id_usergroup=%s
AND id_user=%s"""
params = (group, uid)
res = run_sql(query, params)
return res
def get_all_user_groups(uid):
"""Return a list of the groups the user is a member of or manages."""
query = """ SELECT ug.id,
ug.name
FROM user_usergroup AS uug
JOIN usergroup AS ug
ON ug.id=uug.id_usergroup
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=uug.id_usergroup
WHERE uug.id_user=%s
GROUP BY uug.id_usergroup"""
params = (uid,)
res = run_sql(query, params)
return res
########################## External baskets ###################################
def get_external_baskets_infos(uid):
"""Get general informations about every external basket user uid has subscribed to."""
query = """
SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level
FROM bskBASKET bsk JOIN user_bskBASKET ubsk
ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s)
LEFT JOIN bskREC rec
ON (bsk.id=rec.id_bskBASKET)
LEFT JOIN usergroup_bskBASKET ugbsk
ON (ugbsk.id_bskBASKET=bsk.id AND ugbsk.id_usergroup=0)
WHERE bsk.id_owner!=%s
GROUP BY bsk.id
"""
uid = int(uid)
params = (uid, uid)
res = run_sql(query, params)
if res:
return res
return ()
def get_external_basket_info(bskid):
""""""
query = """ SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
DATE_FORMAT(max(rec.date_added), '%%Y-%%m-%%d %%H:%%i:%%s'),
ugbsk.share_level
FROM bskBASKET AS bsk
LEFT JOIN bskREC AS rec
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON bsk.id=ugbsk.id_bskBASKET
AND ugbsk.id_usergroup=0
WHERE id=%s"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_all_external_basket_ids_and_names(uid):
"""For a given user return all their external baskets
(in tuples: (id, name, number_of_records))."""
query = """ SELECT bsk.id,
bsk.name,
count(rec.id_bibrec_or_bskEXTREC),
ugbsk.id_usergroup
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_user!=bsk.id_owner
LEFT JOIN bskREC AS rec
ON ubsk.id_bskBASKET=rec.id_bskBASKET
LEFT JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_usergroup=0
AND ugbsk.id_bskBASKET=bsk.id
WHERE ubsk.id_user=%s
GROUP BY bsk.id
ORDER BY bsk.name"""
params = (uid,)
res = run_sql(query, params)
return res
def count_external_baskets(uid):
"""Returns the number of external baskets the user is subscribed to."""
query = """ SELECT COUNT(ubsk.id_bskBASKET)
FROM user_bskBASKET ubsk
LEFT JOIN bskBASKET bsk
ON (bsk.id=ubsk.id_bskBASKET AND ubsk.id_user=%s)
WHERE bsk.id_owner!=%s"""
params = (int(uid), int(uid))
res = run_sql(query, params)
return __wash_sql_count(res)
def get_all_external_baskets_names(uid,
min_rights=CFG_WEBBASKET_SHARE_LEVELS['ADDCMT']):
""" for a given user returns every basket which he has subscribed to and in which
he can
return a list of tuples: (bskid, bsk_name)
"""
uid = int(uid)
try:
min_rights_num = CFG_WEBBASKET_SHARE_LEVELS_ORDERED.index(min_rights)
except ValueError:
return ()
where_clause = ' AND ('
for right in CFG_WEBBASKET_SHARE_LEVELS_ORDERED[min_rights_num:-1]:
where_clause += "ugbsk.share_level = '%s' OR " % right
where_clause += "ugbsk.share_level = '%s')" % CFG_WEBBASKET_SHARE_LEVELS_ORDERED[-1]
query = """
SELECT bsk.id,
bsk.name
FROM bskBASKET bsk JOIN usergroup_bskBASKET ugbsk
ON bsk.id=ugbsk.id_bskBASKET
JOIN user_bskBASKET ubsk
ON ubsk.id_bskBASKET=bsk.id
WHERE ugbsk.id_usergroup=0 AND
ubsk.id_user=%s AND
NOT(bsk.id_owner=%s) AND
NOT(ugbsk.share_level='NO')
""" + where_clause
params = (uid, uid)
return run_sql(query, params)
def get_all_items_in_user_public_baskets(uid,
format='hb'):
"""For the specified user, return all the items in the public baskets they
are subscribed to, grouped by basket if local or as a list if external."""
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
ugbsk.share_level,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET"""
params_local = (uid, uid)
res_local = run_sql(query_local, params_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
ugbsk.share_level,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=rec.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
params_external = (uid, uid, format)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_user_public_baskets_by_matching_notes(uid,
p=""):
"""For the specified user, return all the items in the public baskets they
are subscribed to, matching their notes' titles and bodies,
grouped by basket"""
p = p and '%' + p + '%' or '%'
query = """ SELECT notes.id_bskBASKET,
bsk.name,
ugbsk.share_level,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
AND bsk.id_owner!=%s
JOIN user_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=notes.id_bskBASKET
AND ubsk.id_user=%s
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.id_usergroup=0
AND ugbsk.share_level IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
WHERE notes.title like %s
OR notes.body like %s
GROUP BY notes.id_bskBASKET"""
params = (uid, uid, p, p)
res = run_sql(query, params)
return res
def get_all_items_in_all_public_baskets(format='hb'):
"""Return all the items in all the public baskets,
grouped by basket if local or as a list if external."""
query_local = """
SELECT rec.id_bskBASKET,
bsk.name,
ugbsk.share_level,
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC)
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
WHERE rec.id_bibrec_or_bskEXTREC > 0
GROUP BY rec.id_bskBASKET"""
res_local = run_sql(query_local)
query_external = """
SELECT rec.id_bskBASKET,
bsk.name,
ugbsk.share_level,
rec.id_bibrec_or_bskEXTREC,
ext.value
FROM bskREC AS rec
JOIN bskBASKET AS bsk
ON bsk.id=rec.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=rec.id_bskBASKET
AND ugbsk.id_usergroup=0
JOIN bskEXTFMT AS ext
ON ext.id_bskEXTREC=-rec.id_bibrec_or_bskEXTREC
AND ext.format=%s
WHERE rec.id_bibrec_or_bskEXTREC < 0
ORDER BY rec.id_bskBASKET"""
params_external = (format,)
res_external = run_sql(query_external, params_external)
return (res_local, res_external)
def get_all_items_in_all_public_baskets_by_matching_notes(p=""):
"""Return all the items in all the public baskets matching
their notes' titles and bodies, grouped by basket"""
p = p and '%' + p + '%' or '%'
query = """ SELECT notes.id_bskBASKET,
bsk.name,
ugbsk.share_level,
GROUP_CONCAT(DISTINCT(notes.id_bibrec_or_bskEXTREC))
FROM bskRECORDCOMMENT AS notes
JOIN bskBASKET AS bsk
ON bsk.id=notes.id_bskBASKET
JOIN usergroup_bskBASKET AS ugbsk
ON ugbsk.id_bskBASKET=notes.id_bskBASKET
AND ugbsk.id_usergroup=0
AND ugbsk.share_level IS NOT NULL
AND ugbsk.share_level!='NO'
AND ugbsk.share_level!='RI'
WHERE notes.title like %s
OR notes.body like %s
GROUP BY notes.id_bskBASKET"""
params = (p, p)
res = run_sql(query, params)
return res
############################ Public access ####################################
def get_public_basket_infos(bskid):
"""return (id, name, date modification, nb of views, id of owner, nickname of owner, rights for public access)
for a given basket"""
basket = []
query1 = """SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
bsk.id_owner,
user.nickname
FROM bskBASKET bsk LEFT JOIN user
ON bsk.id_owner=user.id
WHERE bsk.id=%s"""
res1 = run_sql(query1, (int(bskid),))
if len(res1):
basket = list(res1[0])
query2 = """SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0 and id_bskBASKET=%s"""
res2 = run_sql(query2, (int(bskid),))
if res2:
basket.append(res2[0][0])
else:
basket.append(None)
return basket
def get_public_basket_info(bskid):
"""Return information about a given public basket."""
query = """ SELECT bsk.id,
bsk.name,
bsk.id_owner,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
COUNT(rec.id_bibrec_or_bskEXTREC),
GROUP_CONCAT(rec.id_bibrec_or_bskEXTREC),
ubsk.share_level
FROM bskBASKET AS bsk
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
JOIN usergroup_bskBASKET AS ubsk
ON ubsk.id_bskBASKET=bsk.id
AND ubsk.id_usergroup=0
WHERE bsk.id=%s
GROUP BY bsk.id;"""
params = (bskid,)
res = run_sql(query, params)
return res
def get_basket_general_infos(bskid):
"""return information about a basket, suited for public access.
@return: a (id, name, date of modification, nb of views, nb of records, id of owner) tuple
"""
query = """SELECT bsk.id,
bsk.name,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
bsk.nb_views,
count(rec.id_bibrec_or_bskEXTREC),
bsk.id_owner
FROM bskBASKET bsk LEFT JOIN bskREC rec
ON bsk.id=rec.id_bskBASKET
WHERE bsk.id=%s
GROUP BY bsk.id"""
res = run_sql(query, (int(bskid),))
if res:
query2 = "UPDATE bskBASKET SET nb_views=nb_views+1 WHERE id=%s"
run_sql(query2, (int(bskid),))
return res[0]
return ()
def get_basket_owner_id(bskid):
"""Return the uid of the owner."""
query = """SELECT id_owner
FROM bskBASKET
WHERE id=%s"""
res = run_sql(query, (bskid, ))
if res:
return res[0][0]
return -1
def count_public_baskets():
"""Returns the number of public baskets."""
query = """ SELECT COUNT(id_bskBASKET)
FROM usergroup_bskBASKET
WHERE id_usergroup=0"""
res = run_sql(query)
return __wash_sql_count(res)
def get_public_baskets_list(inf_limit, max_number, order=1, asc=1):
"""Return list of public baskets
@param inf_limit: limit to baskets from number x
@param max_number: number of baskets to return
@order: 1: order by name of basket, 2: number of views, 3: owner
@return:
[(basket id, basket name, nb of views, uid of owner, nickname of owner)]"""
query = """SELECT bsk.id,
bsk.name,
bsk.nb_views,
u.id,
u.nickname
FROM bskBASKET bsk LEFT JOIN usergroup_bskBASKET ugbsk
on bsk.id=ugbsk.id_bskBASKET
LEFT JOIN user u
on bsk.id_owner=u.id
WHERE ugbsk.id_usergroup=0
"""
if order == 2:
query += 'ORDER BY bsk.nb_views'
elif order == 3:
query += 'ORDER BY u.nickname'
if asc:
query += ' ASC'
else:
query += ' DESC'
query += ', u.id'
else:
query += 'ORDER BY bsk.name'
if asc:
query += ' ASC '
else:
query += ' DESC '
query += "LIMIT %s,%s"
return run_sql(query, (inf_limit, max_number))
def count_all_public_baskets():
"""Return the number of all the public baskets."""
query = """ SELECT count(id_bskBASKET)
FROM usergroup_bskBASKET
WHERE id_usergroup=0"""
res = run_sql(query)
return __wash_sql_count(res)
def get_list_public_baskets(page, max_number, sort='name', asc=1):
"""Return list of public baskets
@param page: limit to baskets from number x
@param max_number: maximum number of baskets to return
@sort: 1: order by name of basket, 2: number of views, 3: owner
@return:
[(basket id, basket name, nb of views, uid of owner, nickname of owner)]"""
query = """ SELECT bsk.id,
bsk.name,
bsk.id_owner,
u.nickname,
DATE_FORMAT(bsk.date_modification, '%%Y-%%m-%%d %%H:%%i:%%s'),
COUNT(rec.id_bibrec_or_bskEXTREC) AS items,
bsk.nb_views
FROM usergroup_bskBASKET AS ugbsk
JOIN bskBASKET AS bsk
ON bsk.id=ugbsk.id_bskBASKET
LEFT JOIN bskREC AS rec
ON rec.id_bskBASKET=bsk.id
LEFT JOIN user AS u
ON u.id=bsk.id_owner
WHERE ugbsk.id_usergroup=0
GROUP BY bsk.id"""
if sort == 'name':
query += """
ORDER BY bsk.name"""
elif sort == 'owner':
query += """
ORDER BY u.nickname"""
elif sort == 'views':
query += """
ORDER BY bsk.nb_views"""
elif sort == 'date':
query += """
ORDER BY bsk.date_modification"""
elif sort == 'items':
query += """
ORDER BY items"""
else:
query += """
ORDER BY bsk.name"""
if asc:
query += """ ASC"""
if sort == """owner""":
query += """, u.id"""
else:
query += """ DESC"""
if sort == """owner""":
query += """, u.id"""
query += """
LIMIT %s, %s"""
page = max(0, page)
res = run_sql(query, (page, max_number))
return res
def is_basket_public(bskid):
"""Check if the given basket is public.
Returns ((0,),) if False, ((1,),) if True."""
query = """ SELECT COUNT(*)
FROM usergroup_bskBASKET
WHERE id_usergroup=0
AND id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return __wash_sql_count(res)
def subscribe(uid, bskid):
"""Subscribe the given user to the given basket."""
query1 = """SELECT COUNT(*)
FROM user_bskBASKET
WHERE id_user=%s
AND id_bskBASKET=%s"""
params1 = (uid, bskid)
res1 = run_sql(query1, params1)
if res1[0][0]:
# The user is either the owner of the basket or is already subscribed.
return False
else:
query2 = """INSERT INTO user_bskBASKET (id_user, id_bskBASKET)
VALUES (%s, %s)"""
params2 = (uid, bskid)
run_sql(query2, params2)
return True
def unsubscribe(uid, bskid):
"""Unsubscribe the given user from the given basket."""
query1 = """SELECT COUNT(*)
FROM bskBASKET
WHERE id_owner=%s
AND id=%s"""
params1 = (uid, bskid)
res1 = run_sql(query1, params1)
if res1[0][0]:
# The user is the owner of the basket.
return False
else:
query2 = """DELETE FROM user_bskBASKET
WHERE id_user=%s
AND id_bskBASKET=%s"""
params2 = (uid, bskid)
res2 = run_sql(query2, params2)
if res2:
return True
else:
return False
def is_user_subscribed_to_basket(uid, bskid):
"""Return ((1,),) if the user is subscribed to the given basket
or ((0,),) if the user is not subscribed or is the owner of the basket."""
query = """ SELECT COUNT(ubsk.id_bskBASKET)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner!=ubsk.id_user
WHERE ubsk.id_user=%s
AND ubsk.id_bskBASKET=%s"""
params = (uid, bskid)
res = run_sql(query, params)
return __wash_sql_count(res)
def count_subscribers(uid, bskid):
"""Returns a (number of users, number of groups, number of alerts) tuple
for the given user (uid) and basket (bskid)."""
uid = int(uid)
bskid = int(bskid)
query_groups = """ SELECT count(id_usergroup)
FROM usergroup_bskBASKET
WHERE id_bskBASKET=%s
AND NOT(share_level='NO')
GROUP BY id_bskBASKET"""
params_groups = (bskid,)
res_groups = run_sql(query_groups, params_groups)
nb_groups = __wash_sql_count(res_groups)
query_users = """ SELECT count(id_user)
FROM user_bskBASKET
WHERE id_bskBASKET=%s
AND id_user!=%s
GROUP BY id_bskBASKET"""
params_users = (bskid, uid)
res_users = run_sql(query_users, params_users)
nb_users = __wash_sql_count(res_users)
query_alerts = """ SELECT count(id_query)
FROM user_query_basket
WHERE id_basket=%s
GROUP BY id_basket"""
params_alerts = (bskid,)
res_alerts = run_sql(query_alerts, params_alerts)
nb_alerts = __wash_sql_count(res_alerts)
return (nb_users, nb_groups, nb_alerts)
def get_groups_subscribing_to_basket(bskid):
""" get list of (group id, group name, rights) tuples for a given basket
Please note that group 0 is used to mean everybody.
"""
query = """SELECT ugb.id_usergroup,
ug.name,
ugb.share_level
FROM usergroup_bskBASKET ugb LEFT JOIN usergroup ug
ON ugb.id_usergroup=ug.id
WHERE ugb.id_bskBASKET=%s
ORDER BY ugb.id_usergroup"""
return run_sql(query, (int(bskid),))
def get_rights_on_public_basket(bskid):
""""""
query = """ SELECT share_level
FROM usergroup_bskBASKET
WHERE id_usergroup=0
AND id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return res
def count_public_basket_subscribers(bskid):
"""Return the number of users subscribed to the given public basket."""
query = """ SELECT COUNT(ubsk.id_user)
FROM user_bskBASKET AS ubsk
JOIN bskBASKET AS bsk
ON bsk.id=ubsk.id_bskBASKET
AND bsk.id_owner!=ubsk.id_user
WHERE ubsk.id_bskBASKET=%s"""
params = (bskid,)
res = run_sql(query, params)
return __wash_sql_count(res)
################################ Notes ########################################
def get_notes(bskid, recid):
"""Return all comments for record recid in basket bskid."""
query = """
SELECT user.id,
user.nickname,
bskcmt.title,
bskcmt.body,
DATE_FORMAT(bskcmt.date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
bskcmt.priority,
bskcmt.id,
bskcmt.in_reply_to_id_bskRECORDCOMMENT
FROM bskRECORDCOMMENT bskcmt LEFT JOIN user
ON (bskcmt.id_user=user.id)
WHERE bskcmt.id_bskBASKET=%s AND
bskcmt.id_bibrec_or_bskEXTREC=%s
ORDER BY bskcmt.reply_order_cached_data
"""
bskid = int(bskid)
recid = int(recid)
res = run_sql(query, (bskid, recid))
if res:
return res
else:
return ()
def get_note(cmtid):
"""Return comment cmtid as a (author's nickname, author's uid, title, body, date of creation, priority) tuple"""
out = ()
query = """
SELECT user.nickname,
user.id,
bskcmt.title,
bskcmt.body,
DATE_FORMAT(bskcmt.date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
bskcmt.priority
FROM bskRECORDCOMMENT bskcmt LEFT JOIN user
ON (bskcmt.id_user=user.id)
WHERE bskcmt.id=%s
"""
cmtid = int(cmtid)
res = run_sql(query, (cmtid,))
if res:
return res[0]
return out
def save_note(uid, bskid, recid, title, body, date_creation=None, reply_to=None):
"""Save then given note (title, body) on the given item in the given basket.
@param date_creation: date in which the note was created
@type date_creation: None or String, e.g: '2011-07-04 14:20:57'
Note: convert_datestruct_to_datetext((2005, 11, 16, 15, 11, 44, 2, 320, 0)) -> '2005-11-16 15:11:44'
"""
if reply_to and CFG_WEBBASKET_MAX_COMMENT_THREAD_DEPTH >= 0:
# Check that we have not reached max depth
note_ancestors = get_note_ancestors(reply_to)
if len(note_ancestors) >= CFG_WEBBASKET_MAX_COMMENT_THREAD_DEPTH:
if CFG_WEBBASKET_MAX_COMMENT_THREAD_DEPTH == 0:
reply_to = None
else:
reply_to = note_ancestors[CFG_WEBBASKET_MAX_COMMENT_THREAD_DEPTH - 1]
if not date_creation:
date = convert_datestruct_to_datetext(localtime())
else: #the date comes with the proper format
date = date_creation
res = run_sql("""INSERT INTO bskRECORDCOMMENT (id_user, id_bskBASKET,
id_bibrec_or_bskEXTREC, title, body, date_creation, in_reply_to_id_bskRECORDCOMMENT)
VALUES (%s, %s, %s, %s, %s, %s, %s)""",
(int(uid), int(bskid), int(recid), title, body, date, reply_to or 0))
if res:
new_comid = int(res)
parent_reply_order = run_sql("""SELECT reply_order_cached_data from bskRECORDCOMMENT where id=%s""", (reply_to,))
if not parent_reply_order or parent_reply_order[0][0] is None:
parent_reply_order = ''
else:
parent_reply_order = parent_reply_order[0][0]
- run_sql("""UPDATE bskRECORDCOMMENT SET reply_order_cached_data=%s WHERE id=%s""",
+ run_sql("""UPDATE bskRECORDCOMMENT SET reply_order_cached_data=_binary %s WHERE id=%s""",
(parent_reply_order + get_reply_order_cache_data(new_comid), new_comid))
return int(res)
return 0
def delete_note(bskid, recid, cmtid):
"""Delete a comment on an item of a basket"""
query = """ DELETE
FROM bskRECORDCOMMENT
WHERE id_bskBASKET=%s
AND id_bibrec_or_bskEXTREC=%s
AND id=%s"""
params = (int(bskid), int(recid), int(cmtid))
run_sql(query, params)
def get_note_ancestors(cmtid, depth=None):
"""
Returns the list of ancestors of the given note, ordered from
oldest to newest ("top-down": direct parent of cmtid is at last position),
up to given depth
@param cmtid: the ID of the note for which we want to retrieve ancestors
@type cmtid: int
@param depth: the maximum of levels up from the given note we
want to retrieve ancestors. None for no limit, 1 for
direct parent only, etc.
@type depth: int
@return the list of ancestors
@rtype: list
"""
if depth == 0:
return []
res = run_sql("SELECT in_reply_to_id_bskRECORDCOMMENT FROM bskRECORDCOMMENT WHERE id=%s", (cmtid,))
if res:
parent_cmtid = res[0][0]
if parent_cmtid == 0:
return []
parent_ancestors = []
if depth:
depth -= 1
parent_ancestors = get_note_ancestors(parent_cmtid, depth)
parent_ancestors.append(parent_cmtid)
return parent_ancestors
else:
return []
def note_belongs_to_item_in_basket_p(cmtid, recid, bskid):
"""Returns 1 (True) if the given note (cmtid) belongs to the given item
(recid) and the given basket (bskid) or 0 (False)."""
query = """ SELECT COUNT(*)
FROM bskRECORDCOMMENT
WHERE id=%s
AND id_bibrec_or_bskEXTREC=%s
AND id_bskBASKET=%s"""
params = (cmtid, recid, bskid)
res = run_sql(query, params)
return __wash_sql_count(res)
def get_number_of_notes_per_record_in_basket(bskid, recids):
"""Returns the number of comments per record
for all the given records in the given basket"""
# We need to convert the list of recids into a string of commma separated
# numbers (recids), instead of a tuple, to cover the case where we have
# single element lists of recids. Example:
# [1] --> '1' instaed of [1] --> (1,)
# Single element tuples would cause the query to fail due to the syntax.
query = """ SELECT rec.id_bibrec_or_bskEXTREC,
COUNT(cmt.id_bibrec_or_bskEXTREC)
FROM bskREC as rec
LEFT JOIN bskRECORDCOMMENT as cmt
ON cmt.id_bibrec_or_bskEXTREC = rec.id_bibrec_or_bskEXTREC
WHERE rec.id_bskBASKET=%%s
AND rec.id_bibrec_or_bskEXTREC IN (%s)
GROUP BY id_bibrec_or_bskEXTREC
ORDER BY rec.score""" % (str(map(int, recids))[1:-1],)
params = (bskid,)
result = run_sql(query, params)
return result
########################## Usergroup functions ################################
def get_group_infos(uid):
"""For each group the user with uid is a member of return the id, name and number of baskets."""
query = """SELECT g.id,
g.name,
count(ugb.id_bskBASKET)
FROM usergroup g LEFT JOIN (user_usergroup ug,
usergroup_bskBASKET ugb)
ON (g.id=ug.id_usergroup
AND
g.id=ugb.id_usergroup)
WHERE ug.id_user=%s AND NOT(ugb.share_level='NO') AND ug.user_status!=%s
GROUP BY g.id
ORDER BY g.name"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
return res
def count_groups_user_member_of(uid):
"""Returns the number of groups the user has joined."""
query = """ SELECT COUNT(id_usergroup)
FROM user_usergroup
WHERE id_user=%s
AND user_status!=%s"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
return __wash_sql_count(res)
def get_groups_user_member_of(uid):
"""
Get uids and names of groups user is member of.
@param uid: user id (int)
@return: a tuple of (group_id, group_name) tuples
"""
query = """
SELECT g.id,
g.name
FROM usergroup g JOIN user_usergroup ug
ON (g.id=ug.id_usergroup)
WHERE ug.id_user=%s and ug.user_status!=%s
ORDER BY g.name
"""
params = (int(uid), CFG_WEBSESSION_USERGROUP_STATUS['PENDING'])
res = run_sql(query, params)
if res:
return res
return ()
########################## auxilliary functions ###############################
def __wash_sql_count(res):
"""Wash the result of SQL COUNT function and return only an integer."""
if res:
return res[0][0]
return 0
def __decompress_last(item):
"""private function, used to shorten code"""
item = list(item)
item[-1] = decompress(item[-1])
return item
def create_pseudo_record(es_title, es_desc, es_url, of="hb"):
"""Return a pseudo record representation given a title and a description."""
if of == 'hb':
record = '\n'.join([es_title, es_desc, es_url])
if of == 'xm':
# In case we want to use the controlfield,
# the -es_id must be used.
#%s
record = """
%s
%s
%s
""" % (encode_for_xml(es_title), encode_for_xml(es_desc), encode_for_xml(es_url))
return record
def prettify_url(url, char_limit=50, nb_dots=3):
"""If the url has more characters than char_limit return a shortened version of it
keeping the beginning and ending and replacing the rest with dots."""
if len(url) > char_limit:
# let's set a minimum character limit
if char_limit < 5:
char_limit = 5
# let's set a maximum number of dots in relation to the character limit
if nb_dots > char_limit/4:
nb_dots = char_limit/5
nb_char_url = char_limit - nb_dots
nb_char_end = nb_char_url/4
nb_char_beg = nb_char_url - nb_char_end
return url[:nb_char_beg] + '.'*nb_dots + url[-nb_char_end:]
else:
return url
diff --git a/modules/webcomment/lib/webcomment.py b/modules/webcomment/lib/webcomment.py
index b3f1fabbd..374f186c6 100644
--- a/modules/webcomment/lib/webcomment.py
+++ b/modules/webcomment/lib/webcomment.py
@@ -1,2039 +1,2039 @@
# -*- coding: utf-8 -*-
# This file is part of Invenio.
-# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012 CERN.
+# Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
""" Comments and reviews for records """
__revision__ = "$Id$"
# non Invenio imports:
import time
import math
import os
import shutil
import cgi
import re
from datetime import datetime, timedelta
# Invenio imports:
from invenio.dbquery import run_sql
from invenio.config import CFG_PREFIX, \
CFG_SITE_LANG, \
CFG_WEBALERT_ALERT_ENGINE_EMAIL,\
CFG_SITE_SUPPORT_EMAIL,\
CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,\
CFG_SITE_URL,\
CFG_SITE_NAME,\
CFG_WEBCOMMENT_ALLOW_REVIEWS,\
CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS,\
CFG_WEBCOMMENT_ALLOW_COMMENTS,\
CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL,\
CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,\
CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS,\
CFG_WEBCOMMENT_DEFAULT_MODERATOR, \
CFG_SITE_RECORD, \
CFG_WEBCOMMENT_EMAIL_REPLIES_TO, \
CFG_WEBCOMMENT_ROUND_DATAFIELD, \
CFG_WEBCOMMENT_RESTRICTION_DATAFIELD, \
CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH
from invenio.webmessage_mailutils import \
email_quote_txt, \
email_quoted_txt2html
from invenio.htmlutils import tidy_html
from invenio.webuser import get_user_info, get_email, collect_user_info
from invenio.dateutils import convert_datetext_to_dategui, \
datetext_default, \
convert_datestruct_to_datetext
from invenio.mailutils import send_email
from invenio.errorlib import register_exception
from invenio.messages import wash_language, gettext_set_language
from invenio.urlutils import wash_url_argument
from invenio.webcomment_config import CFG_WEBCOMMENT_ACTION_CODE
from invenio.access_control_engine import acc_authorize_action
from invenio.search_engine import \
guess_primary_collection_of_a_record, \
check_user_can_view_record, \
get_collection_reclist, \
get_colID
from invenio.search_engine_utils import get_fieldvalues
from invenio.webcomment_washer import EmailWasher
try:
import invenio.template
webcomment_templates = invenio.template.load('webcomment')
except:
pass
def perform_request_display_comments_or_remarks(req, recID, display_order='od', display_since='all', nb_per_page=100, page=1, ln=CFG_SITE_LANG, voted=-1, reported=-1, subscribed=0, reviews=0, uid=-1, can_send_comments=False, can_attach_files=False, user_is_subscribed_to_discussion=False, user_can_unsubscribe_from_discussion=False, display_comment_rounds=None):
"""
Returns all the comments (reviews) of a specific internal record or external basket record.
@param recID: record id where (internal record IDs > 0) or (external basket record IDs < -100)
@param display_order: hh = highest helpful score, review only
lh = lowest helpful score, review only
hs = highest star score, review only
ls = lowest star score, review only
od = oldest date
nd = newest date
@param display_since: all= no filtering by date
nd = n days ago
nw = n weeks ago
nm = n months ago
ny = n years ago
where n is a single digit integer between 0 and 9
@param nb_per_page: number of results per page
@param page: results page
@param voted: boolean, active if user voted for a review, see perform_request_vote function
@param reported: boolean, active if user reported a certain comment/review, perform_request_report function
@param subscribed: int, 1 if user just subscribed to discussion, -1 if unsubscribed
@param reviews: boolean, enabled if reviews, disabled for comments
@param uid: the id of the user who is reading comments
@param can_send_comments: if user can send comment or not
@param can_attach_files: if user can attach file to comment or not
@param user_is_subscribed_to_discussion: True if user already receives new comments by email
@param user_can_unsubscribe_from_discussion: True is user is allowed to unsubscribe from discussion
@return html body.
"""
_ = gettext_set_language(ln)
warnings = []
nb_reviews = 0
nb_comments = 0
# wash arguments
recID = wash_url_argument(recID, 'int')
ln = wash_language(ln)
display_order = wash_url_argument(display_order, 'str')
display_since = wash_url_argument(display_since, 'str')
nb_per_page = wash_url_argument(nb_per_page, 'int')
page = wash_url_argument(page, 'int')
voted = wash_url_argument(voted, 'int')
reported = wash_url_argument(reported, 'int')
reviews = wash_url_argument(reviews, 'int')
# vital argument check
(valid, error_body) = check_recID_is_in_range(recID, warnings, ln)
if not(valid):
return error_body
# CERN hack begins: filter out ATLAS comments
from invenio.config import CFG_CERN_SITE
if CFG_CERN_SITE:
restricted_comments_p = False
for report_number in get_fieldvalues(recID, '088__a'):
if report_number.startswith("ATL-"):
restricted_comments_p = True
break
if restricted_comments_p:
err_code, err_msg = acc_authorize_action(uid, 'viewrestrcoll',
collection='ATLAS Communications')
if err_code:
return err_msg
# CERN hack ends
# Query the database and filter results
user_info = collect_user_info(uid)
res = query_retrieve_comments_or_remarks(recID, display_order, display_since, reviews, user_info=user_info)
# res2 = query_retrieve_comments_or_remarks(recID, display_order, display_since, not reviews, user_info=user_info)
nb_res = len(res)
from invenio.webcommentadminlib import get_nb_reviews, get_nb_comments
nb_reviews = get_nb_reviews(recID, count_deleted=False)
nb_comments = get_nb_comments(recID, count_deleted=False)
# checking non vital arguemnts - will be set to default if wrong
#if page <= 0 or page.lower() != 'all':
if page < 0:
page = 1
if nb_per_page < 0:
nb_per_page = 100
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
if display_order not in ['od', 'nd', 'hh', 'lh', 'hs', 'ls']:
display_order = 'hh'
else:
if display_order not in ['od', 'nd']:
display_order = 'od'
if not display_comment_rounds:
display_comment_rounds = []
# filter results according to page and number of reults per page
if nb_per_page > 0:
if nb_res > 0:
last_page = int(math.ceil(nb_res / float(nb_per_page)))
else:
last_page = 1
if page > last_page:
page = 1
if nb_res > nb_per_page: # if more than one page of results
if page < last_page:
res = res[(page-1)*(nb_per_page) : (page*nb_per_page)]
else:
res = res[(page-1)*(nb_per_page) : ]
else: # one page of results
pass
else:
last_page = 1
# Add information regarding visibility of comment for user
user_collapsed_comments = get_user_collapsed_comments_for_record(uid, recID)
if reviews:
res = [row[:] + (row[10] in user_collapsed_comments,) for row in res]
else:
res = [row[:] + (row[6] in user_collapsed_comments,) for row in res]
# Send to template
avg_score = 0.0
# comments not allowed by admin
if not CFG_WEBCOMMENT_ALLOW_COMMENTS and not CFG_WEBCOMMENT_ALLOW_REVIEWS:
body = webcomment_templates.tmpl_error(
_('Comments on records have been disallowed by the'
' administrator.'), ln)
return body
if reported > 0:
warnings.append((_('Your feedback has been recorded, many thanks.'),
'green'))
elif reported == 0:
warnings.append((_('You have already reported an abuse for this'
' comment.'), ''))
elif reported == -2:
warnings.append((_('The comment you have reported no longer '
'exists.'), ''))
if CFG_WEBCOMMENT_ALLOW_REVIEWS and reviews:
avg_score = calculate_avg_score(res)
if voted > 0:
warnings.append((_('Your feedback has been recorded, many'
' thanks.'), 'green'))
elif voted == 0:
warnings.append((_('Sorry, you have already voted. This vote has '
'not been recorded.'), ''))
if subscribed == 1:
warnings.append(
(_('You have been subscribed to this discussion. From now on, you'
' will receive an email whenever a new comment is posted.'),
'green')
)
elif subscribed == -1:
warnings.append((_('You have been unsubscribed from this discussion.'),
'green'))
grouped_comments = group_comments_by_round(res, reviews)
# Clean list of comments round names
if not display_comment_rounds:
display_comment_rounds = []
elif 'all' in display_comment_rounds:
display_comment_rounds = [cmtgrp[0] for cmtgrp in grouped_comments]
elif 'latest' in display_comment_rounds:
if grouped_comments:
display_comment_rounds.append(grouped_comments[-1][0])
display_comment_rounds.remove('latest')
body = webcomment_templates.tmpl_get_comments(req,
recID,
ln,
nb_per_page, page, last_page,
display_order, display_since,
CFG_WEBCOMMENT_ALLOW_REVIEWS,
grouped_comments, nb_comments, avg_score,
warnings,
border=0,
reviews=reviews,
total_nb_reviews=nb_reviews,
uid=uid,
can_send_comments=can_send_comments,
can_attach_files=can_attach_files,
user_is_subscribed_to_discussion=\
user_is_subscribed_to_discussion,
user_can_unsubscribe_from_discussion=\
user_can_unsubscribe_from_discussion,
display_comment_rounds=display_comment_rounds)
return body
def perform_request_vote(cmt_id, client_ip_address, value, uid=-1):
"""
Vote positively or negatively for a comment/review
@param cmt_id: review id
@param value: +1 for voting positively
-1 for voting negatively
@return: integer 1 if successful, integer 0 if not
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
value = wash_url_argument(value, 'int')
uid = wash_url_argument(uid, 'int')
if cmt_id > 0 and value in [-1, 1] and check_user_can_vote(cmt_id, client_ip_address, uid):
action_date = convert_datestruct_to_datetext(time.localtime())
action_code = CFG_WEBCOMMENT_ACTION_CODE['VOTE']
query = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT,
id_bibrec, id_user, client_host, action_time,
action_code)
VALUES (%s, NULL ,%s, inet_aton(%s), %s, %s)"""
params = (cmt_id, uid, client_ip_address, action_date, action_code)
run_sql(query, params)
return query_record_useful_review(cmt_id, value)
else:
return 0
def check_user_can_comment(recID, client_ip_address, uid=-1):
""" Check if a user hasn't already commented within the last seconds
time limit: CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS
@param recID: record id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
recID = wash_url_argument(recID, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
max_action_time = time.time() - CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_COMMENTS_IN_SECONDS
max_action_time = convert_datestruct_to_datetext(time.localtime(max_action_time))
action_code = CFG_WEBCOMMENT_ACTION_CODE['ADD_COMMENT']
query = """SELECT id_bibrec
FROM cmtACTIONHISTORY
WHERE id_bibrec=%s AND
action_code=%s AND
action_time>%s
"""
params = (recID, action_code, max_action_time)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return len(res) == 0
def check_user_can_review(recID, client_ip_address, uid=-1):
""" Check if a user hasn't already reviewed within the last seconds
time limit: CFG_WEBCOMMENT_TIMELIMIT_PROCESSING_REVIEWS_IN_SECONDS
@param recID: record ID
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
action_code = CFG_WEBCOMMENT_ACTION_CODE['ADD_REVIEW']
query = """SELECT id_bibrec
FROM cmtACTIONHISTORY
WHERE id_bibrec=%s AND
action_code=%s
"""
params = (recID, action_code)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return len(res) == 0
def check_user_can_vote(cmt_id, client_ip_address, uid=-1):
""" Checks if a user hasn't already voted
@param cmt_id: comment id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
query = """SELECT id_cmtRECORDCOMMENT
FROM cmtACTIONHISTORY
WHERE id_cmtRECORDCOMMENT=%s"""
params = (cmt_id,)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid, )
res = run_sql(query, params)
return (len(res) == 0)
def get_comment_collection(cmt_id):
"""
Extract the collection where the comment is written
"""
query = "SELECT id_bibrec FROM cmtRECORDCOMMENT WHERE id=%s"
recid = run_sql(query, (cmt_id,))
record_primary_collection = guess_primary_collection_of_a_record(recid[0][0])
return record_primary_collection
def get_collection_moderators(collection):
"""
Return the list of comment moderators for the given collection.
"""
from invenio.access_control_engine import acc_get_authorized_emails
res = list(acc_get_authorized_emails('moderatecomments', collection=collection))
if not res:
return [CFG_WEBCOMMENT_DEFAULT_MODERATOR,]
return res
def perform_request_report(cmt_id, client_ip_address, uid=-1):
"""
Report a comment/review for inappropriate content.
Will send an email to the administrator if number of reports is a multiple of CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN
@param cmt_id: comment id
@return: integer 1 if successful, integer 0 if not. -2 if comment does not exist
"""
cmt_id = wash_url_argument(cmt_id, 'int')
if cmt_id <= 0:
return 0
(query_res, nb_abuse_reports) = query_record_report_this(cmt_id)
if query_res == 0:
return 0
elif query_res == -2:
return -2
if not(check_user_can_report(cmt_id, client_ip_address, uid)):
return 0
action_date = convert_datestruct_to_datetext(time.localtime())
action_code = CFG_WEBCOMMENT_ACTION_CODE['REPORT_ABUSE']
query = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT, id_bibrec,
id_user, client_host, action_time, action_code)
VALUES (%s, NULL, %s, inet_aton(%s), %s, %s)"""
params = (cmt_id, uid, client_ip_address, action_date, action_code)
run_sql(query, params)
if nb_abuse_reports % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN == 0:
(cmt_id2,
id_bibrec,
id_user,
cmt_body,
cmt_date,
cmt_star,
cmt_vote, cmt_nb_votes_total,
cmt_title,
cmt_reported,
round_name,
restriction) = query_get_comment(cmt_id)
(user_nb_abuse_reports,
user_votes,
user_nb_votes_total) = query_get_user_reports_and_votes(int(id_user))
(nickname, user_email, last_login) = query_get_user_contact_info(id_user)
from_addr = '%s Alert Engine <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(cmt_id)
to_addrs = get_collection_moderators(comment_collection)
subject = "A comment has been reported as inappropriate by a user"
body = '''
The following comment has been reported a total of %(cmt_reported)s times.
Author: nickname = %(nickname)s
email = %(user_email)s
user_id = %(uid)s
This user has:
total number of reports = %(user_nb_abuse_reports)s
%(votes)s
Comment: comment_id = %(cmt_id)s
record_id = %(id_bibrec)s
date written = %(cmt_date)s
nb reports = %(cmt_reported)s
%(review_stuff)s
body =
---start body---
%(cmt_body)s
---end body---
Please go to the record page %(comment_admin_link)s to delete this message if necessary. A warning will be sent to the user in question.''' % \
{ 'cfg-report_max' : CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN,
'nickname' : nickname,
'user_email' : user_email,
'uid' : id_user,
'user_nb_abuse_reports' : user_nb_abuse_reports,
'user_votes' : user_votes,
'votes' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"total number of positive votes\t= %s\n\t\ttotal number of negative votes\t= %s" % \
(user_votes, (user_nb_votes_total - user_votes)) or "\n",
'cmt_id' : cmt_id,
'id_bibrec' : id_bibrec,
'cmt_date' : cmt_date,
'cmt_reported' : cmt_reported,
'review_stuff' : CFG_WEBCOMMENT_ALLOW_REVIEWS and \
"star score\t= %s\n\treview title\t= %s" % (cmt_star, cmt_title) or "",
'cmt_body' : cmt_body,
'comment_admin_link' : CFG_SITE_URL + "/"+ CFG_SITE_RECORD +"/" + str(id_bibrec) + '/comments#' + str(cmt_id),
'user_admin_link' : "user_admin_link" #! FIXME
}
#FIXME to be added to email when websession module is over:
#If you wish to ban the user, you can do so via the User Admin Panel %(user_admin_link)s.
send_email(from_addr, to_addrs, subject, body)
return 1
def check_user_can_report(cmt_id, client_ip_address, uid=-1):
""" Checks if a user hasn't already reported a comment
@param cmt_id: comment id
@param client_ip_address: IP => use: str(req.remote_ip)
@param uid: user id, as given by invenio.webuser.getUid(req)
"""
cmt_id = wash_url_argument(cmt_id, 'int')
client_ip_address = wash_url_argument(client_ip_address, 'str')
uid = wash_url_argument(uid, 'int')
query = """SELECT id_cmtRECORDCOMMENT
FROM cmtACTIONHISTORY
WHERE id_cmtRECORDCOMMENT=%s"""
params = (uid,)
if uid < 0:
query += " AND client_host=inet_aton(%s)"
params += (client_ip_address,)
else:
query += " AND id_user=%s"
params += (uid,)
res = run_sql(query, params)
return (len(res) == 0)
def query_get_user_contact_info(uid):
"""
Get the user contact information
@return: tuple (nickname, email, last_login), if none found return ()
Note: for the moment, if no nickname, will return email address up to the '@'
"""
query1 = """SELECT nickname, email,
DATE_FORMAT(last_login, '%%Y-%%m-%%d %%H:%%i:%%s')
FROM user WHERE id=%s"""
params1 = (uid,)
res1 = run_sql(query1, params1)
if res1:
return res1[0]
else:
return ()
def query_get_user_reports_and_votes(uid):
"""
Retrieve total number of reports and votes of a particular user
@param uid: user id
@return: tuple (total_nb_reports, total_nb_votes_yes, total_nb_votes_total)
if none found return ()
"""
query1 = """SELECT nb_votes_yes,
nb_votes_total,
nb_abuse_reports
FROM cmtRECORDCOMMENT
WHERE id_user=%s"""
params1 = (uid,)
res1 = run_sql(query1, params1)
if len(res1) == 0:
return ()
nb_votes_yes = nb_votes_total = nb_abuse_reports = 0
for cmt_tuple in res1:
nb_votes_yes += int(cmt_tuple[0])
nb_votes_total += int(cmt_tuple[1])
nb_abuse_reports += int(cmt_tuple[2])
return (nb_abuse_reports, nb_votes_yes, nb_votes_total)
def query_get_comment(comID):
"""
Get all fields of a comment
@param comID: comment id
@return: tuple (comID, id_bibrec, id_user, body, date_creation, star_score, nb_votes_yes, nb_votes_total, title, nb_abuse_reports, round_name, restriction)
if none found return ()
"""
query1 = """SELECT id,
id_bibrec,
id_user,
body,
DATE_FORMAT(date_creation, '%%Y-%%m-%%d %%H:%%i:%%s'),
star_score,
nb_votes_yes,
nb_votes_total,
title,
nb_abuse_reports,
round_name,
restriction
FROM cmtRECORDCOMMENT
WHERE id=%s"""
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1)>0:
return res1[0]
else:
return ()
def query_record_report_this(comID):
"""
Increment the number of reports for a comment
@param comID: comment id
@return: tuple (success, new_total_nb_reports_for_this_comment) where
success is integer 1 if success, integer 0 if not, -2 if comment does not exist
"""
#retrieve nb_abuse_reports
query1 = "SELECT nb_abuse_reports FROM cmtRECORDCOMMENT WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1) == 0:
return (-2, 0)
#increment and update
nb_abuse_reports = int(res1[0][0]) + 1
query2 = "UPDATE cmtRECORDCOMMENT SET nb_abuse_reports=%s WHERE id=%s"
params2 = (nb_abuse_reports, comID)
res2 = run_sql(query2, params2)
return (int(res2), nb_abuse_reports)
def query_record_useful_review(comID, value):
"""
private funciton
Adjust the number of useful votes and number of total votes for a comment.
@param comID: comment id
@param value: +1 or -1
@return: integer 1 if successful, integer 0 if not
"""
# retrieve nb_useful votes
query1 = "SELECT nb_votes_total, nb_votes_yes FROM cmtRECORDCOMMENT WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
if len(res1)==0:
return 0
# modify and insert new nb_useful votes
nb_votes_yes = int(res1[0][1])
if value >= 1:
nb_votes_yes = int(res1[0][1]) + 1
nb_votes_total = int(res1[0][0]) + 1
query2 = "UPDATE cmtRECORDCOMMENT SET nb_votes_total=%s, nb_votes_yes=%s WHERE id=%s"
params2 = (nb_votes_total, nb_votes_yes, comID)
res2 = run_sql(query2, params2)
return int(res2)
def query_retrieve_comments_or_remarks(recID, display_order='od', display_since='0000-00-00 00:00:00',
ranking=0, limit='all', user_info=None):
"""
Private function
Retrieve tuple of comments or remarks from the database
@param recID: record id
@param display_order: hh = highest helpful score
lh = lowest helpful score
hs = highest star score
ls = lowest star score
od = oldest date
nd = newest date
@param display_since: datetime, e.g. 0000-00-00 00:00:00
@param ranking: boolean, enabled if reviews, disabled for comments
@param limit: number of comments/review to return
@return: tuple of comment where comment is
tuple (nickname, uid, date_creation, body, status, id) if ranking disabled or
tuple (nickname, uid, date_creation, body, status, nb_votes_yes, nb_votes_total, star_score, title, id)
Note: for the moment, if no nickname, will return email address up to '@'
"""
display_since = calculate_start_date(display_since)
order_dict = { 'hh' : "cmt.nb_votes_yes/(cmt.nb_votes_total+1) DESC, cmt.date_creation DESC ",
'lh' : "cmt.nb_votes_yes/(cmt.nb_votes_total+1) ASC, cmt.date_creation ASC ",
'ls' : "cmt.star_score ASC, cmt.date_creation DESC ",
'hs' : "cmt.star_score DESC, cmt.date_creation DESC ",
'nd' : "cmt.reply_order_cached_data DESC ",
'od' : "cmt.reply_order_cached_data ASC "
}
# Ranking only done for comments and when allowed
if ranking and recID > 0:
try:
display_order = order_dict[display_order]
except:
display_order = order_dict['od']
else:
# in case of recID > 0 => external record => no ranking!
ranking = 0
try:
if display_order[-1] == 'd':
display_order = order_dict[display_order]
else:
display_order = order_dict['od']
except:
display_order = order_dict['od']
#display_order = order_dict['nd']
query = """SELECT user.nickname,
cmt.id_user,
DATE_FORMAT(cmt.date_creation, '%%%%Y-%%%%m-%%%%d %%%%H:%%%%i:%%%%s'),
cmt.body,
cmt.status,
cmt.nb_abuse_reports,
%(ranking)s cmt.id,
cmt.round_name,
cmt.restriction,
%(reply_to_column)s
FROM cmtRECORDCOMMENT cmt LEFT JOIN user ON
user.id=cmt.id_user
WHERE cmt.id_bibrec=%%s
%(ranking_only)s
%(display_since)s
ORDER BY %(display_order)s
""" % {'ranking' : ranking and ' cmt.nb_votes_yes, cmt.nb_votes_total, cmt.star_score, cmt.title, ' or '',
'ranking_only' : ranking and ' AND cmt.star_score>0 ' or ' AND cmt.star_score=0 ',
# 'id_bibrec' : recID > 0 and 'cmt.id_bibrec' or 'cmt.id_bibrec_or_bskEXTREC',
# 'table' : recID > 0 and 'cmtRECORDCOMMENT' or 'bskRECORDCOMMENT',
'display_since' : display_since == '0000-00-00 00:00:00' and ' ' or 'AND cmt.date_creation>=\'%s\' ' % display_since,
'display_order': display_order,
'reply_to_column': recID > 0 and 'cmt.in_reply_to_id_cmtRECORDCOMMENT' or 'cmt.in_reply_to_id_bskRECORDCOMMENT'}
params = (recID,)
res = run_sql(query, params)
# return res
new_limit = limit
comments_list = []
for row in res:
if ranking:
# when dealing with reviews, row[12] holds restriction info:
restriction = row[12]
else:
# when dealing with comments, row[8] holds restriction info:
restriction = row[8]
if user_info and check_user_can_view_comment(user_info, None, restriction)[0] != 0:
# User cannot view comment. Look further
continue
comments_list.append(row)
if limit.isdigit():
new_limit -= 1
if limit < 1:
break
if comments_list:
if limit.isdigit():
return comments_list[:limit]
else:
return comments_list
return ()
# def get_comment_children(comID):
# """
# Returns the list of children (i.e. direct descendants) ordered by time of addition.
# @param comID: the ID of the comment for which we want to retrieve children
# @type comID: int
# @return the list of children
# @rtype: list
# """
# res = run_sql("SELECT id FROM cmtRECORDCOMMENT WHERE in_reply_to_id_cmtRECORDCOMMENT=%s", (comID,))
# return [row[0] for row in res]
# def get_comment_descendants(comID, depth=None):
# """
# Returns the list of descendants of the given comment, orderd from
# oldest to newest ("top-down"), down to depth specified as parameter.
# @param comID: the ID of the comment for which we want to retrieve descendant
# @type comID: int
# @param depth: the max depth down to which we want to retrieve
# descendants. Specify None for no limit, 1 for direct
# children only, etc.
# @return the list of ancestors
# @rtype: list(tuple(comment ID, descendants comments IDs))
# """
# if depth == 0:
# return (comID, [])
# res = run_sql("SELECT id FROM cmtRECORDCOMMENT WHERE in_reply_to_id_cmtRECORDCOMMENT=%s", (comID,))
# if res:
# children_comID = [row[0] for row in res]
# children_descendants = []
# if depth:
# depth -= 1
# children_descendants = [get_comment_descendants(child_comID, depth) for child_comID in children_comID]
# return (comID, children_descendants)
# else:
# return (comID, [])
def get_comment_ancestors(comID, depth=None):
"""
Returns the list of ancestors of the given comment, ordered from
oldest to newest ("top-down": direct parent of comID is at last position),
up to given depth
@param comID: the ID of the comment for which we want to retrieve ancestors
@type comID: int
@param depth: the maximum of levels up from the given comment we
want to retrieve ancestors. None for no limit, 1 for
direct parent only, etc.
@type depth: int
@return the list of ancestors
@rtype: list
"""
if depth == 0:
return []
res = run_sql("SELECT in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT WHERE id=%s", (comID,))
if res:
parent_comID = res[0][0]
if parent_comID == 0:
return []
parent_ancestors = []
if depth:
depth -= 1
parent_ancestors = get_comment_ancestors(parent_comID, depth)
parent_ancestors.append(parent_comID)
return parent_ancestors
else:
return []
def get_reply_order_cache_data(comid):
"""
Prepare a representation of the comment ID given as parameter so
that it is suitable for byte ordering in MySQL.
"""
return "%s%s%s%s" % (chr((comid >> 24) % 256), chr((comid >> 16) % 256),
chr((comid >> 8) % 256), chr(comid % 256))
def query_add_comment_or_remark(reviews=0, recID=0, uid=-1, msg="",
note="", score=0, priority=0,
client_ip_address='', editor_type='textarea',
req=None, reply_to=None, attached_files=None):
"""
Private function
Insert a comment/review or remarkinto the database
@param recID: record id
@param uid: user id
@param msg: comment body
@param note: comment title
@param score: review star score
@param priority: remark priority #!FIXME
@param editor_type: the kind of editor used to submit the comment: 'textarea', 'ckeditor'
@param req: request object. If provided, email notification are sent after we reply to user request.
@param reply_to: the id of the comment we are replying to with this inserted comment.
@return: integer >0 representing id if successful, integer 0 if not
"""
current_date = calculate_start_date('0d')
#change utf-8 message into general unicode
msg = msg.decode('utf-8')
note = note.decode('utf-8')
#change general unicode back to utf-8
msg = msg.encode('utf-8')
note = note.encode('utf-8')
msg_original = msg
(restriction, round_name) = get_record_status(recID)
if attached_files is None:
attached_files = {}
if reply_to and CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH >= 0:
# Check that we have not reached max depth
comment_ancestors = get_comment_ancestors(reply_to)
if len(comment_ancestors) >= CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH:
if CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH == 0:
reply_to = None
else:
reply_to = comment_ancestors[CFG_WEBCOMMENT_MAX_COMMENT_THREAD_DEPTH - 1]
# Inherit restriction and group/round of 'parent'
comment = query_get_comment(reply_to)
if comment:
(round_name, restriction) = comment[10:12]
if editor_type == 'ckeditor':
# Here we remove the line feeds introduced by CKEditor (they
# have no meaning for the user) and replace the HTML line
# breaks by linefeeds, so that we are close to an input that
# would be done without the CKEditor. That's much better if a
# reply to a comment is made with a browser that does not
# support CKEditor.
msg = msg.replace('\n', '').replace('\r', '')
# We clean the quotes that could have been introduced by
# CKEditor when clicking the 'quote' button, as well as those
# that we have introduced when quoting the original message.
# We can however not use directly '>>' chars to quote, as it
# will be washed/fixed when calling tidy_html(): double-escape
# all > first, and use >>
msg = msg.replace('>', '>')
msg = re.sub('^\s*\s*<(p|div).*?>', '>>', msg)
msg = re.sub('\s*', '', msg)
# Then definitely remove any blockquote, whatever it is
msg = re.sub('', '', msg)
msg = re.sub('', '', msg)
# Tidy up the HTML
msg = tidy_html(msg)
# We remove EOL that might have been introduced when tidying
msg = msg.replace('\n', '').replace('\r', '')
# Now that HTML has been cleaned, unescape >
msg = msg.replace('>', '>')
msg = msg.replace('>', '>')
msg = re.sub('
)', '\n', msg)
msg = msg.replace(' ', ' ')
# In case additional or
got inserted, interpret
# these as new lines (with a sad trick to do it only once)
# (note that it has been deactivated, as it is messing up
# indentation with >>)
#msg = msg.replace('<', '\n<')
#msg = msg.replace('<', '\n<')
query = """INSERT INTO cmtRECORDCOMMENT (id_bibrec,
id_user,
body,
date_creation,
star_score,
nb_votes_total,
title,
round_name,
restriction,
in_reply_to_id_cmtRECORDCOMMENT)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
params = (recID, uid, msg, current_date, score, 0, note, round_name, restriction, reply_to or 0)
res = run_sql(query, params)
if res:
new_comid = int(res)
move_attached_files_to_storage(attached_files, recID, new_comid)
parent_reply_order = run_sql("""SELECT reply_order_cached_data from cmtRECORDCOMMENT where id=%s""", (reply_to,))
if not parent_reply_order or parent_reply_order[0][0] is None:
# This is not a reply, but a first 0-level comment
parent_reply_order = ''
else:
parent_reply_order = parent_reply_order[0][0]
- run_sql("""UPDATE cmtRECORDCOMMENT SET reply_order_cached_data=%s WHERE id=%s""",
+ run_sql("""UPDATE cmtRECORDCOMMENT SET reply_order_cached_data=_binary %s WHERE id=%s""",
(parent_reply_order + get_reply_order_cache_data(new_comid), new_comid))
action_code = CFG_WEBCOMMENT_ACTION_CODE[reviews and 'ADD_REVIEW' or 'ADD_COMMENT']
action_time = convert_datestruct_to_datetext(time.localtime())
query2 = """INSERT INTO cmtACTIONHISTORY (id_cmtRECORDCOMMENT,
id_bibrec, id_user, client_host, action_time, action_code)
VALUES (%s, %s, %s, inet_aton(%s), %s, %s)"""
params2 = (res, recID, uid, client_ip_address, action_time, action_code)
run_sql(query2, params2)
def notify_subscribers_callback(data):
"""
Define a callback that retrieves subscribed users, and
notify them by email.
@param data: contains the necessary parameters in a tuple:
(recid, uid, comid, msg, note, score, editor_type, reviews)
"""
recid, uid, comid, msg, note, score, editor_type, reviews = data
# Email this comment to 'subscribers'
(subscribers_emails1, subscribers_emails2) = \
get_users_subscribed_to_discussion(recid)
email_subscribers_about_new_comment(recid, reviews=reviews,
emails1=subscribers_emails1,
emails2=subscribers_emails2,
comID=comid, msg=msg,
note=note, score=score,
editor_type=editor_type, uid=uid)
# Register our callback to notify subscribed people after
# having replied to our current user.
data = (recID, uid, res, msg, note, score, editor_type, reviews)
if req:
req.register_cleanup(notify_subscribers_callback, data)
else:
notify_subscribers_callback(data)
return int(res)
def move_attached_files_to_storage(attached_files, recID, comid):
"""
Move the files that were just attached to a new comment to their
final location.
@param attached_files: the mappings of desired filename to attach
and path where to find the original file
@type attached_files: dict {filename, filepath}
@param recID: the record ID to which we attach the files
@param comid: the comment ID to which we attach the files
"""
for filename, filepath in attached_files.iteritems():
dest_dir = os.path.join(CFG_PREFIX, 'var', 'data', 'comments',
str(recID), str(comid))
try:
os.makedirs(dest_dir)
except:
# Dir most probably already existed
pass
shutil.move(filepath,
os.path.join(dest_dir, filename))
def get_attached_files(recid, comid):
"""
Returns a list with tuples (filename, filepath, fileurl)
@param recid: the recid to which the comment belong
@param comid: the commment id for which we want to retrieve files
"""
base_dir = os.path.join(CFG_PREFIX, 'var', 'data', 'comments',
str(recid), str(comid))
if os.path.isdir(base_dir):
filenames = os.listdir(base_dir)
return [(filename, os.path.join(CFG_PREFIX, 'var', 'data', 'comments',
str(recid), str(comid), filename),
CFG_SITE_URL + '/'+ CFG_SITE_RECORD +'/' + str(recid) + '/comments/attachments/get/' + str(comid) + '/' + filename) \
for filename in filenames]
else:
return []
def subscribe_user_to_discussion(recID, uid):
"""
Subscribe a user to a discussion, so the she receives by emails
all new new comments for this record.
@param recID: record ID corresponding to the discussion we want to
subscribe the user
@param uid: user id
"""
query = """INSERT INTO cmtSUBSCRIPTION (id_bibrec, id_user, creation_time)
VALUES (%s, %s, %s)"""
params = (recID, uid, convert_datestruct_to_datetext(time.localtime()))
try:
run_sql(query, params)
except:
return 0
return 1
def unsubscribe_user_from_discussion(recID, uid):
"""
Unsubscribe users from a discussion.
@param recID: record ID corresponding to the discussion we want to
unsubscribe the user
@param uid: user id
@return 1 if successful, 0 if not
"""
query = """DELETE FROM cmtSUBSCRIPTION
WHERE id_bibrec=%s AND id_user=%s"""
params = (recID, uid)
try:
res = run_sql(query, params)
except:
return 0
if res > 0:
return 1
return 0
def get_user_subscription_to_discussion(recID, uid):
"""
Returns the type of subscription for the given user to this
discussion. This does not check authorizations (for eg. if user
was subscribed, but is suddenly no longer authorized).
@param recID: record ID
@param uid: user id
@return:
- 0 if user is not subscribed to discussion
- 1 if user is subscribed, and is allowed to unsubscribe
- 2 if user is subscribed, but cannot unsubscribe
"""
user_email = get_email(uid)
(emails1, emails2) = get_users_subscribed_to_discussion(recID, check_authorizations=False)
if user_email in emails1:
return 1
elif user_email in emails2:
return 2
else:
return 0
def get_users_subscribed_to_discussion(recID, check_authorizations=True):
"""
Returns the lists of users subscribed to a given discussion.
Two lists are returned: the first one is the list of emails for
users who can unsubscribe from the discussion, the second list
contains the emails of users who cannot unsubscribe (for eg. author
of the document, etc).
Users appear in only one list. If a user has manually subscribed
to a discussion AND is an automatic recipients for updates, it
will only appear in the second list.
@param recID: record ID for which we want to retrieve subscribed users
@param check_authorizations: if True, check again if users are authorized to view comment
@return tuple (emails1, emails2)
"""
subscribers_emails = {}
# Get users that have subscribed to this discussion
query = """SELECT id_user FROM cmtSUBSCRIPTION WHERE id_bibrec=%s"""
params = (recID,)
res = run_sql(query, params)
for row in res:
uid = row[0]
if check_authorizations:
user_info = collect_user_info(uid)
(auth_code, auth_msg) = check_user_can_view_comments(user_info, recID)
else:
# Don't check and grant access
auth_code = False
if auth_code:
# User is no longer authorized to view comments.
# Delete subscription
unsubscribe_user_from_discussion(recID, uid)
else:
email = get_email(uid)
if '@' in email:
subscribers_emails[email] = True
# Get users automatically subscribed, based on the record metadata
collections_with_auto_replies = CFG_WEBCOMMENT_EMAIL_REPLIES_TO.keys()
for collection in collections_with_auto_replies:
if (get_colID(collection) is not None) and \
(recID in get_collection_reclist(collection)):
fields = CFG_WEBCOMMENT_EMAIL_REPLIES_TO[collection]
for field in fields:
emails = get_fieldvalues(recID, field)
for email in emails:
if not '@' in email:
# Is a group: add domain name
subscribers_emails[email + '@' + \
CFG_SITE_SUPPORT_EMAIL.split('@')[1]] = False
else:
subscribers_emails[email] = False
return ([email for email, can_unsubscribe_p \
in subscribers_emails.iteritems() if can_unsubscribe_p],
[email for email, can_unsubscribe_p \
in subscribers_emails.iteritems() if not can_unsubscribe_p] )
def email_subscribers_about_new_comment(recID, reviews, emails1,
emails2, comID, msg="",
note="", score=0,
editor_type='textarea',
ln=CFG_SITE_LANG, uid=-1):
"""
Notify subscribers that a new comment was posted.
FIXME: consider recipient preference to send email in correct language.
@param recID: record id
@param emails1: list of emails for users who can unsubscribe from discussion
@param emails2: list of emails for users who cannot unsubscribe from discussion
@param comID: the comment id
@param msg: comment body
@param note: comment title
@param score: review star score
@param editor_type: the kind of editor used to submit the comment: 'textarea', 'ckeditor'
@rtype: bool
@return: True if email was sent okay, False if it was not.
"""
_ = gettext_set_language(ln)
if not emails1 and not emails2:
return 0
# Get title
titles = get_fieldvalues(recID, "245__a")
if not titles:
# usual title not found, try conference title:
titles = get_fieldvalues(recID, "111__a")
title = ''
if titles:
title = titles[0]
else:
title = _("Record %i") % recID
# Get report number
report_numbers = get_fieldvalues(recID, "037__a")
if not report_numbers:
report_numbers = get_fieldvalues(recID, "088__a")
if not report_numbers:
report_numbers = get_fieldvalues(recID, "021__a")
# Prepare email subject and body
if reviews:
email_subject = _('%(report_number)s"%(title)s" has been reviewed') % \
{'report_number': report_numbers and ('[' + report_numbers[0] + '] ') or '',
'title': title}
else:
email_subject = _('%(report_number)s"%(title)s" has been commented') % \
{'report_number': report_numbers and ('[' + report_numbers[0] + '] ') or '',
'title': title}
washer = EmailWasher()
msg = washer.wash(msg)
msg = msg.replace('>>', '>')
email_content = msg
if note:
email_content = note + email_content
# Send emails to people who can unsubscribe
email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=True,
ln=ln,
uid=uid)
email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=True,
ln=ln)
res1 = True
if emails1:
res1 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
toaddr=emails1,
subject=email_subject,
content=email_content,
header=email_header,
footer=email_footer,
ln=ln)
# Then send email to people who have been automatically
# subscribed to the discussion (they cannot unsubscribe)
email_header = webcomment_templates.tmpl_email_new_comment_header(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=False,
ln=ln,
uid=uid)
email_footer = webcomment_templates.tmpl_email_new_comment_footer(recID,
title,
reviews,
comID,
report_numbers,
can_unsubscribe=False,
ln=ln)
res2 = True
if emails2:
res2 = send_email(fromaddr=CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL,
toaddr=emails2,
subject=email_subject,
content=email_content,
header=email_header,
footer=email_footer,
ln=ln)
return res1 and res2
def get_record_status(recid):
"""
Returns the current status of the record, i.e. current restriction to apply for newly submitted
comments, and current commenting round.
The restriction to apply can be found in the record metadata, in
field(s) defined by config CFG_WEBCOMMENT_RESTRICTION_DATAFIELD. The restriction is empty string ""
in cases where the restriction has not explicitely been set, even
if the record itself is restricted.
@param recid: the record id
@type recid: int
@return tuple(restriction, round_name), where 'restriction' is empty string when no restriction applies
@rtype (string, int)
"""
collections_with_rounds = CFG_WEBCOMMENT_ROUND_DATAFIELD.keys()
commenting_round = ""
for collection in collections_with_rounds:
# Find the first collection defines rounds field for this
# record
if get_colID(collection) is not None and \
(recid in get_collection_reclist(collection)):
commenting_rounds = get_fieldvalues(recid, CFG_WEBCOMMENT_ROUND_DATAFIELD.get(collection, ""))
if commenting_rounds:
commenting_round = commenting_rounds[0]
break
collections_with_restrictions = CFG_WEBCOMMENT_RESTRICTION_DATAFIELD.keys()
restriction = ""
for collection in collections_with_restrictions:
# Find the first collection that defines restriction field for
# this record
if get_colID(collection) is not None and \
recid in get_collection_reclist(collection):
restrictions = get_fieldvalues(recid, CFG_WEBCOMMENT_RESTRICTION_DATAFIELD.get(collection, ""))
if restrictions:
restriction = restrictions[0]
break
return (restriction, commenting_round)
def calculate_start_date(display_since):
"""
Private function
Returns the datetime of display_since argument in MYSQL datetime format
calculated according to the local time.
@param display_since: = all= no filtering
nd = n days ago
nw = n weeks ago
nm = n months ago
ny = n years ago
where n is a single digit number
@return: string of wanted datetime.
If 'all' given as argument, will return datetext_default
datetext_default is defined in miscutils/lib/dateutils and
equals 0000-00-00 00:00:00 => MySQL format
If bad arguement given, will return datetext_default
If library 'dateutil' is not found return datetext_default
and register exception.
"""
time_types = {'d':0, 'w':0, 'm':0, 'y':0}
today = datetime.today()
try:
nb = int(display_since[:-1])
except:
return datetext_default
if display_since in [None, 'all']:
return datetext_default
if str(display_since[-1]) in time_types:
time_type = str(display_since[-1])
else:
return datetext_default
# year
if time_type == 'y':
if (int(display_since[:-1]) > today.year - 1) or (int(display_since[:-1]) < 1):
# 1 < nb years < 2008
return datetext_default
else:
final_nb_year = today.year - nb
yesterday = today.replace(year=final_nb_year)
# month
elif time_type == 'm':
try:
from dateutil.relativedelta import relativedelta
except ImportError:
# The dateutil library is only recommended: if not
# available, then send warning about this.
register_exception(alert_admin=True)
return datetext_default
# obtain only the date: yyyy-mm-dd
date_today = datetime.now().date()
final_date = date_today - relativedelta(months=nb)
yesterday = today.replace(year=final_date.year, month=final_date.month, day=final_date.day)
# week
elif time_type == 'w':
delta = timedelta(weeks=nb)
yesterday = today - delta
# day
elif time_type == 'd':
delta = timedelta(days=nb)
yesterday = today - delta
return yesterday.strftime("%Y-%m-%d %H:%M:%S")
def get_first_comments_or_remarks(recID=-1,
ln=CFG_SITE_LANG,
nb_comments='all',
nb_reviews='all',
voted=-1,
reported=-1,
user_info=None,
show_reviews=False):
"""
Gets nb number comments/reviews or remarks.
In the case of comments, will get both comments and reviews
Comments and remarks sorted by most recent date, reviews sorted by highest helpful score
@param recID: record id
@param ln: language
@param nb_comments: number of comment or remarks to get
@param nb_reviews: number of reviews or remarks to get
@param voted: 1 if user has voted for a remark
@param reported: 1 if user has reported a comment or review
@return: if comment, tuple (comments, reviews) both being html of first nb comments/reviews
if remark, tuple (remakrs, None)
"""
_ = gettext_set_language(ln)
warnings = []
voted = wash_url_argument(voted, 'int')
reported = wash_url_argument(reported, 'int')
## check recID argument
if type(recID) is not int:
return ()
if recID >= 1: #comment or review. NB: suppressed reference to basket (handled in webbasket)
if CFG_WEBCOMMENT_ALLOW_REVIEWS:
res_reviews = query_retrieve_comments_or_remarks(recID=recID, display_order="hh", ranking=1,
limit=nb_comments, user_info=user_info)
nb_res_reviews = len(res_reviews)
## check nb argument
if type(nb_reviews) is int and nb_reviews < len(res_reviews):
first_res_reviews = res_reviews[:nb_reviews]
else:
first_res_reviews = res_reviews
if CFG_WEBCOMMENT_ALLOW_COMMENTS:
res_comments = query_retrieve_comments_or_remarks(recID=recID, display_order="od", ranking=0,
limit=nb_reviews, user_info=user_info)
nb_res_comments = len(res_comments)
## check nb argument
if type(nb_comments) is int and nb_comments < len(res_comments):
first_res_comments = res_comments[:nb_comments]
else:
first_res_comments = res_comments
else:
body = webcomment_templates.tmpl_error(
_('%s is an invalid record ID') % recID, ln)
return body
# comment
if recID >= 1:
comments = reviews = ""
if reported > 0:
warnings.append((_('Your feedback has been recorded, many '
'thanks.'), 'green'))
elif reported == 0:
warnings.append((_('Your feedback could not be recorded, please'
' try again.'), ''))
# normal comments
if CFG_WEBCOMMENT_ALLOW_COMMENTS:
grouped_comments = group_comments_by_round(first_res_comments, ranking=0)
comments = webcomment_templates.tmpl_get_first_comments_without_ranking(recID, ln, grouped_comments, nb_res_comments, warnings)
if show_reviews:
# ranked comments
if CFG_WEBCOMMENT_ALLOW_REVIEWS:
# calculate average score
avg_score = calculate_avg_score(res_reviews)
if voted > 0:
warnings.append((_('Your feedback has been recorded, '
'many thanks.'), 'green'))
elif voted == 0:
warnings.append((_('Your feedback could not be recorded, '
'please try again.'), ''))
grouped_reviews = group_comments_by_round(first_res_reviews, ranking=0)
reviews = webcomment_templates.tmpl_get_first_comments_with_ranking(recID, ln, grouped_reviews, nb_res_reviews, avg_score, warnings)
return (comments, reviews)
# remark
else:
return(webcomment_templates.tmpl_get_first_remarks(first_res_comments, ln, nb_res_comments), None)
def group_comments_by_round(comments, ranking=0):
"""
Group comments by the round to which they belong
"""
comment_rounds = {}
ordered_comment_round_names = []
for comment in comments:
comment_round_name = ranking and comment[11] or comment[7]
if not comment_rounds.has_key(comment_round_name):
comment_rounds[comment_round_name] = []
ordered_comment_round_names.append(comment_round_name)
comment_rounds[comment_round_name].append(comment)
return [(comment_round_name, comment_rounds[comment_round_name]) \
for comment_round_name in ordered_comment_round_names]
def calculate_avg_score(res):
"""
private function
Calculate the avg score of reviews present in res
@param res: tuple of tuple returned from query_retrieve_comments_or_remarks
@return: a float of the average score rounded to the closest 0.5
"""
c_star_score = 6
avg_score = 0.0
nb_reviews = 0
for comment in res:
if comment[c_star_score] > 0:
avg_score += comment[c_star_score]
nb_reviews += 1
if nb_reviews == 0:
return 0.0
avg_score = avg_score / nb_reviews
avg_score_unit = avg_score - math.floor(avg_score)
if avg_score_unit < 0.25:
avg_score = math.floor(avg_score)
elif avg_score_unit > 0.75:
avg_score = math.floor(avg_score) + 1
else:
avg_score = math.floor(avg_score) + 0.5
if avg_score > 5:
avg_score = 5.0
return avg_score
def perform_request_add_comment_or_remark(recID=0,
uid=-1,
action='DISPLAY',
ln=CFG_SITE_LANG,
msg=None,
score=None,
note=None,
priority=None,
reviews=0,
comID=0,
client_ip_address=None,
editor_type='textarea',
can_attach_files=False,
subscribe=False,
req=None,
attached_files=None,
warnings=None):
"""
Add a comment/review or remark
@param recID: record id
@param uid: user id
@param action: 'DISPLAY' to display add form
'SUBMIT' to submit comment once form is filled
'REPLY' to reply to an existing comment
@param ln: language
@param msg: the body of the comment/review or remark
@param score: star score of the review
@param note: title of the review
@param priority: priority of remark (int)
@param reviews: boolean, if enabled will add a review, if disabled will add a comment
@param comID: if replying, this is the comment id of the comment we are replying to
@param editor_type: the kind of editor/input used for the comment: 'textarea', 'ckeditor'
@param can_attach_files: if user can attach files to comments or not
@param subscribe: if True, subscribe user to receive new comments by email
@param req: request object. Used to register callback to send email notification
@param attached_files: newly attached files to this comment, mapping filename to filepath
@type attached_files: dict
@param warnings: list of warning tuples (warning_text, warning_color) that should be considered
@return:
- html add form if action is display or reply
- html successful added form if action is submit
"""
_ = gettext_set_language(ln)
if warnings is None:
warnings = []
actions = ['DISPLAY', 'REPLY', 'SUBMIT']
_ = gettext_set_language(ln)
## check arguments
check_recID_is_in_range(recID, warnings, ln)
if uid <= 0:
body = webcomment_templates.tmpl_error(
_('%s is an invalid user ID.') % uid, ln)
return body
if attached_files is None:
attached_files = {}
user_contact_info = query_get_user_contact_info(uid)
nickname = ''
if user_contact_info:
if user_contact_info[0]:
nickname = user_contact_info[0]
# show the form
if action == 'DISPLAY':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
return webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files)
else:
body = webcomment_templates.tmpl_error(
_('Comments on records have been disallowed by the '
'administrator.'), ln)
return body
elif action == 'REPLY':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
body = webcomment_templates.tmpl_error(
_('Cannot reply to a review.'), ln)
return body
elif not reviews and CFG_WEBCOMMENT_ALLOW_COMMENTS:
textual_msg = msg
if comID > 0:
comment = query_get_comment(comID)
if comment:
user_info = get_user_info(comment[2])
if user_info:
date_creation = convert_datetext_to_dategui(str(comment[4]))
# Build two msg: one mostly textual, the other one with HTML markup, for the CkEditor.
msg = _("%(x_name)s wrote on %(x_date)s:")% {'x_name': user_info[2], 'x_date': date_creation}
textual_msg = msg
# 1 For CkEditor input
msg += '\n\n'
msg += comment[3]
msg = email_quote_txt(text=msg)
# Now that we have a text-quoted version, transform into
# something that CkEditor likes, using that
# do still enable users to insert comments inline
msg = email_quoted_txt2html(text=msg,
indent_html=('', '
'),
linebreak_html="
",
indent_block=False)
# Add some space for users to easily add text
# around the quoted message
msg = '
' + msg + '
'
# Due to how things are done, we need to
# escape the whole msg again for the editor
msg = cgi.escape(msg)
# 2 For textarea input
textual_msg += "\n\n"
textual_msg += comment[3]
textual_msg = email_quote_txt(text=textual_msg)
return webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, textual_msg, can_attach_files=can_attach_files, reply_to=comID)
else:
body = webcomment_templates.tmpl_error(
_('Comments on records have been disallowed by the '
'administrator.'), ln)
return body
# check before submitting form
elif action == 'SUBMIT':
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
if note.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
warnings.append((_('You must enter a title.'), ''))
if score == 0 or score > 5:
warnings.append((_('You must choose a score.'), ''))
if msg.strip() in ["", "None"] and not CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
warnings.append((_('You must enter a text.'), ''))
# if no warnings, submit
if len(warnings) == 0:
if reviews:
if check_user_can_review(recID, client_ip_address, uid):
success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
note=note, score=score, priority=0,
client_ip_address=client_ip_address,
editor_type=editor_type,
req=req,
reply_to=comID)
else:
warnings.append((_('You already wrote a review for '
'this record.'), ''))
success = 1
else:
if check_user_can_comment(recID, client_ip_address, uid):
success = query_add_comment_or_remark(reviews, recID=recID, uid=uid, msg=msg,
note=note, score=score, priority=0,
client_ip_address=client_ip_address,
editor_type=editor_type,
req=req,
reply_to=comID, attached_files=attached_files)
if success > 0 and subscribe:
subscribe_user_to_discussion(recID, uid)
else:
warnings.append((_('You already posted a comment '
'short ago. Please retry later.'), ''))
success = 1
if success > 0:
if CFG_WEBCOMMENT_ADMIN_NOTIFICATION_LEVEL > 0:
notify_admin_of_new_comment(comID=success)
return webcomment_templates.tmpl_add_comment_successful(recID, ln, reviews, warnings, success)
else:
register_exception(req=req)
body = webcomment_templates.tmpl_error(
_('Failed to insert your comment to the database.'
' Please try again.'), ln)
return body
# if are warnings or if inserting comment failed, show user where warnings are
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, nickname, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
else:
return webcomment_templates.tmpl_add_comment_form(recID, uid, nickname, ln, msg, warnings, can_attach_files=can_attach_files)
# unknown action send to display
else:
warnings.append((_('Unknown action --> showing you the default '
'add comment form.'), ''))
if reviews and CFG_WEBCOMMENT_ALLOW_REVIEWS:
return webcomment_templates.tmpl_add_comment_form_with_ranking(recID, uid, ln, msg, score, note, warnings, can_attach_files=can_attach_files)
else:
return webcomment_templates.tmpl_add_comment_form(recID, uid, ln, msg, warnings, can_attach_files=can_attach_files)
return ''
def notify_admin_of_new_comment(comID):
"""
Sends an email to the admin with details regarding comment with ID = comID
"""
comment = query_get_comment(comID)
if len(comment) > 0:
(comID2,
id_bibrec,
id_user,
body,
date_creation,
star_score, nb_votes_yes, nb_votes_total,
title,
nb_abuse_reports, round_name, restriction) = comment
else:
return
user_info = query_get_user_contact_info(id_user)
if len(user_info) > 0:
(nickname, email, last_login) = user_info
if not len(nickname) > 0:
nickname = email.split('@')[0]
else:
nickname = email = last_login = "ERROR: Could not retrieve"
review_stuff = '''
Star score = %s
Title = %s''' % (star_score, title)
washer = EmailWasher()
try:
body = washer.wash(body)
except:
body = cgi.escape(body)
record_info = webcomment_templates.tmpl_email_new_comment_admin(id_bibrec)
out = '''
The following %(comment_or_review)s has just been posted (%(date)s).
AUTHOR:
Nickname = %(nickname)s
Email = %(email)s
User ID = %(uid)s
RECORD CONCERNED:
Record ID = %(recID)s
URL = <%(siteurl)s/%(CFG_SITE_RECORD)s/%(recID)s/%(comments_or_reviews)s/>
%(record_details)s
%(comment_or_review_caps)s:
%(comment_or_review)s ID = %(comID)s %(review_stuff)s
Body =
<--------------->
%(body)s
<--------------->
ADMIN OPTIONS:
To moderate the %(comment_or_review)s go to %(siteurl)s/%(CFG_SITE_RECORD)s/%(recID)s/%(comments_or_reviews)s/display?%(arguments)s
''' % \
{ 'comment_or_review' : star_score > 0 and 'review' or 'comment',
'comment_or_review_caps': star_score > 0 and 'REVIEW' or 'COMMENT',
'comments_or_reviews' : star_score > 0 and 'reviews' or 'comments',
'date' : date_creation,
'nickname' : nickname,
'email' : email,
'uid' : id_user,
'recID' : id_bibrec,
'record_details' : record_info,
'comID' : comID2,
'review_stuff' : star_score > 0 and review_stuff or "",
'body' : body.replace('
','\n'),
'siteurl' : CFG_SITE_URL,
'CFG_SITE_RECORD' : CFG_SITE_RECORD,
'arguments' : 'ln=en&do=od#%s' % comID
}
from_addr = '%s WebComment <%s>' % (CFG_SITE_NAME, CFG_WEBALERT_ALERT_ENGINE_EMAIL)
comment_collection = get_comment_collection(comID)
to_addrs = get_collection_moderators(comment_collection)
rec_collection = guess_primary_collection_of_a_record(id_bibrec)
report_nums = get_fieldvalues(id_bibrec, "037__a")
report_nums += get_fieldvalues(id_bibrec, "088__a")
report_nums = ', '.join(report_nums)
subject = "A new comment/review has just been posted [%s|%s]" % (rec_collection, report_nums)
send_email(from_addr, to_addrs, subject, out)
def check_recID_is_in_range(recID, warnings=[], ln=CFG_SITE_LANG):
"""
Check that recID is >= 0
@param recID: record id
@param warnings: list of warning tuples (warning_text, warning_color)
@return: tuple (boolean, html) where boolean (1=true, 0=false)
and html is the body of the page to display if there was a problem
"""
_ = gettext_set_language(ln)
try:
recID = int(recID)
except:
pass
if type(recID) is int:
if recID > 0:
from invenio.search_engine import record_exists
success = record_exists(recID)
if success == 1:
return (1, "")
else:
if success == -1:
status = 'deleted'
warning_message = _('The record has been deleted.')
else:
status = 'inexistant'
warning_message = _(
'Record ID %s does not exist in the database.') % recID
warnings.append((warning_message, ''))
return (0, webcomment_templates.tmpl_record_not_found(
status=status, recID=recID, ln=ln))
elif recID == 0:
warnings.append((_('No record ID was given.'), ''))
return (0, webcomment_templates.tmpl_record_not_found(
status='missing', recID=recID, ln=ln))
else:
warnings.append((_('Record ID %s is an invalid ID.') % recID, ''))
return (0, webcomment_templates.tmpl_record_not_found(
status='invalid', recID=recID, ln=ln))
else:
warnings.append((_('Record ID %s is not a number.') % recID, ''))
return (0, webcomment_templates.tmpl_record_not_found(
status='nan', recID=recID, ln=ln))
def check_int_arg_is_in_range(value, name, gte_value, lte_value=None):
"""
Check that variable with name 'name' >= gte_value & optionally <= lte_value
@param value: variable value
@param name: variable name
@param errors: list of error tuples (error_id, value)
@param gte_value: greater than or equal to value
@param lte_value: less than or equal to value
@return: boolean (1=true, 0=false)
"""
if type(value) is not int:
body = webcomment_templates.tmpl_error('%s is not a number.' % value)
return body
if value < gte_value:
body = webcomment_templates.tmpl_error('%s invalid argument.' % value)
return body
if lte_value:
if value > lte_value:
body = webcomment_templates.tmpl_error(
'%s invalid argument.' % value)
return body
return 1
def get_mini_reviews(recid, ln=CFG_SITE_LANG):
"""
Returns the web controls to add reviews to a record from the
detailed record pages mini-panel.
@param recid: the id of the displayed record
@param ln: the user's language
"""
if CFG_WEBCOMMENT_ALLOW_SHORT_REVIEWS:
action = 'SUBMIT'
else:
action = 'DISPLAY'
reviews = query_retrieve_comments_or_remarks(recid, ranking=1)
return webcomment_templates.tmpl_mini_review(recid, ln, action=action,
avg_score=calculate_avg_score(reviews),
nb_comments_total=len(reviews))
def check_user_can_view_comments(user_info, recid):
"""Check if the user is authorized to view comments for given
recid.
Returns the same type as acc_authorize_action
"""
# Check user can view the record itself first
(auth_code, auth_msg) = check_user_can_view_record(user_info, recid)
if auth_code:
return (auth_code, auth_msg)
# Check if user can view the comments
## But first can we find an authorization for this case action,
## for this collection?
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'viewcomment', authorized_if_no_roles=True, collection=record_primary_collection)
def check_user_can_view_comment(user_info, comid, restriction=None):
"""Check if the user is authorized to view a particular comment,
given the comment restriction. Note that this function does not
check if the record itself is restricted to the user, which would
mean that the user should not see the comment.
You can omit 'comid' if you already know the 'restriction'
@param user_info: the user info object
@param comid: the comment id of that we want to check
@param restriction: the restriction applied to given comment (if known. Otherwise retrieved automatically)
@return: the same type as acc_authorize_action
"""
if restriction is None:
comment = query_get_comment(comid)
if comment:
restriction = comment[11]
else:
return (1, 'Comment %i does not exist' % comid)
if restriction == "":
return (0, '')
return acc_authorize_action(user_info, 'viewrestrcomment', status=restriction)
def check_user_can_send_comments(user_info, recid):
"""Check if the user is authorized to comment the given
recid. This function does not check that user can view the record
or view the comments
Returns the same type as acc_authorize_action
"""
## First can we find an authorization for this case, action + collection
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'sendcomment', authorized_if_no_roles=True, collection=record_primary_collection)
def check_comment_belongs_to_record(comid, recid):
"""
Return True if the comment is indeed part of given record (even if comment or/and record have
been "deleted"). Else return False.
@param comid: the id of the comment to check membership
@param recid: the recid of the record we want to check if comment belongs to
"""
query = """SELECT id_bibrec from cmtRECORDCOMMENT WHERE id=%s"""
params = (comid,)
res = run_sql(query, params)
if res and res[0][0] == recid:
return True
return False
def check_user_can_attach_file_to_comments(user_info, recid):
"""Check if the user is authorized to attach a file to comments
for given recid. This function does not check that user can view
the comments or send comments.
Returns the same type as acc_authorize_action
"""
## First can we find an authorization for this case action, for
## this collection?
record_primary_collection = guess_primary_collection_of_a_record(recid)
return acc_authorize_action(user_info, 'attachcommentfile', authorized_if_no_roles=False, collection=record_primary_collection)
def toggle_comment_visibility(uid, comid, collapse, recid):
"""
Toggle the visibility of the given comment (collapse) for the
given user. Return the new visibility
@param uid: the user id for which the change applies
@param comid: the comment id to close/open
@param collapse: if the comment is to be closed (1) or opened (0)
@param recid: the record id to which the comment belongs
@return: if the comment is visible or not after the update
"""
# We rely on the client to tell if comment should be collapsed or
# developed, to ensure consistency between our internal state and
# client state. Even if not strictly necessary, we store the
# record ID for quicker retrieval of the collapsed comments of a
# given discussion page. To prevent unnecessary population of the
# table, only one distinct tuple (record ID, comment ID, user ID)
# can be inserted (due to table definition). For the same purpose
# we also check that comment to collapse exists, and corresponds
# to an existing record: we cannot rely on the recid found as part
# of the URL, as no former check is done. This rule is not applied
# when deleting an entry, as in the worst case no line would be
# removed. For optimized retrieval of row to delete, the id_bibrec
# column is used, though not strictly necessary.
if collapse:
query = """SELECT id_bibrec from cmtRECORDCOMMENT WHERE id=%s"""
params = (comid,)
res = run_sql(query, params)
if res:
query = """INSERT IGNORE INTO cmtCOLLAPSED (id_bibrec, id_cmtRECORDCOMMENT, id_user)
VALUES (%s, %s, %s)"""
params = (res[0][0], comid, uid)
run_sql(query, params)
return True
else:
query = """DELETE FROM cmtCOLLAPSED WHERE
id_cmtRECORDCOMMENT=%s and
id_user=%s and
id_bibrec=%s"""
params = (comid, uid, recid)
run_sql(query, params)
return False
def get_user_collapsed_comments_for_record(uid, recid):
"""
Get the comments collapsed for given user on given recid page
"""
# Collapsed state is not an attribute of cmtRECORDCOMMENT table
# (vary per user) so it cannot be found when querying for the
# comment. We must therefore provide a efficient way to retrieve
# the collapsed state for a given discussion page and user.
query = """SELECT id_cmtRECORDCOMMENT from cmtCOLLAPSED WHERE id_user=%s and id_bibrec=%s"""
params = (uid, recid)
return [res[0] for res in run_sql(query, params)]
def is_comment_deleted(comid):
"""
Return True of the comment is deleted. Else False
@param comid: ID of comment to check
"""
query = "SELECT status from cmtRECORDCOMMENT WHERE id=%s"
params = (comid,)
res = run_sql(query, params)
if res and res[0][0] != 'ok':
return True
return False
def perform_display_your_comments(user_info,
page_number=1,
selected_order_by_option="lcf",
selected_display_number_option="all",
selected_display_format_option="rc",
ln=CFG_SITE_LANG):
"""
Display all comments submitted by the user.
@TODO: support reviews too
@param user_info: standard user info object.
@param comments: ordered list of tuples (id_bibrec, comid, date_creation, body, status, in_reply_to_id_cmtRECORDCOMMENT)
@param page_number: page on which the user is.
@type page_number: integer
@param selected_order_by_option: seleccted ordering option. Can be one of:
- ocf: Oldest comment first
- lcf: Latest comment first
- grof: Group by record, oldest commented first
- grlf: Group by record, latest commented first
@type selected_order_by_option: string
@param selected_display_number_option: number of results to show per page. Can be a string-digit or 'all'.
@type selected_display_number_option: string
@param selected_display_format_option: how to show records. Can be one of:
- rc: Records and comments
- ro: Records only
- co: Comments only
@type selected_display_format_option: string
@ln: language
@type ln: string
"""
query_params = ""
nb_total_pages = 0
if selected_display_format_option in ('rc', 'co'):
nb_total_results = run_sql("SELECT count(id) from cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0", \
(user_info['uid'], ))[0][0]
else:
if selected_order_by_option in ('grlf', 'grof'):
nb_total_results = run_sql("SELECT count(distinct(id_bibrec)) from cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0", \
(user_info['uid'], ))[0][0]
else:
nb_total_results = run_sql("SELECT count(id_bibrec) from cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0", \
(user_info['uid'], ))[0][0]
if page_number < 1:
page_number = 1
if selected_display_number_option != 'all' and \
not selected_display_number_option.isdigit():
# must be some garbage
selected_display_number_option = 'all'
query = ''
if selected_order_by_option == "lcf":
query_params += " ORDER BY date_creation DESC"
elif selected_order_by_option == "ocf":
query_params += " ORDER BY date_creation ASC"
elif selected_order_by_option == "grlf":
query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT max(date_creation) as maxdatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.maxdatecreation DESC, cmt.date_creation DESC"
elif selected_order_by_option == "grof":
query = "SELECT cmt.id_bibrec, cmt.id, cmt.date_creation, cmt.body, cmt.status, cmt.in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT as cmt left join (SELECT min(date_creation) as mindatecreation, id_bibrec FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0 GROUP BY id_bibrec) as grp on cmt.id_bibrec = grp.id_bibrec WHERE id_user=%s AND star_score = 0 ORDER BY grp.mindatecreation ASC"
if selected_display_number_option.isdigit():
selected_display_number_option_as_int = int(selected_display_number_option)
if selected_display_number_option_as_int < 5:
selected_display_number_option_as_int = 5
selected_display_number_option = str(selected_display_number_option_as_int)
from_index = (page_number - 1) * int(selected_display_number_option)
query_params += ' LIMIT ' + \
str(from_index) + \
',' + \
str(int(selected_display_number_option))
nb_total_pages = int(math.ceil(float(nb_total_results) / selected_display_number_option_as_int))
if selected_order_by_option in ("grlf", "grof"):
res = run_sql(query + query_params, (user_info['uid'], user_info['uid']))
else:
res = run_sql("SELECT id_bibrec, id, date_creation, body, status, in_reply_to_id_cmtRECORDCOMMENT FROM cmtRECORDCOMMENT WHERE id_user=%s AND star_score = 0" + query_params, (user_info['uid'], ))
return webcomment_templates.tmpl_your_comments(user_info, res,
page_number=page_number,
selected_order_by_option=selected_order_by_option,
selected_display_number_option=selected_display_number_option,
selected_display_format_option=selected_display_format_option,
nb_total_results=nb_total_results,
nb_total_pages=nb_total_pages,
ln=ln)
diff --git a/modules/webcomment/lib/webcommentadminlib.py b/modules/webcomment/lib/webcommentadminlib.py
index fac14b4b6..b586d7ccd 100644
--- a/modules/webcomment/lib/webcommentadminlib.py
+++ b/modules/webcomment/lib/webcommentadminlib.py
@@ -1,690 +1,690 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
-# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
+# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__revision__ = "$Id$"
from invenio.config import CFG_SITE_LANG, CFG_SITE_URL, \
CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN
from invenio.webcomment_config import InvenioWebCommentWarning
from invenio.webcomment import query_get_comment, \
get_reply_order_cache_data
from invenio.urlutils import wash_url_argument
from invenio.dbquery import run_sql
from invenio.messages import gettext_set_language, wash_language
from invenio.errorlib import register_exception
from invenio.webuser import get_user_info, collect_user_info, \
isUserAdmin
from invenio.access_control_engine import acc_authorize_action, \
acc_get_authorized_emails
from invenio.search_engine import perform_request_search
import invenio.template
webcomment_templates = invenio.template.load('webcomment')
def getnavtrail(previous = '', ln=CFG_SITE_LANG):
"""Get the navtrail"""
previous = wash_url_argument(previous, 'str')
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail = """%s """ % (CFG_SITE_URL, _("Admin Area"))
navtrail = navtrail + previous
return navtrail
def get_nb_reviews(recID, count_deleted=True):
"""
Return number of reviews for the record recID
if count_deleted is True, deleted reviews are also counted
"""
query = """SELECT count(*)
FROM cmtRECORDCOMMENT c
WHERE c.id_bibrec = %s and c.star_score > 0
"""
if not count_deleted:
query += "and c.status != 'dm' and c.status != 'da'"
res = run_sql(query, (recID,))
return res[0][0]
def get_nb_comments(recID, count_deleted=True):
"""
Return number of comments for the record recID
if count_deleted is True, deleted comments are also counted
"""
query = """SELECT count(*)
FROM cmtRECORDCOMMENT c
WHERE c.id_bibrec = %s and c.star_score = 0
"""
if not count_deleted:
query += "and c.status != 'dm' and c.status != 'da'"
res = run_sql(query, (recID,))
return res[0][0]
def get_user_collections(req):
"""
Return collections for which the user is moderator
"""
user_info = collect_user_info(req)
res = []
collections = run_sql('SELECT name FROM collection')
for collection in collections:
collection_emails = acc_get_authorized_emails('moderatecomments', collection=collection[0])
if user_info['email'] in collection_emails or isUserAdmin(user_info):
res.append(collection[0])
return res
def perform_request_index(ln=CFG_SITE_LANG):
"""
"""
return webcomment_templates.tmpl_admin_index(ln=ln)
def perform_request_delete(comID=-1, recID=-1, uid=-1, reviews="", ln=CFG_SITE_LANG):
"""
"""
_ = gettext_set_language(ln)
from invenio.search_engine import record_exists
warnings = []
ln = wash_language(ln)
comID = wash_url_argument(comID, 'int')
recID = wash_url_argument(recID, 'int')
uid = wash_url_argument(uid, 'int')
# parameter reviews is deduced from comID when needed
if comID is not None and recID is not None and uid is not None:
if comID <= 0 and recID <= 0 and uid <= 0:
if comID != -1:
try:
raise InvenioWebCommentWarning(_('Invalid comment ID.'))
except InvenioWebCommentWarning, exc:
register_exception(stream='warning')
warnings.append((exc.message, ''))
#warnings.append(("WRN_WEBCOMMENT_ADMIN_INVALID_COMID",))
return webcomment_templates.tmpl_admin_delete_form(ln, warnings)
if comID > 0 and not recID > 0:
comment = query_get_comment(comID)
if comment:
# Figure out if this is a review or a comment
c_star_score = 5
if comment[c_star_score] > 0:
reviews = 1
else:
reviews = 0
return (perform_request_comments(ln=ln, comID=comID, recID=recID, reviews=reviews), None, warnings)
else:
try:
raise InvenioWebCommentWarning(_('Comment ID %s does not exist.') % comID)
except InvenioWebCommentWarning, exc:
register_exception(stream='warning')
warnings.append((exc.message, ''))
#warnings.append(('WRN_WEBCOMMENT_ADMIN_COMID_INEXISTANT', comID))
return webcomment_templates.tmpl_admin_delete_form(ln, warnings)
elif recID > 0:
if record_exists(recID):
comID = ''
reviews = wash_url_argument(reviews, 'int')
return (perform_request_comments(ln=ln, comID=comID, recID=recID, reviews=reviews), None, warnings)
else:
try:
raise InvenioWebCommentWarning(_('Record ID %s does not exist.') % comID)
except InvenioWebCommentWarning, exc:
register_exception(stream='warning')
warnings.append((exc.message, ''))
#warnings.append(('WRN_WEBCOMMENT_ADMIN_RECID_INEXISTANT', comID))
return webcomment_templates.tmpl_admin_delete_form(ln, warnings)
else:
return webcomment_templates.tmpl_admin_delete_form(ln, warnings)
else:
return webcomment_templates.tmpl_admin_delete_form(ln, warnings)
def perform_request_users(ln=CFG_SITE_LANG):
"""
"""
ln = wash_language(ln)
users_data = query_get_users_reported()
return webcomment_templates.tmpl_admin_users(ln=ln, users_data=users_data)
def query_get_users_reported():
"""
Get the users who have been reported at least once.
@return: tuple of ct, i.e. (ct, ct, ...)
where ct is a tuple (total_number_reported, total_comments_reported, total_reviews_reported,
total_nb_votes_yes_of_reported, total_nb_votes_total_of_reported, user_id, user_email, user_nickname)
sorted by order of ct having highest total_number_reported
"""
query1 = "SELECT c.nb_abuse_reports, c.nb_votes_yes, c.nb_votes_total, u.id, u.email, u.nickname, c.star_score " \
"FROM user AS u, cmtRECORDCOMMENT AS c " \
"WHERE c.id_user=u.id AND c.nb_abuse_reports > 0 " \
"ORDER BY u.id "
res1 = run_sql(query1)
if type(res1) is None:
return ()
users = {}
for cmt in res1:
uid = int(cmt[3])
if users.has_key(uid):
users[uid] = (users[uid][0]+int(cmt[0]), int(cmt[6])>0 and users[uid][1] or users[uid][1]+1, int(cmt[6])>0 and users[uid][2]+1 or users[uid][2],
users[uid][3]+int(cmt[1]), users[uid][4]+int(cmt[2]), int(cmt[3]), cmt[4], cmt[5])
else:
users[uid] = (int(cmt[0]), int(cmt[6])==0 and 1 or 0, int(cmt[6])>0 and 1 or 0, int(cmt[1]), int(cmt[2]), int(cmt[3]), cmt[4], cmt[5])
users = users.values()
users.sort()
users.reverse()
users = tuple(users)
return users
def perform_request_comments(req=None, ln=CFG_SITE_LANG, uid="", comID="", recID="", reviews=0, abuse=False, collection=""):
"""
Display the list of comments/reviews along with information about the comment.
Display the comment given by its ID, or the list of comments for
the given record ID.
If abuse == True, only list records reported as abuse.
If comID and recID are not provided, list all comments, or all
abused comments (check parameter 'abuse')
"""
ln = wash_language(ln)
uid = wash_url_argument(uid, 'int')
comID = wash_url_argument(comID, 'int')
recID = wash_url_argument(recID, 'int')
reviews = wash_url_argument(reviews, 'int')
collection = wash_url_argument(collection, 'str')
user_info = collect_user_info(req)
user_collections = ['Show all']
user_collections.extend(get_user_collections(req))
if collection and collection != 'Show all':
(auth_code, auth_msg) = acc_authorize_action(req, 'moderatecomments', collection=collection)
if auth_code:
return webcomment_templates.tmpl_admin_comments(ln=ln, uid=uid,
comID=comID,
recID=recID,
comment_data=None,
reviews=reviews,
error=1,
user_collections=user_collections,
collection=collection)
if collection:
if recID or uid:
comments = query_get_comments(uid, comID, recID, reviews, ln, abuse=abuse, user_collections=user_collections, collection=collection)
else:
comments = query_get_comments('', comID, '', reviews, ln, abuse=abuse, user_collections=user_collections, collection=collection)
else:
if recID or uid:
comments = query_get_comments(uid, comID, recID, reviews, ln, abuse=abuse, user_collections=user_collections, collection=user_collections[0])
else:
comments = query_get_comments('', comID, '', reviews, ln, abuse=abuse, user_collections=user_collections, collection=user_collections[0])
if comments:
return webcomment_templates.tmpl_admin_comments(ln=ln, uid=uid,
comID=comID,
recID=recID,
comment_data=comments,
reviews=reviews,
error=0,
user_collections=user_collections,
collection=collection)
else:
return webcomment_templates.tmpl_admin_comments(ln=ln, uid=uid,
comID=comID,
recID=recID,
comment_data=comments,
reviews=reviews,
error=2,
user_collections=user_collections,
collection=collection)
def perform_request_hot(req=None, ln=CFG_SITE_LANG, comments=1, top=10, collection="Show all"):
"""
Display the list of hottest comments/reviews along with information about the comment.
@param req: request object for obtaining user information
@param ln: language
@param comments: boolean activated if using comments, deactivated for reviews
@param top: specify number of results to be shown
@param collection: filter by collection
"""
ln = wash_language(ln)
comments = wash_url_argument(comments, 'int')
top = wash_url_argument(top, 'int')
collection = wash_url_argument(collection, 'str')
user_info = collect_user_info(req)
user_collections = ['Show all']
user_collections.extend(get_user_collections(req))
if collection and collection != 'Show all':
(auth_code, auth_msg) = acc_authorize_action(req, 'moderatecomments', collection=collection)
if auth_code:
return webcomment_templates.tmpl_admin_hot(ln=ln,
comment_data = None,
comments=comments, error=1, user_collections=user_collections, collection=collection)
if collection:
comments_retrieved = query_get_hot(comments, ln, top, user_collections, collection)
else:
comments_retrieved = query_get_hot(comments, ln, top, user_collections, user_collections[0])
if comments_retrieved:
return webcomment_templates.tmpl_admin_hot(ln=ln,
comment_data=comments_retrieved,
comments=comments, error=0, user_collections=user_collections, collection=collection)
else:
return webcomment_templates.tmpl_admin_hot(ln=ln,
comment_data=comments_retrieved,
comments=comments, error=2, user_collections=user_collections, collection=collection)
def perform_request_latest(req=None, ln=CFG_SITE_LANG, comments=1, top=10, collection=""):
"""
Display the list of latest comments/reviews along with information about the comment.
@param req: request object for obtaining user information
@param ln: language
@param comments: boolean activated if using comments, deactivated for reviews
@param top: Specify number of results to be shown
@param collection: filter by collection
"""
ln = wash_language(ln)
comments = wash_url_argument(comments, 'int')
top = wash_url_argument(top, 'int')
collection = wash_url_argument(collection, 'str')
user_info = collect_user_info(req)
user_collections = ['Show all']
user_collections.extend(get_user_collections(req))
if collection and collection != 'Show all':
(auth_code, auth_msg) = acc_authorize_action(req, 'moderatecomments', collection=collection)
if auth_code:
return webcomment_templates.tmpl_admin_latest(ln=ln,
comment_data=None,
comments=comments, error=1, user_collections=user_collections, collection=collection)
if collection:
comments_retrieved = query_get_latest(comments, ln, top, user_collections, collection)
else:
comments_retrieved = query_get_latest(comments, ln, top, user_collections, user_collections[0])
if comments_retrieved:
return webcomment_templates.tmpl_admin_latest(ln=ln,
comment_data=comments_retrieved,
comments=comments, error=0, user_collections=user_collections, collection=collection)
else:
return webcomment_templates.tmpl_admin_latest(ln=ln,
comment_data=comments_retrieved,
comments=comments, error=2, user_collections=user_collections, collection=collection)
def perform_request_undel_single_com(ln=CFG_SITE_LANG, id=id):
"""
Mark comment referenced by id as active
"""
ln = wash_language(ln)
id = wash_url_argument(id, 'int')
return query_undel_single_comment(id)
def query_get_comments(uid, cmtID, recID, reviews, ln, abuse=False, user_collections='', collection=''):
"""
private function
@param user_collections: allowed collections for the user
@param collection: collection to display
@return tuple of comment where comment is
tuple (nickname, uid, date_creation, body, id, status) if ranking disabled or
tuple (nickname, uid, date_creation, body, nb_votes_yes, nb_votes_total, star_score, title, id, status)
"""
qdict = {'id': 0, 'id_bibrec': 1, 'uid': 2, 'date_creation': 3, 'body': 4,
'status': 5, 'nb_abuse_reports': 6, 'nb_votes_yes': 7, 'nb_votes_total': 8,
'star_score': 9, 'title': 10, 'email': -2, 'nickname': -1}
query = """SELECT c.id, c.id_bibrec, c.id_user,
DATE_FORMAT(c.date_creation, '%%Y-%%m-%%d %%H:%%i:%%S'), c.body,
c.status, c.nb_abuse_reports,
%s
u.email, u.nickname
FROM cmtRECORDCOMMENT c LEFT JOIN user u
ON c.id_user = u.id
%s
ORDER BY c.nb_abuse_reports DESC, c.nb_votes_yes DESC, c.date_creation
"""
select_fields = reviews and 'c.nb_votes_yes, c.nb_votes_total, c.star_score, c.title,' or ''
where_clause = "WHERE " + (reviews and 'c.star_score>0' or 'c.star_score=0')
if uid:
where_clause += ' AND c.id_user=%i' % uid
if recID:
where_clause += ' AND c.id_bibrec=%i' % recID
if cmtID:
where_clause += ' AND c.id=%i' % cmtID
if abuse:
where_clause += ' AND c.nb_abuse_reports>0'
res = run_sql(query % (select_fields, where_clause))
collection_records = []
if collection == 'Show all':
for collection_name in user_collections:
collection_records.extend(perform_request_search(cc=collection_name))
else:
collection_records.extend(perform_request_search(cc=collection))
output = []
for qtuple in res:
if qtuple[qdict['id_bibrec']] in collection_records:
nickname = qtuple[qdict['nickname']] or get_user_info(qtuple[qdict['uid']], ln)[2]
if reviews:
comment_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['date_creation']],
qtuple[qdict['body']],
qtuple[qdict['nb_votes_yes']],
qtuple[qdict['nb_votes_total']],
qtuple[qdict['star_score']],
qtuple[qdict['title']],
qtuple[qdict['id']],
qtuple[qdict['status']])
else:
comment_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['date_creation']],
qtuple[qdict['body']],
qtuple[qdict['id']],
qtuple[qdict['status']])
general_infos_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['email']],
qtuple[qdict['id']],
qtuple[qdict['id_bibrec']],
qtuple[qdict['nb_abuse_reports']])
out_tuple = (comment_tuple, general_infos_tuple)
output.append(out_tuple)
return tuple(output)
def query_get_hot(comments, ln, top, user_collections, collection):
"""
private function
@param comments: boolean indicating if we want to retrieve comments or reviews
@param ln: language
@param top: number of results to display
@param user_collections: allowed collections for the user
@param collection: collection to display
@return: tuple (id_bibrec, date_last_comment, users, count)
"""
qdict = {'id_bibrec': 0, 'date_last_comment': 1, 'users': 2, 'total_count': 3}
query = """SELECT c.id_bibrec,
DATE_FORMAT(max(c.date_creation), '%%Y-%%m-%%d %%H:%%i:%%S') as date_last_comment,
count(distinct c.id_user) as users,
count(*) as count
FROM cmtRECORDCOMMENT c
%s
GROUP BY c.id_bibrec
ORDER BY count(*) DESC
LIMIT %s
"""
where_clause = "WHERE " + (comments and 'c.star_score=0' or 'c.star_score>0') + ' AND c.status="ok" AND c.nb_abuse_reports < %s' % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN
res = run_sql(query % (where_clause, top))
collection_records = []
if collection == 'Show all':
for collection_name in user_collections:
collection_records.extend(perform_request_search(cc=collection_name))
else:
collection_records.extend(perform_request_search(cc=collection))
output = []
for qtuple in res:
if qtuple[qdict['id_bibrec']] in collection_records:
general_infos_tuple = (qtuple[qdict['id_bibrec']],
qtuple[qdict['date_last_comment']],
qtuple[qdict['users']],
qtuple[qdict['total_count']])
output.append(general_infos_tuple)
return tuple(output)
def query_get_latest(comments, ln, top, user_collections, collection):
"""
private function
@param comments: boolean indicating if we want to retrieve comments or reviews
@param ln: language
@param top: number of results to display
@param user_collections: allowed collections for the user
@param collection: collection to display
@return tuple of comment where comment is
tuple (nickname, uid, date_creation, body, id) if latest comments or
tuple (nickname, uid, date_creation, body, star_score, id) if latest reviews
"""
qdict = {'id': 0, 'id_bibrec': 1, 'uid': 2, 'date_creation': 3, 'body': 4,
'nb_abuse_reports': 5, 'star_score': 6, 'nickname': -1}
query = """SELECT c.id, c.id_bibrec, c.id_user,
DATE_FORMAT(c.date_creation, '%%Y-%%m-%%d %%H:%%i:%%S'), c.body,
c.nb_abuse_reports,
%s
u.nickname
FROM cmtRECORDCOMMENT c LEFT JOIN user u
ON c.id_user = u.id
%s
ORDER BY c.date_creation DESC
LIMIT %s
"""
select_fields = not comments and 'c.star_score, ' or ''
where_clause = "WHERE " + (comments and 'c.star_score=0' or 'c.star_score>0') + ' AND c.status="ok" AND c.nb_abuse_reports < %s' % CFG_WEBCOMMENT_NB_REPORTS_BEFORE_SEND_EMAIL_TO_ADMIN
res = run_sql(query % (select_fields, where_clause, top))
collection_records = []
if collection == 'Show all':
for collection_name in user_collections:
collection_records.extend(perform_request_search(cc=collection_name))
else:
collection_records.extend(perform_request_search(cc=collection))
output = []
for qtuple in res:
if qtuple[qdict['id_bibrec']] in collection_records:
nickname = qtuple[qdict['nickname']] or get_user_info(qtuple[qdict['uid']], ln)[2]
if not comments:
comment_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['date_creation']],
qtuple[qdict['body']],
qtuple[qdict['star_score']],
qtuple[qdict['id']])
else:
comment_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['date_creation']],
qtuple[qdict['body']],
qtuple[qdict['id']])
general_infos_tuple = (nickname,
qtuple[qdict['uid']],
qtuple[qdict['id']],
qtuple[qdict['id_bibrec']],
qtuple[qdict['nb_abuse_reports']])
out_tuple = (comment_tuple, general_infos_tuple)
output.append(out_tuple)
return tuple(output)
def perform_request_del_com(ln=CFG_SITE_LANG, comIDs=[]):
"""
private function
Delete the comments and say whether successful or not
@param ln: language
@param comIDs: list of comment ids
"""
ln = wash_language(ln)
comIDs = wash_url_argument(comIDs, 'list')
# map ( fct, list, arguments of function)
comIDs = map(wash_url_argument, comIDs, ('int '*len(comIDs)).split(' ')[:-1])
if not comIDs:
comIDs = map(coerce, comIDs, ('0 '*len(comIDs)).split(' ')[:-1])
return webcomment_templates.tmpl_admin_del_com(del_res=comIDs, ln=ln)
del_res = []
for comID in comIDs:
del_res.append((comID, query_delete_comment_mod(comID)))
return webcomment_templates.tmpl_admin_del_com(del_res=del_res, ln=ln)
def perform_request_undel_com(ln=CFG_SITE_LANG, comIDs=[]):
"""
private function
Undelete the comments and say whether successful or not
@param ln: language
@param comIDs: list of comment ids
"""
ln = wash_language(ln)
comIDs = wash_url_argument(comIDs, 'list')
# map ( fct, list, arguments of function)
comIDs = map(wash_url_argument, comIDs, ('int '*len(comIDs)).split(' ')[:-1])
if not comIDs:
comIDs = map(coerce, comIDs, ('0 '*len(comIDs)).split(' ')[:-1])
return webcomment_templates.tmpl_admin_undel_com(del_res=comIDs, ln=ln)
del_res = []
for comID in comIDs:
del_res.append((comID, query_undel_single_comment(comID)))
return webcomment_templates.tmpl_admin_undel_com(del_res=del_res, ln=ln)
def perform_request_del_single_com_mod(ln=CFG_SITE_LANG, id=id):
"""
private function
Delete a single comment requested by a moderator
@param ln: language
@param id: comment id to be deleted
"""
ln = wash_language(ln)
id = wash_url_argument(id, 'int')
return query_delete_comment_mod(id)
def perform_request_del_single_com_auth(ln=CFG_SITE_LANG, id=id):
"""
private function
Delete a single comment requested by the author
@param ln: language
@param id: comment id to be deleted
"""
ln = wash_language(ln)
id = wash_url_argument(id, 'int')
return query_delete_comment_auth(id)
def perform_request_unreport_single_com(ln=CFG_SITE_LANG, id=""):
"""
private function
Unreport a single comment
@param ln: language
@param id: comment id to be deleted
"""
ln = wash_language(ln)
id = wash_url_argument(id, 'int')
return query_suppress_abuse_report(id)
def suppress_abuse_report(ln=CFG_SITE_LANG, comIDs=[]):
"""
private function
suppress the abuse reports for the given comIDs.
@param ln: language
@param comIDs: list of ids to suppress attached reports.
"""
ln = wash_language(ln)
comIDs = wash_url_argument(comIDs, 'list')
# map ( fct, list, arguments of function)
comIDs = map(wash_url_argument, comIDs, ('int '*len(comIDs)).split(' ')[:-1])
if not comIDs:
comIDs = map(coerce, comIDs, ('0 '*len(comIDs)).split(' ')[:-1])
return webcomment_templates.tmpl_admin_del_com(del_res=comIDs, ln=ln)
del_res = []
for comID in comIDs:
del_res.append((comID, query_suppress_abuse_report(comID)))
return webcomment_templates.tmpl_admin_suppress_abuse_report(del_res=del_res, ln=ln)
def query_suppress_abuse_report(comID):
""" suppress abuse report for a given comment
@return: integer 1 if successful, integer 0 if not
"""
query = "UPDATE cmtRECORDCOMMENT SET nb_abuse_reports=0, status='ap' WHERE id=%s"
params = (comID,)
res = run_sql(query, params)
return int(res)
def query_delete_comment_mod(comID):
"""
delete comment with id comID
@return: integer 1 if successful, integer 0 if not
"""
query1 = "UPDATE cmtRECORDCOMMENT SET status='dm' WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
return int(res1)
def query_delete_comment_auth(comID):
"""
delete comment with id comID
@return: integer 1 if successful, integer 0 if not
"""
query1 = "UPDATE cmtRECORDCOMMENT SET status='da' WHERE id=%s"
params1 = (comID,)
res1 = run_sql(query1, params1)
return int(res1)
def query_undel_single_comment(comID):
"""
undelete comment with id comID
@return: integer 1 if successful, integer 0 if not
"""
query = "UPDATE cmtRECORDCOMMENT SET status='ok' WHERE id=%s"
params = (comID,)
res = run_sql(query, params)
return int(res)
def check_user_is_author(user_id, com_id):
""" Check if the user is the author of the given comment """
res = run_sql("SELECT id, id_user FROM cmtRECORDCOMMENT WHERE id=%s and id_user=%s", (str(com_id), str(user_id)))
if res:
return 1
return 0
def migrate_comments_populate_threads_index():
"""
Fill in the `reply_order_cached_data' columns in cmtRECORDCOMMENT and
bskRECORDCOMMENT tables with adequate values so that thread
are displayed correctly.
"""
# Update WebComment comments
res = run_sql("SELECT id FROM cmtRECORDCOMMENT WHERE reply_order_cached_data is NULL")
for row in res:
reply_order_cached_data = get_reply_order_cache_data(row[0])
- run_sql("UPDATE cmtRECORDCOMMENT set reply_order_cached_data=%s WHERE id=%s",
+ run_sql("UPDATE cmtRECORDCOMMENT set reply_order_cached_data=_binary %s WHERE id=%s",
(reply_order_cached_data, row[0]))
# Update WebBasket comments
res = run_sql("SELECT id FROM bskRECORDCOMMENT WHERE reply_order_cached_data is NULL")
for row in res:
reply_order_cached_data = get_reply_order_cache_data(row[0])
- run_sql("UPDATE cmtRECORDCOMMENT set reply_order_cached_data=%s WHERE id=%s",
+ run_sql("UPDATE cmtRECORDCOMMENT set reply_order_cached_data=_binary %s WHERE id=%s",
(reply_order_cached_data, row[0]))
diff --git a/modules/websearch/lib/websearch_webcoll.py b/modules/websearch/lib/websearch_webcoll.py
index 16662eb90..259a3d7b9 100644
--- a/modules/websearch/lib/websearch_webcoll.py
+++ b/modules/websearch/lib/websearch_webcoll.py
@@ -1,1224 +1,1224 @@
# This file is part of Invenio.
-# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 CERN.
+# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Create Invenio collection cache."""
__revision__ = "$Id$"
import calendar
import copy
import sys
import cgi
import re
import os
import string
import time
import cPickle
from invenio.config import \
CFG_CERN_SITE, \
CFG_WEBSEARCH_INSTANT_BROWSE, \
CFG_WEBSEARCH_NARROW_SEARCH_SHOW_GRANDSONS, \
CFG_WEBSEARCH_I18N_LATEST_ADDITIONS, \
CFG_CACHEDIR, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_SITE_LANGS, \
CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE, \
CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS, \
CFG_SCOAP3_SITE
from invenio.messages import gettext_set_language
from invenio.search_engine import search_pattern_parenthesised, get_creation_date, get_field_i18nname, collection_restricted_p, sort_records, EM_REPOSITORY
from invenio.search_engine_config import CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES
from invenio.dbquery import run_sql, Error, get_table_update_time
from invenio.bibrank_record_sorter import get_bibrank_methods
from invenio.dateutils import convert_datestruct_to_dategui, strftime
from invenio.bibformat import format_record
from invenio.shellutils import mymkdir
from invenio.intbitset import intbitset
from invenio.websearch_external_collections import \
external_collection_load_states, \
dico_collection_external_searches, \
external_collection_sort_engine_by_name
from invenio.bibtask import task_init, task_get_option, task_set_option, \
write_message, task_has_option, task_update_progress, \
task_sleep_now_if_required
import invenio.template
websearch_templates = invenio.template.load('websearch')
from invenio.websearch_external_collections_searcher import external_collections_dictionary
from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_TIMEOUT
from invenio.websearch_external_collections_config import CFG_HOSTED_COLLECTION_TIMEOUT_NBRECS
# global vars
COLLECTION_HOUSE = {} # will hold collections we treat in this run of the program; a dict of {collname2, collobject1}, ...
# CFG_CACHE_LAST_UPDATED_TIMESTAMP_TOLERANCE -- cache timestamp
# tolerance (in seconds), to account for the fact that an admin might
# accidentally happen to edit the collection definitions at exactly
# the same second when some webcoll process was about to be started.
# In order to be safe, let's put an exaggerated timestamp tolerance
# value such as 20 seconds:
CFG_CACHE_LAST_UPDATED_TIMESTAMP_TOLERANCE = 20
# CFG_CACHE_LAST_UPDATED_TIMESTAMP_FILE -- location of the cache
# timestamp file:
CFG_CACHE_LAST_UPDATED_TIMESTAMP_FILE = "%s/collections/last_updated" % CFG_CACHEDIR
# CFG_CACHE_LAST_FAST_UPDATED_TIMESTAMP_FILE -- location of the cache
# timestamp file usef when running webcoll in the fast-mode.
CFG_CACHE_LAST_FAST_UPDATED_TIMESTAMP_FILE = "%s/collections/last_fast_updated" % CFG_CACHEDIR
def get_collection(colname):
"""Return collection object from the collection house for given colname.
If does not exist, then create it."""
if not COLLECTION_HOUSE.has_key(colname):
colobject = Collection(colname)
COLLECTION_HOUSE[colname] = colobject
return COLLECTION_HOUSE[colname]
# auxiliary functions:
def is_selected(var, fld):
"Checks if the two are equal, and if yes, returns ' selected'. Useful for select boxes."
if var == fld:
return ' selected="selected"'
else:
return ""
def get_field(recID, tag):
"Gets list of field 'tag' for the record with 'recID' system number."
out = []
digit = tag[0:2]
bx = "bib%sx" % digit
bibx = "bibrec_bib%sx" % digit
query = "SELECT bx.value FROM %s AS bx, %s AS bibx WHERE bibx.id_bibrec='%s' AND bx.id=bibx.id_bibxxx AND bx.tag='%s'" \
% (bx, bibx, recID, tag)
res = run_sql(query)
for row in res:
out.append(row[0])
return out
def check_nbrecs_for_all_external_collections():
"""Check if any of the external collections have changed their total number of records, aka nbrecs.
Return True if any of the total numbers of records have changed and False if they're all the same."""
res = run_sql("SELECT name FROM collection WHERE dbquery LIKE 'hostedcollection:%';")
for row in res:
coll_name = row[0]
if (get_collection(coll_name)).check_nbrecs_for_external_collection():
return True
return False
class Collection:
"Holds the information on collections (id,name,dbquery)."
def __init__(self, name=""):
"Creates collection instance by querying the DB configuration database about 'name'."
self.calculate_reclist_run_already = 0 # to speed things up without much refactoring
self.update_reclist_run_already = 0 # to speed things up without much refactoring
self.reclist_updated_since_start = 0 # to check if webpage cache need rebuilding
self.reclist_with_nonpublic_subcolls = intbitset()
# temporary counters for the number of records in hosted collections
self.nbrecs_tmp = None # number of records in a hosted collection
self.nbrecs_from_hosted_collections = 0 # total number of records from
# descendant hosted collections
if not name:
self.name = CFG_SITE_NAME # by default we are working on the home page
self.id = 1
self.dbquery = None
self.nbrecs = None
self.reclist = intbitset()
self.old_reclist = intbitset()
self.reclist_updated_since_start = 1
else:
self.name = name
try:
res = run_sql("""SELECT id,name,dbquery,nbrecs,reclist FROM collection
WHERE name=%s""", (name,))
if res:
self.id = res[0][0]
self.name = res[0][1]
self.dbquery = res[0][2]
self.nbrecs = res[0][3]
try:
self.reclist = intbitset(res[0][4])
except:
self.reclist = intbitset()
self.reclist_updated_since_start = 1
else: # collection does not exist!
self.id = None
self.dbquery = None
self.nbrecs = None
self.reclist = intbitset()
self.reclist_updated_since_start = 1
self.old_reclist = intbitset(self.reclist)
except Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit(1)
def get_example_search_queries(self):
"""Returns list of sample search queries for this collection.
"""
res = run_sql("""SELECT example.body FROM example
LEFT JOIN collection_example on example.id=collection_example.id_example
WHERE collection_example.id_collection=%s ORDER BY collection_example.score""", (self.id,))
return [query[0] for query in res]
def get_name(self, ln=CFG_SITE_LANG, name_type="ln", prolog="", epilog="", prolog_suffix=" ", epilog_suffix=""):
"""Return nicely formatted collection name for language LN.
The NAME_TYPE may be 'ln' (=long name), 'sn' (=short name), etc."""
out = prolog
i18name = ""
res = run_sql("SELECT value FROM collectionname WHERE id_collection=%s AND ln=%s AND type=%s", (self.id, ln, name_type))
try:
i18name += res[0][0]
except IndexError:
pass
if i18name:
out += i18name
else:
out += self.name
out += epilog
return out
def get_collectionbox_name(self, ln=CFG_SITE_LANG, box_type="r"):
"""
Return collection-specific labelling of 'Focus on' (regular
collection), 'Narrow by' (virtual collection) and 'Latest
addition' boxes.
If translation for given language does not exist, use label
for CFG_SITE_LANG. If no custom label is defined for
CFG_SITE_LANG, return default label for the box.
@param ln: the language of the label
@param box_type: can be 'r' (=Narrow by), 'v' (=Focus on), 'l' (=Latest additions)
"""
i18name = ""
res = run_sql("SELECT value FROM collectionboxname WHERE id_collection=%s AND ln=%s AND type=%s", (self.id, ln, box_type))
try:
i18name = res[0][0]
except IndexError:
res = run_sql("SELECT value FROM collectionboxname WHERE id_collection=%s AND ln=%s AND type=%s", (self.id, CFG_SITE_LANG, box_type))
try:
i18name = res[0][0]
except IndexError:
pass
if not i18name:
# load the right message language
_ = gettext_set_language(ln)
if box_type == "v":
i18name = _('Focus on:')
elif box_type == "r":
if CFG_SCOAP3_SITE:
i18name = _('Narrow by publisher/journal:')
else:
i18name = _('Narrow by collection:')
elif box_type == "l":
i18name = _('Latest additions:')
return i18name
def get_ancestors(self):
"Returns list of ancestors of the current collection."
ancestors = []
ancestors_ids = intbitset()
id_son = self.id
while 1:
query = "SELECT cc.id_dad,c.name FROM collection_collection AS cc, collection AS c "\
"WHERE cc.id_son=%d AND c.id=cc.id_dad" % int(id_son)
res = run_sql(query, None, 1)
if res:
col_ancestor = get_collection(res[0][1])
# looking for loops
if self.id in ancestors_ids:
write_message("Loop found in collection %s" % self.name, stream=sys.stderr)
raise OverflowError("Loop found in collection %s" % self.name)
else:
ancestors.append(col_ancestor)
ancestors_ids.add(col_ancestor.id)
id_son = res[0][0]
else:
break
ancestors.reverse()
return ancestors
def restricted_p(self):
"""Predicate to test if the collection is restricted or not. Return the contect of the
`restrited' column of the collection table (typically Apache group). Otherwise return
None if the collection is public."""
if collection_restricted_p(self.name):
return 1
return None
def get_sons(self, type='r'):
"Returns list of direct sons of type 'type' for the current collection."
sons = []
id_dad = self.id
query = "SELECT cc.id_son,c.name FROM collection_collection AS cc, collection AS c "\
"WHERE cc.id_dad=%d AND cc.type='%s' AND c.id=cc.id_son ORDER BY score ASC, c.name ASC" % (int(id_dad), type)
res = run_sql(query)
for row in res:
sons.append(get_collection(row[1]))
return sons
def get_descendants(self, type='r'):
"Returns list of all descendants of type 'type' for the current collection."
descendants = []
descendant_ids = intbitset()
id_dad = self.id
query = "SELECT cc.id_son,c.name FROM collection_collection AS cc, collection AS c "\
"WHERE cc.id_dad=%d AND cc.type='%s' AND c.id=cc.id_son ORDER BY score ASC" % (int(id_dad), type)
res = run_sql(query)
for row in res:
col_desc = get_collection(row[1])
# looking for loops
if self.id in descendant_ids:
write_message("Loop found in collection %s" % self.name, stream=sys.stderr)
raise OverflowError("Loop found in collection %s" % self.name)
else:
descendants.append(col_desc)
descendant_ids.add(col_desc.id)
tmp_descendants = col_desc.get_descendants()
for descendant in tmp_descendants:
descendant_ids.add(descendant.id)
descendants += tmp_descendants
return descendants
def write_cache_file(self, filename='', filebody={}):
"Write a file inside collection cache."
# open file:
dirname = "%s/collections" % (CFG_CACHEDIR)
mymkdir(dirname)
fullfilename = dirname + "/%s.html" % filename.replace('/', '___SLASH___')
try:
os.umask(022)
f = open(fullfilename, "wb")
except IOError, v:
try:
(code, message) = v
except:
code = 0
message = v
print "I/O Error: " + str(message) + " (" + str(code) + ")"
sys.exit(1)
# print user info:
write_message("... creating %s" % fullfilename, verbose=6)
# print page body:
cPickle.dump(filebody, f, cPickle.HIGHEST_PROTOCOL)
# close file:
f.close()
def update_webpage_cache(self, lang):
"""Create collection page header, navtrail, body (including left and right stripes) and footer, and
call write_cache_file() afterwards to update the collection webpage cache."""
## precalculate latest additions for non-aggregate
## collections (the info is ln and as independent)
if self.dbquery:
if CFG_WEBSEARCH_I18N_LATEST_ADDITIONS:
self.create_latest_additions_info(ln=lang)
else:
self.create_latest_additions_info()
# load the right message language
_ = gettext_set_language(lang)
# create dictionary with data
cache = {"te_portalbox" : self.create_portalbox(lang, 'te'),
"np_portalbox" : self.create_portalbox(lang, 'np'),
"ne_portalbox" : self.create_portalbox(lang, 'ne'),
"tp_portalbox" : self.create_portalbox(lang, "tp"),
"lt_portalbox" : self.create_portalbox(lang, "lt"),
"rt_portalbox" : self.create_portalbox(lang, "rt"),
"last_updated" : convert_datestruct_to_dategui(time.localtime(),
ln=lang)}
for aas in CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES: # do light, simple and advanced search pages:
cache["navtrail_%s" % aas] = self.create_navtrail_links(aas, lang)
cache["searchfor_%s" % aas] = self.create_searchfor(aas, lang)
cache["narrowsearch_%s" % aas] = self.create_narrowsearch(aas, lang, 'r')
cache["focuson_%s" % aas] = self.create_narrowsearch(aas, lang, "v")+ \
self.create_external_collections_box(lang)
cache["instantbrowse_%s" % aas] = self.create_instant_browse(aas=aas, ln=lang)
# write cache file
self.write_cache_file("%s-ln=%s"%(self.name, lang), cache)
return cache
def create_navtrail_links(self, aas=CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE, ln=CFG_SITE_LANG):
"""Creates navigation trail links, i.e. links to collection
ancestors (except Home collection). If aas==1, then links to
Advanced Search interfaces; otherwise Simple Search.
"""
dads = []
for dad in self.get_ancestors():
if dad.name != CFG_SITE_NAME: # exclude Home collection
dads.append((dad.name, dad.get_name(ln)))
return websearch_templates.tmpl_navtrail_links(
aas=aas, ln=ln, dads=dads)
def create_portalbox(self, lang=CFG_SITE_LANG, position="rt"):
"""Creates portalboxes of language CFG_SITE_LANG of the position POSITION by consulting DB configuration database.
The position may be: 'lt'='left top', 'rt'='right top', etc."""
out = ""
query = "SELECT p.title,p.body FROM portalbox AS p, collection_portalbox AS cp "\
" WHERE cp.id_collection=%d AND p.id=cp.id_portalbox AND cp.ln='%s' AND cp.position='%s' "\
" ORDER BY cp.score DESC" % (self.id, lang, position)
res = run_sql(query)
for row in res:
title, body = row[0], row[1]
if title:
out += websearch_templates.tmpl_portalbox(title = title,
body = body)
else:
# no title specified, so print body ``as is'' only:
out += body
return out
def create_narrowsearch(self, aas=CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE, ln=CFG_SITE_LANG, type="r"):
"""Creates list of collection descendants of type 'type' under title 'title'.
If aas==1, then links to Advanced Search interfaces; otherwise Simple Search.
Suitable for 'Narrow search' and 'Focus on' boxes."""
# get list of sons and analyse it
sons = self.get_sons(type)
if not sons:
return ''
# get descendents
descendants = self.get_descendants(type)
grandsons = []
if CFG_WEBSEARCH_NARROW_SEARCH_SHOW_GRANDSONS:
# load grandsons for each son
for son in sons:
grandsons.append(son.get_sons())
# return ""
return websearch_templates.tmpl_narrowsearch(
aas = aas,
ln = ln,
type = type,
father = self,
has_grandchildren = len(descendants)>len(sons),
sons = sons,
display_grandsons = CFG_WEBSEARCH_NARROW_SEARCH_SHOW_GRANDSONS,
grandsons = grandsons
)
def create_external_collections_box(self, ln=CFG_SITE_LANG):
external_collection_load_states()
if not dico_collection_external_searches.has_key(self.id):
return ""
engines_list = external_collection_sort_engine_by_name(dico_collection_external_searches[self.id])
return websearch_templates.tmpl_searchalso(ln, engines_list, self.id)
def create_latest_additions_info(self, rg=CFG_WEBSEARCH_INSTANT_BROWSE, ln=CFG_SITE_LANG):
"""
Create info about latest additions that will be used for
create_instant_browse() later.
"""
self.latest_additions_info = []
if self.nbrecs and self.reclist:
# firstly, get last 'rg' records:
recIDs = list(self.reclist)
of = 'hb'
# CERN hack begins: tweak latest additions for selected collections:
if CFG_CERN_SITE:
# alter recIDs list for some CERN collections:
this_year = time.strftime("%Y", time.localtime())
if self.name in ['CERN Yellow Reports','Videos']:
last_year = str(int(this_year) - 1)
# detect recIDs only from this and past year:
recIDs = list(self.reclist & \
search_pattern_parenthesised(p='year:%s or year:%s' % \
(this_year, last_year)))
# apply special filters:
if self.name in ['Videos']:
# select only videos with movies:
recIDs = list(intbitset(recIDs) & \
search_pattern_parenthesised(p='collection:"PUBLVIDEOMOVIE" -"Virtual Visit"'))
of = 'hvp'
if self.name in ['General Talks', 'Academic Training Lectures', 'Summer Student Lectures']:
#select only the lectures with material
recIDs = list(self.reclist & search_pattern_parenthesised(p='856:MediaArchive'))
# sort some CERN collections specially:
if self.name in ['Videos',
'Video Clips',
'Video Movies',
'Video News',
'Video Rushes',
'Webcast',
'ATLAS Videos',
'Restricted Video Movies',
'Restricted Video Rushes',
'LHC First Beam Videos',
'CERN openlab Videos']:
recIDs = sort_records(None, recIDs, '269__c', 'a')
elif self.name in ['LHCb Talks']:
recIDs = sort_records(None, recIDs, 'reportnumber', 'a')
elif self.name in ['CERN Yellow Reports']:
recIDs = sort_records(None, recIDs, '084__a', 'a')
elif self.name in ['CERN Courier Issues',
'CERN Courier Articles',
'CERN Bulletin Issues',
'CERN Bulletin Articles']:
recIDs = sort_records(None, recIDs, '773__y', 'a')
# CERN hack ends.
total = len(recIDs)
to_display = min(rg, total)
for idx in range(total-1, total-to_display-1, -1):
recid = recIDs[idx]
self.latest_additions_info.append({'id': recid,
'format': format_record(recid, of, ln=ln),
'date': get_creation_date(recid, fmt="%Y-%m-%d
%H:%i")})
return
def create_instant_browse(self, rg=CFG_WEBSEARCH_INSTANT_BROWSE, aas=CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE, ln=CFG_SITE_LANG):
"Searches database and produces list of last 'rg' records."
if self.restricted_p():
return websearch_templates.tmpl_box_restricted_content(ln = ln)
if str(self.dbquery).startswith("hostedcollection:"):
return websearch_templates.tmpl_box_hosted_collection(ln = ln)
if rg == 0:
# do not show latest additions box
return ""
# CERN hack: do not display latest additions for some CERN collections:
if CFG_CERN_SITE and self.name in ['Periodicals', 'Electronic Journals',
'Press Office Photo Selection',
'Press Office Video Selection']:
return ""
try:
self.latest_additions_info
latest_additions_info_p = True
except:
latest_additions_info_p = False
if latest_additions_info_p:
passIDs = []
for idx in range(0, min(len(self.latest_additions_info), rg)):
# CERN hack: display the records in a grid layout, so do not show the related links
if CFG_CERN_SITE and self.name in ['Videos']:
passIDs.append({'id': self.latest_additions_info[idx]['id'],
'body': self.latest_additions_info[idx]['format'],
'date': self.latest_additions_info[idx]['date']})
else:
passIDs.append({'id': self.latest_additions_info[idx]['id'],
'body': self.latest_additions_info[idx]['format'] + \
websearch_templates.tmpl_record_links(recid=self.latest_additions_info[idx]['id'],
rm='citation',
ln=ln),
'date': self.latest_additions_info[idx]['date']})
if self.nbrecs > rg:
url = websearch_templates.build_search_url(
cc=self.name, jrec=rg+1, ln=ln, aas=aas)
else:
url = ""
# CERN hack: display the records in a grid layout
if CFG_CERN_SITE and self.name in ['Videos']:
return websearch_templates.tmpl_instant_browse(
aas=aas, ln=ln, recids=passIDs, more_link=url, grid_layout=True, father=self)
return websearch_templates.tmpl_instant_browse(
aas=aas, ln=ln, recids=passIDs, more_link=url, father=self)
return websearch_templates.tmpl_box_no_records(ln=ln)
def create_searchoptions(self):
"Produces 'Search options' portal box."
box = ""
query = """SELECT DISTINCT(cff.id_field),f.code,f.name FROM collection_field_fieldvalue AS cff, field AS f
WHERE cff.id_collection=%d AND cff.id_fieldvalue IS NOT NULL AND cff.id_field=f.id
ORDER BY cff.score DESC""" % self.id
res = run_sql(query)
if res:
for row in res:
field_id = row[0]
field_code = row[1]
field_name = row[2]
query_bis = """SELECT fv.value,fv.name FROM fieldvalue AS fv, collection_field_fieldvalue AS cff
WHERE cff.id_collection=%d AND cff.type='seo' AND cff.id_field=%d AND fv.id=cff.id_fieldvalue
ORDER BY cff.score_fieldvalue DESC, cff.score DESC, fv.name ASC""" % (self.id, field_id)
res_bis = run_sql(query_bis)
if res_bis:
values = [{'value' : '', 'text' : 'any' + ' ' + field_name}] # FIXME: internationalisation of "any"
for row_bis in res_bis:
values.append({'value' : cgi.escape(row_bis[0], 1), 'text' : row_bis[1]})
box += websearch_templates.tmpl_select(
fieldname = field_code,
values = values
)
return box
def create_sortoptions(self, ln=CFG_SITE_LANG):
"""Produces 'Sort options' portal box."""
# load the right message language
_ = gettext_set_language(ln)
box = ""
query = """SELECT f.code,f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE id_collection=%d AND cff.type='soo' AND cff.id_field=f.id
ORDER BY cff.score DESC, f.name ASC""" % self.id
values = [{'value' : '', 'text': "- %s -" % _("latest first")}]
res = run_sql(query)
if res:
for row in res:
values.append({'value' : row[0], 'text': get_field_i18nname(row[1], ln)})
else:
for tmp in ('title', 'author', 'report number', 'year'):
values.append({'value' : tmp.replace(' ', ''), 'text' : get_field_i18nname(tmp, ln)})
box = websearch_templates.tmpl_select(
fieldname = 'sf',
css_class = 'address',
values = values
)
box += websearch_templates.tmpl_select(
fieldname = 'so',
css_class = 'address',
values = [
{'value' : 'a' , 'text' : _("asc.")},
{'value' : 'd' , 'text' : _("desc.")}
]
)
return box
def create_rankoptions(self, ln=CFG_SITE_LANG):
"Produces 'Rank options' portal box."
# load the right message language
_ = gettext_set_language(ln)
values = [{'value' : '', 'text': "- %s %s -" % (string.lower(_("OR")), _("rank by"))}]
for (code, name) in get_bibrank_methods(self.id, ln):
values.append({'value' : code, 'text': name})
box = websearch_templates.tmpl_select(
fieldname = 'rm',
css_class = 'address',
values = values
)
return box
def create_displayoptions(self, ln=CFG_SITE_LANG):
"Produces 'Display options' portal box."
# load the right message language
_ = gettext_set_language(ln)
values = []
for i in ['10', '25', '50', '100', '250', '500']:
values.append({'value' : i, 'text' : i + ' ' + _("results")})
box = websearch_templates.tmpl_select(
fieldname = 'rg',
selected = str(CFG_WEBSEARCH_DEF_RECORDS_IN_GROUPS),
css_class = 'address',
values = values
)
if self.get_sons():
box += websearch_templates.tmpl_select(
fieldname = 'sc',
css_class = 'address',
values = [
{'value' : '1' , 'text' : CFG_SCOAP3_SITE and _("split by publisher/journal") or _("split by collection")},
{'value' : '0' , 'text' : _("single list")}
]
)
return box
def create_formatoptions(self, ln=CFG_SITE_LANG):
"Produces 'Output format options' portal box."
# load the right message language
_ = gettext_set_language(ln)
box = ""
values = []
query = """SELECT f.code,f.name FROM format AS f, collection_format AS cf
WHERE cf.id_collection=%d AND cf.id_format=f.id AND f.visibility='1'
ORDER BY cf.score DESC, f.name ASC""" % self.id
res = run_sql(query)
if res:
for row in res:
values.append({'value' : row[0], 'text': row[1]})
else:
values.append({'value' : 'hb', 'text' : "HTML %s" % _("brief")})
box = websearch_templates.tmpl_select(
fieldname = 'of',
css_class = 'address',
values = values
)
return box
def create_searchwithin_selection_box(self, fieldname='f', value='', ln='en'):
"""Produces 'search within' selection box for the current collection."""
# get values
query = """SELECT f.code,f.name FROM field AS f, collection_field_fieldvalue AS cff
WHERE cff.type='sew' AND cff.id_collection=%d AND cff.id_field=f.id
ORDER BY cff.score DESC, f.name ASC""" % self.id
res = run_sql(query)
values = [{'value' : '', 'text' : get_field_i18nname("any field", ln)}]
if res:
for row in res:
values.append({'value' : row[0], 'text' : get_field_i18nname(row[1], ln)})
else:
if CFG_CERN_SITE:
for tmp in ['title', 'author', 'abstract', 'report number', 'year']:
values.append({'value' : tmp.replace(' ', ''), 'text' : get_field_i18nname(tmp, ln)})
else:
for tmp in ['title', 'author', 'abstract', 'keyword', 'report number', 'journal', 'year', 'fulltext', 'reference']:
values.append({'value' : tmp.replace(' ', ''), 'text' : get_field_i18nname(tmp, ln)})
return websearch_templates.tmpl_searchwithin_select(
fieldname = fieldname,
ln = ln,
selected = value,
values = values
)
def create_searchexample(self):
"Produces search example(s) for the current collection."
out = "$collSearchExamples = getSearchExample(%d, $se);" % self.id
return out
def create_searchfor(self, aas=CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE, ln=CFG_SITE_LANG):
"Produces either Simple or Advanced 'Search for' box for the current collection."
if aas == 2:
return self.create_searchfor_addtosearch(ln)
elif aas == 1:
return self.create_searchfor_advanced(ln)
elif aas == 0:
return self.create_searchfor_simple(ln)
else:
return self.create_searchfor_light(ln)
def create_searchfor_addtosearch(self, ln=CFG_SITE_LANG):
"Produces add-to-search 'Search for' box for the current collection."
return websearch_templates.tmpl_searchfor_addtosearch(
ln=ln,
collection_id=self.name,
record_count=self.nbrecs,
searchwithin= self.create_searchwithin_selection_box(fieldname='f1', ln=ln),
)
def create_searchfor_light(self, ln=CFG_SITE_LANG):
"Produces light 'Search for' box for the current collection."
return websearch_templates.tmpl_searchfor_light(
ln=ln,
collection_id = self.name,
collection_name=self.get_name(ln=ln),
record_count=self.nbrecs,
example_search_queries=self.get_example_search_queries(),
)
def create_searchfor_simple(self, ln=CFG_SITE_LANG):
"Produces simple 'Search for' box for the current collection."
return websearch_templates.tmpl_searchfor_simple(
ln=ln,
collection_id = self.name,
collection_name=self.get_name(ln=ln),
record_count=self.nbrecs,
middle_option = self.create_searchwithin_selection_box(ln=ln),
)
def create_searchfor_advanced(self, ln=CFG_SITE_LANG):
"Produces advanced 'Search for' box for the current collection."
return websearch_templates.tmpl_searchfor_advanced(
ln = ln,
collection_id = self.name,
collection_name=self.get_name(ln=ln),
record_count=self.nbrecs,
middle_option_1 = self.create_searchwithin_selection_box('f1', ln=ln),
middle_option_2 = self.create_searchwithin_selection_box('f2', ln=ln),
middle_option_3 = self.create_searchwithin_selection_box('f3', ln=ln),
searchoptions = self.create_searchoptions(),
sortoptions = self.create_sortoptions(ln),
rankoptions = self.create_rankoptions(ln),
displayoptions = self.create_displayoptions(ln),
formatoptions = self.create_formatoptions(ln)
)
def calculate_reclist(self):
"""
Calculate, set and return the (reclist,
reclist_with_nonpublic_subcolls,
nbrecs_from_hosted_collections)
tuple for the given collection."""
if str(self.dbquery).startswith("hostedcollection:"):
# we don't normally use this function to calculate the reclist
# for hosted collections. In case we do, recursively for a regular
# ancestor collection, then quickly return the object attributes.
return (self.reclist,
self.reclist_with_nonpublic_subcolls,
self.nbrecs)
if self.calculate_reclist_run_already:
# do we really have to recalculate? If not,
# then return the object attributes
return (self.reclist,
self.reclist_with_nonpublic_subcolls,
self.nbrecs_from_hosted_collections)
write_message("... calculating reclist of %s" % self.name, verbose=6)
reclist = intbitset() # will hold results for public sons only; good for storing into DB
reclist_with_nonpublic_subcolls = intbitset() # will hold results for both public and nonpublic sons; good for deducing total
# number of documents
nbrecs_from_hosted_collections = 0 # will hold the total number of records from descendant hosted collections
if not self.dbquery:
# A - collection does not have dbquery, so query recursively all its sons
# that are either non-restricted or that have the same restriction rules
for coll in self.get_sons():
coll_reclist,\
coll_reclist_with_nonpublic_subcolls,\
coll_nbrecs_from_hosted_collection = coll.calculate_reclist()
if ((coll.restricted_p() is None) or
(coll.restricted_p() == self.restricted_p())):
# add this reclist ``for real'' only if it is public
reclist.union_update(coll_reclist)
reclist_with_nonpublic_subcolls.union_update(coll_reclist_with_nonpublic_subcolls)
# increment the total number of records from descendant hosted collections
nbrecs_from_hosted_collections += coll_nbrecs_from_hosted_collection
else:
# B - collection does have dbquery, so compute it:
# (note: explicitly remove DELETED records)
if CFG_CERN_SITE:
reclist = search_pattern_parenthesised(None, self.dbquery + \
' -980__:"DELETED" -980__:"DUMMY"', ap=-9) #ap=-9 for allow queries containing hidden tags
else:
reclist = search_pattern_parenthesised(None, self.dbquery + ' -980__:"DELETED"', ap=-9) #ap=-9 allow queries containing hidden tags
reclist_with_nonpublic_subcolls = copy.deepcopy(reclist)
# store the results:
self.nbrecs_from_hosted_collections = nbrecs_from_hosted_collections
self.nbrecs = len(reclist_with_nonpublic_subcolls) + \
nbrecs_from_hosted_collections
self.reclist = reclist
self.reclist_with_nonpublic_subcolls = reclist_with_nonpublic_subcolls
# last but not least, update the speed-up flag:
self.calculate_reclist_run_already = 1
# return the two sets, as well as
# the total number of records from descendant hosted collections:
return (self.reclist,
self.reclist_with_nonpublic_subcolls,
self.nbrecs_from_hosted_collections)
def calculate_nbrecs_for_external_collection(self, timeout=CFG_EXTERNAL_COLLECTION_TIMEOUT):
"""Calculate the total number of records, aka nbrecs, for given external collection."""
#if self.calculate_reclist_run_already:
# do we have to recalculate?
#return self.nbrecs
#write_message("... calculating nbrecs of external collection %s" % self.name, verbose=6)
if external_collections_dictionary.has_key(self.name):
engine = external_collections_dictionary[self.name]
if engine.parser:
self.nbrecs_tmp = engine.parser.parse_nbrecs(timeout)
if self.nbrecs_tmp >= 0: return self.nbrecs_tmp
# the parse_nbrecs() function returns negative values for some specific cases
# maybe we can handle these specific cases, some warnings or something
# for now the total number of records remains silently the same
else: return self.nbrecs
else: write_message("External collection %s does not have a parser!" % self.name, verbose=6)
else: write_message("External collection %s not found!" % self.name, verbose=6)
return 0
# last but not least, update the speed-up flag:
#self.calculate_reclist_run_already = 1
def check_nbrecs_for_external_collection(self):
"""Check if the external collections has changed its total number of records, aka nbrecs.
Rerurns True if the total number of records has changed and False if it's the same"""
write_message("*** self.nbrecs = %s / self.cal...ion = %s ***" % (str(self.nbrecs), str(self.calculate_nbrecs_for_external_collection())), verbose=6)
write_message("*** self.nbrecs != self.cal...ion = %s ***" % (str(self.nbrecs != self.calculate_nbrecs_for_external_collection()),), verbose=6)
return self.nbrecs != self.calculate_nbrecs_for_external_collection(CFG_HOSTED_COLLECTION_TIMEOUT_NBRECS)
def set_nbrecs_for_external_collection(self):
"""Set this external collection's total number of records, aka nbrecs"""
if self.calculate_reclist_run_already:
# do we have to recalculate?
return
write_message("... calculating nbrecs of external collection %s" % self.name, verbose=6)
if self.nbrecs_tmp:
self.nbrecs = self.nbrecs_tmp
else:
self.nbrecs = self.calculate_nbrecs_for_external_collection(CFG_HOSTED_COLLECTION_TIMEOUT_NBRECS)
# last but not least, update the speed-up flag:
self.calculate_reclist_run_already = 1
def update_reclist(self):
"Update the record universe for given collection; nbrecs, reclist of the collection table."
if self.update_reclist_run_already:
# do we have to reupdate?
return 0
write_message("... updating reclist of %s (%s recs)" % (self.name, self.nbrecs), verbose=6)
sys.stdout.flush()
try:
## In principle we could skip this update if old_reclist==reclist
## however we just update it here in case of race-conditions.
- run_sql("UPDATE collection SET nbrecs=%s, reclist=%s WHERE id=%s",
+ run_sql("UPDATE collection SET nbrecs=%s, reclist=_binary %s WHERE id=%s",
(self.nbrecs, self.reclist.fastdump(), self.id))
if self.old_reclist != self.reclist:
self.reclist_updated_since_start = 1
else:
write_message("... no changes in reclist detected", verbose=6)
except Error, e:
print "Database Query Error %d: %s." % (e.args[0], e.args[1])
sys.exit(1)
# last but not least, update the speed-up flag:
self.update_reclist_run_already = 1
return 0
def perform_display_collection(colID, colname, aas, ln, em, show_help_boxes):
"""Returns the data needed to display a collection page
The arguments are as follows:
colID - id of the collection to display
colname - name of the collection to display
aas - 0 if simple search, 1 if advanced search
ln - language of the page
em - code to display just part of the page
show_help_boxes - whether to show the help boxes or not"""
# check and update cache if necessary
cachedfile = open(r"%s/collections/%s-ln=%s.html" %
(CFG_CACHEDIR, colname.replace('/', '___SLASH___'), ln),
"rb")
try:
data = cPickle.load(cachedfile)
except ValueError:
data = get_collection(colname).update_webpage_cache(ln)
cachedfile.close()
# check em value to return just part of the page
if em != "":
if EM_REPOSITORY["search_box"] not in em:
data["searchfor_%s" % aas] = ""
if EM_REPOSITORY["see_also_box"] not in em:
data["focuson_%s" % aas] = ""
if EM_REPOSITORY["all_portalboxes"] not in em:
if EM_REPOSITORY["te_portalbox"] not in em:
data["te_portalbox"] = ""
if EM_REPOSITORY["np_portalbox"] not in em:
data["np_portalbox"] = ""
if EM_REPOSITORY["ne_portalbox"] not in em:
data["ne_portalbox"] = ""
if EM_REPOSITORY["tp_portalbox"] not in em:
data["tp_portalbox"] = ""
if EM_REPOSITORY["lt_portalbox"] not in em:
data["lt_portalbox"] = ""
if EM_REPOSITORY["rt_portalbox"] not in em:
data["rt_portalbox"] = ""
c_body = websearch_templates.tmpl_webcoll_body(ln, colID, data["te_portalbox"],
data["searchfor_%s"%aas], data["np_portalbox"], data["narrowsearch_%s"%aas],
data["focuson_%s"%aas], data["instantbrowse_%s"%aas], data["ne_portalbox"],
em=="" or EM_REPOSITORY["body"] in em)
if show_help_boxes <= 0:
data["rt_portalbox"] = ""
return (c_body, data["navtrail_%s"%aas], data["lt_portalbox"], data["rt_portalbox"],
data["tp_portalbox"], data["te_portalbox"], data["last_updated"])
def get_datetime(var, format_string="%Y-%m-%d %H:%M:%S"):
"""Returns a date string according to the format string.
It can handle normal date strings and shifts with respect
to now."""
date = time.time()
shift_re = re.compile("([-\+]{0,1})([\d]+)([dhms])")
factors = {"d":24*3600, "h":3600, "m":60, "s":1}
m = shift_re.match(var)
if m:
sign = m.groups()[0] == "-" and -1 or 1
factor = factors[m.groups()[2]]
value = float(m.groups()[1])
date = time.localtime(date + sign * factor * value)
date = strftime(format_string, date)
else:
date = time.strptime(var, format_string)
date = strftime(format_string, date)
return date
def get_current_time_timestamp():
"""Return timestamp corresponding to the current time."""
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
def compare_timestamps_with_tolerance(timestamp1,
timestamp2,
tolerance=0):
"""Compare two timestamps TIMESTAMP1 and TIMESTAMP2, of the form
'2005-03-31 17:37:26'. Optionally receives a TOLERANCE argument
(in seconds). Return -1 if TIMESTAMP1 is less than TIMESTAMP2
minus TOLERANCE, 0 if they are equal within TOLERANCE limit,
and 1 if TIMESTAMP1 is greater than TIMESTAMP2 plus TOLERANCE.
"""
# remove any trailing .00 in timestamps:
timestamp1 = re.sub(r'\.[0-9]+$', '', timestamp1)
timestamp2 = re.sub(r'\.[0-9]+$', '', timestamp2)
# first convert timestamps to Unix epoch seconds:
timestamp1_seconds = calendar.timegm(time.strptime(timestamp1, "%Y-%m-%d %H:%M:%S"))
timestamp2_seconds = calendar.timegm(time.strptime(timestamp2, "%Y-%m-%d %H:%M:%S"))
# now compare them:
if timestamp1_seconds < timestamp2_seconds - tolerance:
return -1
elif timestamp1_seconds > timestamp2_seconds + tolerance:
return 1
else:
return 0
def get_database_last_updated_timestamp():
"""Return last updated timestamp for collection-related and
record-related database tables.
"""
database_tables_timestamps = []
database_tables_timestamps.append(get_table_update_time('bibrec'))
## In INSPIRE bibfmt is on innodb and there is not such configuration
bibfmt_last_update = run_sql("SELECT max(last_updated) FROM bibfmt")
if bibfmt_last_update and bibfmt_last_update[0][0]:
database_tables_timestamps.append(str(bibfmt_last_update[0][0]))
try:
database_tables_timestamps.append(get_table_update_time('idxWORD%'))
except ValueError:
# There are no indexes in the database. That's OK.
pass
database_tables_timestamps.append(get_table_update_time('collection%'))
database_tables_timestamps.append(get_table_update_time('portalbox'))
database_tables_timestamps.append(get_table_update_time('field%'))
database_tables_timestamps.append(get_table_update_time('format%'))
database_tables_timestamps.append(get_table_update_time('rnkMETHODNAME'))
database_tables_timestamps.append(get_table_update_time('accROLE_accACTION_accARGUMENT', run_on_slave=True))
return max(database_tables_timestamps)
def get_cache_last_updated_timestamp():
"""Return last updated cache timestamp."""
try:
f = open(CFG_CACHE_LAST_UPDATED_TIMESTAMP_FILE, "r")
except:
return "1970-01-01 00:00:00"
timestamp = f.read()
f.close()
return timestamp
def set_cache_last_updated_timestamp(timestamp):
"""Set last updated cache timestamp to TIMESTAMP."""
try:
f = open(CFG_CACHE_LAST_UPDATED_TIMESTAMP_FILE, "w")
except:
pass
f.write(timestamp)
f.close()
return timestamp
def main():
"""Main that construct all the bibtask."""
task_init(authorization_action="runwebcoll",
authorization_msg="WebColl Task Submission",
description="""Description:
webcoll updates the collection cache (record universe for a
given collection plus web page elements) based on invenio.conf and DB
configuration parameters. If the collection name is passed as an argument,
only this collection's cache will be updated. If the recursive option is
set as well, the collection's descendants will also be updated.\n""",
help_specific_usage=" -c, --collection\t Update cache for the given "
"collection only. [all]\n"
" -r, --recursive\t Update cache for the given collection and all its\n"
"\t\t\t descendants (to be used in combination with -c). [no]\n"
" -q, --quick\t\t Skip webpage cache update for those collections whose\n"
"\t\t\t reclist was not changed. Note: if you use this option, it is advised\n"
"\t\t\t to schedule, e.g. a nightly 'webcoll --force'. [no]\n"
" -f, --force\t\t Force update even if cache is up to date. [no]\n"
" -p, --part\t\t Update only certain cache parts (1=reclist,"
" 2=webpage). [both]\n"
" -l, --language\t Update pages in only certain language"
" (e.g. fr,it,...). [all]\n",
version=__revision__,
specific_params=("c:rqfp:l:", [
"collection=",
"recursive",
"quick",
"force",
"part=",
"language="
]),
task_submit_elaborate_specific_parameter_fnc=task_submit_elaborate_specific_parameter,
task_submit_check_options_fnc=task_submit_check_options,
task_run_fnc=task_run_core)
def task_submit_elaborate_specific_parameter(key, value, opts, args):
""" Given the string key it checks it's meaning, eventually using the value.
Usually it fills some key in the options dict.
It must return True if it has elaborated the key, False, if it doesn't
know that key.
eg:
if key in ['-n', '--number']:
self.options['number'] = value
return True
return False
"""
if key in ("-c", "--collection"):
task_set_option("collection", value)
elif key in ("-r", "--recursive"):
task_set_option("recursive", 1)
elif key in ("-f", "--force"):
task_set_option("force", 1)
elif key in ("-q", "--quick"):
task_set_option("quick", 1)
elif key in ("-p", "--part"):
task_set_option("part", int(value))
elif key in ("-l", "--language"):
languages = task_get_option("language", [])
languages += value.split(',')
for ln in languages:
if ln not in CFG_SITE_LANGS:
print 'ERROR: "%s" is not a recognized language code' % ln
return False
task_set_option("language", languages)
else:
return False
return True
def task_submit_check_options():
if task_has_option('collection'):
coll = get_collection(task_get_option("collection"))
if coll.id is None:
print 'ERROR: Collection "%s" does not exist' % coll.name
return False
return True
def task_run_core():
""" Reimplement to add the body of the task."""
#
# ------->--->time--->------>
# (-1) | ( 0) | ( 1)
# | | |
# [T.db] | [T.fc] | [T.db]
# | | |
# |<-tol|tol->|
#
# the above is the compare_timestamps_with_tolerance result "diagram"
# [T.db] stands fore the database timestamp and [T.fc] for the file cache timestamp
# ( -1, 0, 1) stand for the returned value
# tol stands for the tolerance in seconds
#
# When a record has been added or deleted from one of the collections the T.db becomes greater that the T.fc
# and when webcoll runs it is fully ran. It recalculates the reclists and nbrecs, and since it updates the
# collections db table it also updates the T.db. The T.fc is set as the moment the task started running thus
# slightly before the T.db (practically the time distance between the start of the task and the last call of
# update_reclist). Therefore when webcoll runs again, and even if no database changes have taken place in the
# meanwhile, it fully runs (because compare_timestamps_with_tolerance returns 0). This time though, and if
# no databases changes have taken place, the T.db remains the same while T.fc is updated and as a result if
# webcoll runs again it will not be fully ran
#
task_run_start_timestamp = get_current_time_timestamp()
colls = []
# decide whether we need to run or not, by comparing last updated timestamps:
write_message("Database timestamp is %s." % get_database_last_updated_timestamp(), verbose=3)
write_message("Collection cache timestamp is %s." % get_cache_last_updated_timestamp(), verbose=3)
if task_has_option("part"):
write_message("Running cache update part %s only." % task_get_option("part"), verbose=3)
if check_nbrecs_for_all_external_collections() or task_has_option("force") or \
compare_timestamps_with_tolerance(get_database_last_updated_timestamp(),
get_cache_last_updated_timestamp(),
CFG_CACHE_LAST_UPDATED_TIMESTAMP_TOLERANCE) >= 0:
## either forced update was requested or cache is not up to date, so recreate it:
# firstly, decide which collections to do:
if task_has_option("collection"):
coll = get_collection(task_get_option("collection"))
colls.append(coll)
if task_has_option("recursive"):
r_type_descendants = coll.get_descendants(type='r')
colls += r_type_descendants
v_type_descendants = coll.get_descendants(type='v')
colls += v_type_descendants
else:
res = run_sql("SELECT name FROM collection ORDER BY id")
for row in res:
colls.append(get_collection(row[0]))
# secondly, update collection reclist cache:
if task_get_option('part', 1) == 1:
i = 0
for coll in colls:
i += 1
write_message("%s / reclist cache update" % coll.name)
if str(coll.dbquery).startswith("hostedcollection:"):
coll.set_nbrecs_for_external_collection()
else:
coll.calculate_reclist()
coll.update_reclist()
task_update_progress("Part 1/2: done %d/%d" % (i, len(colls)))
task_sleep_now_if_required(can_stop_too=True)
# thirdly, update collection webpage cache:
if task_get_option("part", 2) == 2:
# Updates cache only for chosen languages or for all available ones if none was chosen
languages = task_get_option("language", CFG_SITE_LANGS)
write_message("Cache update for the following languages: %s" % str(languages), verbose=3)
i = 0
for coll in colls:
i += 1
if coll.reclist_updated_since_start or task_has_option("collection") or task_get_option("force") or not task_get_option("quick"):
write_message("%s / webpage cache update" % coll.name)
for lang in languages:
coll.update_webpage_cache(lang)
else:
write_message("%s / webpage cache seems not to need an update and --quick was used" % coll.name, verbose=2)
task_update_progress("Part 2/2: done %d/%d" % (i, len(colls)))
task_sleep_now_if_required(can_stop_too=True)
# finally update the cache last updated timestamp:
# (but only when all collections were updated, not when only
# some of them were forced-updated as per admin's demand)
if not task_has_option("collection"):
set_cache_last_updated_timestamp(task_run_start_timestamp)
write_message("Collection cache timestamp is set to %s." % get_cache_last_updated_timestamp(), verbose=3)
else:
## cache up to date, we don't have to run
write_message("Collection cache is up to date, no need to run.")
## we are done:
return True
### okay, here we go:
if __name__ == '__main__':
main()
diff --git a/modules/websession/lib/session.py b/modules/websession/lib/session.py
index 1ae199a37..9f1f0bd4a 100644
--- a/modules/websession/lib/session.py
+++ b/modules/websession/lib/session.py
@@ -1,629 +1,629 @@
# -*- coding: utf-8 -*-
# This file is part of Invenio.
-# Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015 CERN.
+# Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Session management adapted from mod_python Session class.
Just use L{get_session} to obtain a session object (with a dictionary
interface, which will let you store permanent information).
"""
from invenio.webinterface_handler_wsgi_utils import Cookie, get_cookie
import cPickle
import time
import random
import re
import sys
import os
if sys.hexversion < 0x2060000:
from md5 import md5
else:
from hashlib import md5
from invenio.dateutils import convert_datestruct_to_datetext
from invenio.dbquery import blob_to_string
from invenio.config import (CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER,
CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT,
CFG_SITE_URL,
CFG_SITE_SECURE_URL,
CFG_WEBSESSION_IPADDR_CHECK_SKIP_BITS,
CFG_WEBSEARCH_PREV_NEXT_HIT_FOR_GUESTS,
CFG_WEBSESSION_STORAGE)
from invenio.websession_config import (CFG_WEBSESSION_COOKIE_NAME,
CFG_WEBSESSION_ONE_DAY,
CFG_WEBSESSION_CLEANUP_CHANCE)
from invenio.dbquery import run_sql
from invenio.redisutils import get_redis
CFG_FULL_HTTPS = CFG_SITE_URL.lower().startswith("https://")
if CFG_WEBSEARCH_PREV_NEXT_HIT_FOR_GUESTS:
_CFG_SESSION_NON_USEFUL_KEYS = ('uid', 'user_info')
else:
_CFG_SESSION_NON_USEFUL_KEYS = ('uid', 'user_info', 'websearch-last-query', 'websearch-last-query-hits')
def get_session(req, sid=None):
"""
Obtain a session.
If the session has already been created for the current request,
returns the already existing session.
@param req: the mod_python request object.
@type req: mod_python request object
@param sid: the session identifier of an already existing session.
@type sid: 32 hexadecimal string
@return: the session.
@rtype: InvenioSession
@raise ValueError: if C{sid} is provided and it doesn't correspond to a
valid session.
"""
if sid is not None:
req._session = InvenioSession(req, sid)
return req._session
if not hasattr(req, '_session'):
req._session = InvenioSession(req, sid)
return req._session
class InvenioSessionBase(dict):
"""
This class implements a Session handling based on MySQL.
@param req: the mod_python request object.
@type req: mod_python request object
@param sid: the session identifier if already known
@type sid: 32 hexadecimal string
@ivar _remember_me: if the session cookie should last one day or until
the browser is closed.
@type _remember_me: bool
@note: The code is heavily based on ModPython 3.3.1 DBMSession
implementation.
@note: This class implements IP verification to prevent basic cookie
stealing.
@raise ValueError: if C{sid} is provided and correspond to a broken
session.
"""
def __init__(self, req, sid=None):
self._remember_me = False
self._req, self._sid, self._secret = req, sid, None
self._new = 1
self._created = 0
self._accessed = 0
self._timeout = 0
self._invalid = 0
self._dirty = False
self._http_ip = None
self._https_ip = None
self.__need_https = False
self._cleanup_function = None
dict.__init__(self)
if not self._sid:
# check to see if cookie exists
cookie = get_cookie(req, CFG_WEBSESSION_COOKIE_NAME)
if cookie:
self._sid = cookie.value
else:
stub_cookie = get_cookie(req, CFG_WEBSESSION_COOKIE_NAME + 'stub')
self.__need_https = stub_cookie and stub_cookie.value == 'HTTPS'
if self._sid:
if not _check_sid(self._sid):
if sid:
# Supplied explicitly by user of the class,
# raise an exception and make the user code
# deal with it.
raise ValueError("Invalid Session ID: sid=%s" % sid)
else:
# Derived from the cookie sent by browser,
# wipe it out so it gets replaced with a
# correct value.
self._sid = None
if self._sid:
# attempt to load ourselves
if self.load():
self._new = 0
if self._new:
# make a new session
self._sid = _new_sid(self._req)
remote_ip = self._req.remote_ip
if self._req.is_https():
self._https_ip = remote_ip
else:
self._http_ip = remote_ip
self._created = time.time()
self._timeout = CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT * \
CFG_WEBSESSION_ONE_DAY
self._accessed = time.time()
# need cleanup?
if random.randint(1, CFG_WEBSESSION_CLEANUP_CHANCE) == 1:
self.cleanup()
if self._new and (not self.__need_https or self._req.is_https()):
## We want to issue cookies only in case this is a new session
## and there is not already a session cookie that is available
## only over HTTPS
for cookie in self.make_cookies():
self._req.set_cookie(cookie)
def get_dirty(self):
"""
Is this session dirty?
"""
return self._dirty
def set_dirty(self, dummy=True):
"""
Flag this session as dirty. It takes a parameter, just in order
to be used within a property
"""
self._dirty = True
dirty = property(get_dirty, set_dirty)
def __setitem__(self, key, value):
if self.get(key) != value:
dict.__setitem__(self, key, value)
self._dirty = True
def __delitem__(self, key):
if key in self:
dict.__delitem__(self, key)
self._dirty = True
def set_remember_me(self, remember_me=True):
"""
Set/Unset the L{_remember_me} flag.
@param remember_me: True if the session cookie should last one day or
until the browser is closed.
@type remember_me: bool
"""
self._remember_me = remember_me
if remember_me:
self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_REMEMBER *
CFG_WEBSESSION_ONE_DAY)
else:
self.set_timeout(CFG_WEBSESSION_EXPIRY_LIMIT_DEFAULT *
CFG_WEBSESSION_ONE_DAY)
for cookie in self.make_cookies():
self._req.set_cookie(cookie)
def load(self):
"""
Load the session from the database.
@return: 1 in case of success, 0 otherwise.
@rtype: integer
"""
session_dict = None
invalid = False
res = self.load_from_storage(self._sid)
if res:
session_dict = cPickle.loads(blob_to_string(res))
remote_ip = self._req.remote_ip
if self._req.is_https():
if session_dict['_https_ip'] is not None:
if ':' in remote_ip:
## IPV6 address, we don't skip bits
if session_dict['_https_ip'] != remote_ip:
invalid = True
else:
if _mkip(session_dict['_https_ip']) >> \
CFG_WEBSESSION_IPADDR_CHECK_SKIP_BITS != \
_mkip(remote_ip) >> \
CFG_WEBSESSION_IPADDR_CHECK_SKIP_BITS:
invalid = True
else:
session_dict['_https_ip'] = remote_ip
else:
if session_dict['_http_ip'] is not None:
if ':' in remote_ip:
## IPV6 address, we don't skip bits
if session_dict['_http_ip'] != remote_ip:
invalid = True
else:
if _mkip(session_dict['_http_ip']) >> \
CFG_WEBSESSION_IPADDR_CHECK_SKIP_BITS != \
_mkip(remote_ip) >> \
CFG_WEBSESSION_IPADDR_CHECK_SKIP_BITS:
invalid = True
else:
session_dict['_http_ip'] = remote_ip
if session_dict is None:
return 0
if invalid:
return 0
if (time.time() - session_dict["_accessed"]) > \
session_dict["_timeout"]:
return 0
self._created = session_dict["_created"]
self._accessed = session_dict["_accessed"]
self._timeout = session_dict["_timeout"]
self._remember_me = session_dict["_remember_me"]
self.update(session_dict["_data"])
return 1
def is_useful(self):
"""
Return True if the session contains some key considered
useful (i.e. that deserve being preserved)
"""
for key in self:
if key not in _CFG_SESSION_NON_USEFUL_KEYS:
return True
return False
def save(self):
"""
Save the session to the database.
"""
uid = self.get('uid', -1)
if (not self.__need_https or self._req.is_https()) and not self._invalid and self._sid and self._dirty and (uid > 0 or self.is_useful()):
## We store something only for real users or useful sessions.
session_dict = {"_data" : self.copy(),
"_created" : self._created,
"_accessed": self._accessed,
"_timeout" : self._timeout,
"_http_ip" : self._http_ip,
"_https_ip" : self._https_ip,
"_remember_me" : self._remember_me
}
session_object = cPickle.dumps(session_dict, -1)
self.save_in_storage(self._sid,
session_object,
self._timeout,
uid)
for cookie in self.make_cookies():
self._req.set_cookie(cookie)
## No more dirty :-)
self._dirty = False
def delete(self):
"""
Delete the session.
"""
self.delete_from_storage(self._sid)
self.clear()
def invalidate(self):
"""
Declare the session as invalid.
"""
cookies = self.make_cookies()
for cookie in cookies:
cookie.expires = 0
self._req.set_cookie(cookie)
self.delete()
self._invalid = 1
if hasattr(self._req, '_session'):
delattr(self._req, '_session')
def make_cookies(self):
"""
Create the necessary cookies to implement secure session handling
(possibly over HTTPS).
@return: a list of cookies.
"""
cookies = []
uid = self.get('uid', -1)
if uid > 0 and CFG_SITE_SECURE_URL.startswith("https://"):
stub_cookie = Cookie(CFG_WEBSESSION_COOKIE_NAME + 'stub', 'HTTPS', HttpOnly=True)
else:
stub_cookie = Cookie(CFG_WEBSESSION_COOKIE_NAME + 'stub', 'NO', HttpOnly=True)
cookies.append(stub_cookie)
if self._req.is_https() or not CFG_SITE_SECURE_URL.startswith("https://") or uid <= 0:
cookie = Cookie(CFG_WEBSESSION_COOKIE_NAME, self._sid, HttpOnly=True)
if CFG_SITE_SECURE_URL.startswith("https://") and uid > 0:
cookie.secure = True
cookies.append(cookie)
for cookie in cookies:
cookie.path = '/'
if self._remember_me:
cookie.expires = time.time() + self._timeout
return cookies
def initial_http_ip(self):
"""
@return: the initial ip addressed for the HTTP protocol for which this
session was issued.
@rtype: string
@note: it returns None if this session has always been used through
HTTPS requests.
"""
return self._http_ip
def initial_https_ip(self):
"""
@return: the initial ip addressed for the HTTPS protocol for which this
session was issued.
@rtype: string
@note: it returns None if this session has always been used through
HTTP requests.
"""
return self._https_ip
def is_new(self):
"""
@return: True if the session has just been created.
@rtype: bool
"""
return not not self._new
def sid(self):
"""
@return: the session identifier.
@rtype: 32 hexadecimal string
"""
return self._sid
def created(self):
"""
@return: the UNIX timestamp for when the session has been created.
@rtype: double
"""
return self._created
def last_accessed(self):
"""
@return: the UNIX timestamp for when the session has been last
accessed.
@rtype: double
"""
return self._accessed
def timeout(self):
"""
@return: the number of seconds from the last accessed timestamp,
after which the session is invalid.
@rtype: double
"""
return self._timeout
def set_timeout(self, secs):
"""
Set the number of seconds from the last accessed timestamp,
after which the session is invalid.
@param secs: the number of seconds.
@type secs: double
"""
self._timeout = secs
def cleanup(self):
"""
Perform the database session cleanup.
"""
if self._cleanup_function:
self._req.register_cleanup(self._cleanup_function)
self._req.log_error("InvenioSession: registered database cleanup.")
## NOTE: Let's disable __del__ to avoid garbage collection not to
## be able to delete circular references involving the session
## We can .save() anyway in good points, such as at the end of
## of the application request
#def __del__(self):
#self.save()
def get_need_https(self):
return self.__need_https
## This property will be True if the connection need to be set to HTTPS
## in order for the session to be successfully read. This can actually
## be checked by not having a cookie, but just having the stub_cookie.
## The default cookie is only sent via HTTPS, while the stub_cookie
## is also sent via HTTP and contains the uid, of the user. So if there
## is actually a stub cookie and its value is different than -1 this
## property will be True, meaning the server should redirect the client
## to an HTTPS connection if she really wants to access authenticated
## resources.
need_https = property(get_need_https)
def _init_rnd():
"""
Initialize random number generators.
This is key in multithreaded env, see Python docs for random.
@return: the generators.
@rtype: list of generators
"""
# query max number of threads
gennum = 10
# make generators
# this bit is from Python lib reference
random_generator = random.Random(time.time())
result = [random_generator]
for dummy in range(gennum - 1):
laststate = random_generator.getstate()
random_generator = random.Random()
random_generator.setstate(laststate)
random_generator.jumpahead(1000000)
result.append(random_generator)
return result
_RANDOM_GENERATORS = _init_rnd()
_RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)
def _get_generator():
"""
get rnd_iter.next(), or start over
if we reached the end of it
@return: the next random number.
@rtype: double
"""
global _RANDOM_ITERATOR
try:
return _RANDOM_ITERATOR.next()
except StopIteration:
# the small potential for two threads doing this
# seems does not warrant use of a lock
_RANDOM_ITERATOR = iter(_RANDOM_GENERATORS)
return _RANDOM_ITERATOR.next()
_RE_VALIDATE_SID = re.compile('[0-9a-f]{32}$')
def _check_sid(sid):
"""
Check the validity of the session identifier.
The sid must be 32 characters long, and consisting of the characters
0-9 and a-f.
The sid may be passed in a cookie from the client and as such
should not be trusted. This is particularly important in
FileSession, where the session filename is derived from the sid.
A sid containing '/' or '.' characters could result in a directory
traversal attack
@param sid: the session identifier.
@type sid: string
@return: True if the session identifier is valid.
@rtype: bool
"""
return not not _RE_VALIDATE_SID.match(sid)
def _new_sid(req):
"""
Make a number based on current time, pid, remote ip
and two random ints, then hash with md5. This should
be fairly unique and very difficult to guess.
@param req: the mod_python request object.
@type req: mod_python request object.
@return: the session identifier.
@rtype: 32 hexadecimal string
@warning: The current implementation of _new_sid returns an
md5 hexdigest string. To avoid a possible directory traversal
attack in FileSession the sid is validated using
the _check_sid() method and the compiled regex
validate_sid_re. The sid will be accepted only if len(sid) == 32
and it only contains the characters 0-9 and a-f.
If you change this implementation of _new_sid, make sure to also
change the validation scheme, as well as the test_Session_illegal_sid()
unit test in test/test.py.
"""
the_time = long(time.time()*10000)
pid = os.getpid()
random_generator = _get_generator()
rnd1 = random_generator.randint(0, 999999999)
rnd2 = random_generator.randint(0, 999999999)
remote_ip = req.remote_ip
return md5("%d%d%d%d%s" % (
the_time,
pid,
rnd1,
rnd2,
remote_ip)
).hexdigest()
def _mkip(ip):
"""
Compute a numerical value for a dotted IP
"""
num = 0L
for i in ip.split('.'):
num = (num << 8) + int(i)
return num
class InvenioSessionMySQL(InvenioSessionBase):
def __init__(self, req, sid=None):
def cb_session_cleanup(data=None):
"""
Session cleanup procedure which to be executed at the end
of the request handling.
"""
run_sql("""DELETE LOW_PRIORITY FROM session
WHERE session_expiry <= UTC_TIMESTAMP()""")
self.cleanup_function = cb_session_cleanup
super(InvenioSessionMySQL, self).__init__(req, sid)
def load_from_storage(self, sid):
ret = run_sql("""SELECT session_object FROM session
WHERE session_key = %s""", [sid])
if ret:
return ret[0][0]
def delete_from_storage(self, sid):
return run_sql("""DELETE LOW_PRIORITY FROM session
WHERE session_key=%s""", [sid])
def save_in_storage(self, sid, session_object, timeout, uid):
session_key = sid
session_expiry = time.time() + timeout + CFG_WEBSESSION_ONE_DAY
session_expiry = convert_datestruct_to_datetext(time.gmtime(session_expiry))
run_sql("""INSERT INTO session(
session_key,
session_expiry,
session_object,
uid
- ) VALUES (%s, %s, %s, %s)
+ ) VALUES (%s, %s, _binary %s, %s)
ON DUPLICATE KEY UPDATE
session_expiry=%s,
- session_object=%s,
+ session_object=_binary %s,
uid=%s
""", (session_key, session_expiry, session_object, uid,
session_expiry, session_object, uid))
class InvenioSessionRedis(InvenioSessionBase):
def generate_key(self, sid):
return 'session_%s' % sid
def load_from_storage(self, sid):
return get_redis().get(self.generate_key(sid))
def delete_from_storage(self, sid):
return get_redis().delete(self.generate_key(sid))
def save_in_storage(self, sid, session_object, timeout, uid): # pylint: disable=W0613
return get_redis().setex(self.generate_key(sid),
session_object,
timeout)
if CFG_WEBSESSION_STORAGE == 'mysql':
InvenioSession = InvenioSessionMySQL
elif CFG_WEBSESSION_STORAGE == 'redis':
InvenioSession = InvenioSessionRedis
diff --git a/requirements.txt b/requirements.txt
index 160ec6b0b..77a74e64f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,31 +1,31 @@
# Invenio requirements.
MySQL-python==1.2.5
rdflib==2.4.2
reportlab==2.5
python-dateutil<=1.9999
python-magic==0.4.2
-http://www.reportlab.com/ftp/pyRXP-1.16-daily-unix.tar.gz
+http://invenio-software.org/download/python/pyRXP-1.16-daily-unix.tar.gz
numpy==1.7.0
lxml==3.1.2
mechanize==0.2.5
python-Levenshtein==0.10.2
PyStemmer==1.3.0
-https://py-editdist.googlecode.com/files/py-editdist-0.3.tar.gz
+http://invenio-software.org/download/python/py-editdist-0.3.tar.gz
feedparser==5.1.3
BeautifulSoup==3.2.1
beautifulsoup4==4.1.3
python-twitter==2.0
msgpack-python==0.3.0
pyparsing==1.5.6
requests==2.2.0
PyPDF2==1.19
rauth==0.6.2
unidecode==0.04.14
python-openid==2.2.5
qrcode==4.0.4
pillow==2.3.0
jinja2==2.7.2
redis==2.9.0
nydus==0.10.6
Cerberus==0.5
matplotlib==1.0.1
diff --git a/scripts/create-instance.sh b/scripts/create-instance.sh
index 39257fd04..d52334771 100755
--- a/scripts/create-instance.sh
+++ b/scripts/create-instance.sh
@@ -1,264 +1,264 @@
#!/usr/bin/env bash
#
# This file is part of Invenio.
# Copyright (C) 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
# quit on errors:
set -o errexit
# check environment variables:
if [ "${INVENIO_MYSQL_HOST}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_MYSQL_HOST=192.168.50.11"
exit 1
fi
if [ "${INVENIO_MYSQL_DBNAME}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBNAME before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBNAME=invenio1"
exit 1
fi
if [ "${INVENIO_MYSQL_DBUSER}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBUSER before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBUSER=invenio1"
exit 1
fi
if [ "${INVENIO_MYSQL_DBPASS}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBPASS before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBPASS=dbpass123"
exit 1
fi
if [ "${INVENIO_WEB_HOST}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10"
exit 1
fi
if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio"
exit 1
fi
if [ "${INVENIO_WEB_USER}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_USER=www-data"
exit 1
fi
# check optional environment variables:
INVENIO_WEB_SMTP_PORT=${INVENIO_WEB_SMTP_PORT:=25}
# quit on unbound symbols:
set -o nounset
# runs inside virtual environment?
VIRTUAL_ENV=${VIRTUAL_ENV:=}
# runs as root or needs sudo?
if [[ "$EUID" -ne 0 ]]; then
sudo='sudo'
else
sudo=''
fi
create_apache_vhost_ubuntu_precise () {
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y ssl-cert
sudo mkdir -p /etc/apache2/ssl
if [ ! -e /etc/apache2/ssl/apache.pem ]; then
sudo DEBIAN_FRONTEND=noninteractive /usr/sbin/make-ssl-cert \
/usr/share/ssl-cert/ssleay.cnf /etc/apache2/ssl/apache.pem
fi
if [ ! -L /etc/apache2/sites-available/invenio.conf ]; then
sudo ln -fs "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" \
/etc/apache2/sites-available/invenio.conf
fi
if [ ! -e "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" ]; then
# create them empty for the time being so that apache would start
sudo mkdir -p "${INVENIO_WEB_DSTDIR}/etc/apache/"
sudo touch "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf"
sudo chown -R "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}"
fi
if [ ! -L /etc/apache2/sites-available/invenio-ssl.conf ]; then
sudo ln -fs "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" \
/etc/apache2/sites-available/invenio-ssl.conf
fi
if [ ! -e "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" ]; then
# create them empty for the time being so that apache would start
sudo mkdir -p "${INVENIO_WEB_DSTDIR}/etc/apache/"
sudo touch "${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf"
sudo chown -R "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}"
fi
if [ -e /etc/apache2/sites-available/default-ssl ]; then
sudo /usr/sbin/a2dissite "*default*"
fi
sudo /usr/sbin/a2ensite "invenio*"
sudo /usr/sbin/a2enmod ssl
sudo /usr/sbin/a2enmod version || echo "[WARNING] Ignoring 'a2enmod version' command; hoping IfVersion is built-in."
sudo /usr/sbin/a2enmod xsendfile
sudo /etc/init.d/apache2 restart
}
create_apache_vhost_centos6 () {
if ! grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" /etc/httpd/conf/httpd.conf; then
echo "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" | sudo tee -a /etc/httpd/conf/httpd.conf
fi
if ! grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" /etc/httpd/conf/httpd.conf; then
echo "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" | sudo tee -a /etc/httpd/conf/httpd.conf
fi
if ! grep -q "TraceEnable off" /etc/httpd/conf/httpd.conf; then
echo "TraceEnable off" | sudo tee -a /etc/httpd/conf/httpd.conf
fi
if ! grep -q "SSLProtocol all -SSLv2" /etc/httpd/conf/httpd.conf; then
echo "SSLProtocol all -SSLv2" | sudo tee -a /etc/httpd/conf/httpd.conf
fi
sudo sed -i 's,^Alias /error/,#Alias /error/,g' /etc/httpd/conf/httpd.conf
}
create_symlinks () {
$sudo mkdir -p "${INVENIO_WEB_DSTDIR}"
$sudo chown "${INVENIO_WEB_USER}.${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}"
$sudo -u "${INVENIO_WEB_USER}" mkdir -p "${INVENIO_WEB_DSTDIR}/lib/python/invenio"
for pythonversion in python2.4 python2.6 python2.7; do
for libversion in lib lib64 local/lib local/lib64; do
for packageversion in site-packages dist-packages; do
if [ -d "/usr/$libversion/$pythonversion/$packageversion/" ] && [ ! -L "/usr/$libversion/$pythonversion/$packageversion/invenio" ]; then
$sudo ln -s "${INVENIO_WEB_DSTDIR}/lib/python/invenio" "/usr/$libversion/$pythonversion/$packageversion/invenio"
fi
done
done
done
}
install_sources () {
cd "${INVENIO_SRCDIR}"
rm -rf autom4te.cache/
aclocal
automake -a
autoconf
./configure --prefix="${INVENIO_WEB_DSTDIR}"
make clean -s
make -s
sudo -u "${INVENIO_WEB_USER}" make -s install
- #sudo -u "${INVENIO_WEB_USER}" make -s install-jquery-plugins
+ sudo -u "${INVENIO_WEB_USER}" make -s install-jquery-plugins
sudo -u "${INVENIO_WEB_USER}" make -s install-mathjax-plugin
sudo -u "${INVENIO_WEB_USER}" make -s install-ckeditor-plugin
sudo -u "${INVENIO_WEB_USER}" make -s install-pdfa-helper-files
sudo -u "${INVENIO_WEB_USER}" make -s install-mediaelement
}
create_openoffice_tmp_space () {
sudo mkdir -p "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files"
sudo chown -R nobody "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files"
sudo chmod -R 755 "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files"
}
configure_instance () {
# create invenio-local.conf
echo "[Invenio]
CFG_SITE_URL = http://${INVENIO_WEB_HOST}
CFG_SITE_SECURE_URL = https://${INVENIO_WEB_HOST}
CFG_DATABASE_HOST = ${INVENIO_MYSQL_HOST}
CFG_DATABASE_NAME = ${INVENIO_MYSQL_DBNAME}
CFG_DATABASE_USER = ${INVENIO_MYSQL_DBUSER}
CFG_DATABASE_PASS = ${INVENIO_MYSQL_DBPASS}
CFG_SITE_ADMIN_EMAIL = ${INVENIO_ADMIN_EMAIL}
CFG_SITE_SUPPORT_EMAIL = ${INVENIO_ADMIN_EMAIL}
CFG_WEBALERT_ALERT_ENGINE_EMAIL = ${INVENIO_ADMIN_EMAIL}
CFG_WEBCOMMENT_ALERT_ENGINE_EMAIL = ${INVENIO_ADMIN_EMAIL}
CFG_WEBCOMMENT_DEFAULT_MODERATOR = ${INVENIO_ADMIN_EMAIL}
CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL = ${INVENIO_ADMIN_EMAIL}
CFG_BIBCATALOG_SYSTEM_EMAIL_ADDRESS = ${INVENIO_ADMIN_EMAIL}
CFG_BIBSCHED_PROCESS_USER = ${INVENIO_WEB_USER}
CFG_MISCUTIL_SMTP_PORT = ${INVENIO_WEB_SMTP_PORT}
" | \
sudo -u "${INVENIO_WEB_USER}" tee "${INVENIO_WEB_DSTDIR}/etc/invenio-local.conf"
# update instance with this information:
sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --update-all
}
create_tables () {
sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --create-tables --yes-i-know
}
create_apache_configuration () {
sudo -u "${INVENIO_WEB_USER}" VIRTUAL_ENV="${VIRTUAL_ENV}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --create-apache-conf
}
restart_apache_ubuntu_precise () {
$sudo /etc/init.d/apache2 restart
}
restart_apache_centos6 () {
$sudo /etc/init.d/httpd restart
}
main () {
# detect OS distribution and release version:
if hash lsb_release 2> /dev/null; then
os_distribution=$(lsb_release -i | cut -f 2)
os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1)
elif [ -e /etc/redhat-release ]; then
os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release)
os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1)
else
os_distribution="UNDETECTED"
os_release="UNDETECTED"
fi
# call appropriate provisioning functions:
if [ "$os_distribution" = "Ubuntu" ]; then
if [ "$os_release" = "12" ]; then
create_apache_vhost_ubuntu_precise
create_symlinks
install_sources
create_openoffice_tmp_space
configure_instance
create_tables
create_apache_configuration
restart_apache_ubuntu_precise
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
elif [ "$os_distribution" = "CentOS" ]; then
if [ "$os_release" = "6" ]; then
create_apache_vhost_centos6
create_symlinks
install_sources
create_openoffice_tmp_space
configure_instance
create_tables
create_apache_configuration
restart_apache_centos6
exit 1
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
else
echo "[ERROR] Sorry, unsupported distribution ${os_distribution}."
exit 1
fi
}
main
diff --git a/scripts/drop-instance.sh b/scripts/drop-instance.sh
index ae23a0443..8f0a81369 100755
--- a/scripts/drop-instance.sh
+++ b/scripts/drop-instance.sh
@@ -1,184 +1,185 @@
#!/usr/bin/env bash
#
# This file is part of Invenio.
# Copyright (C) 2015, 2016 CERN.
#
# Invenio is free software; you can redistribute it
# and/or modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307, USA.
#
# In applying this license, CERN does not
# waive the privileges and immunities granted to it by virtue of its status
# as an Intergovernmental Organization or submit itself to any jurisdiction.
# quit on errors:
set -o errexit
# check environment variables:
if [ "${INVENIO_MYSQL_HOST}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_MYSQL_HOST=192.168.50.11"
exit 1
fi
if [ "${INVENIO_MYSQL_DBNAME}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBNAME before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBNAME=invenio1"
exit 1
fi
if [ "${INVENIO_MYSQL_DBUSER}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBUSER before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBUSER=invenio1"
exit 1
fi
if [ "${INVENIO_MYSQL_DBPASS}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_MYSQL_DBPASS before runnning this script."
echo "[ERROR] Example: INVENIO_MYSQL_DBPASS=dbpass123"
exit 1
fi
if [ "${INVENIO_WEB_HOST}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_HOST before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_HOST=192.168.50.10"
exit 1
fi
if [ "${INVENIO_WEB_DSTDIR}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_DSTDIR before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_DSTDIR=/opt/invenio"
exit 1
fi
if [ "${INVENIO_WEB_USER}" = "" ]; then
echo "[ERROR] Please set environment variable INVENIO_WEB_USER before runnning this script."
echo "[ERROR] Example: export INVENIO_WEB_USER=www-data"
exit 1
fi
# quit on unbound symbols:
set -o nounset
# runs as root or needs sudo?
if [[ "$EUID" -ne 0 ]]; then
sudo='sudo'
else
sudo=''
fi
start_apache_ubuntu_precise () {
$sudo /etc/init.d/apache2 start
}
stop_apache_ubuntu_precise () {
$sudo /etc/init.d/apache2 stop
}
start_apache_centos6 () {
$sudo /etc/init.d/httpd start
}
stop_apache_centos6 () {
$sudo /etc/init.d/httpd stop
}
drop_apache_vhost_ubuntu_precise () {
stop_apache_ubuntu_precise
if [ -e /etc/apache2/sites-available/default-ssl ]; then
$sudo /usr/sbin/a2ensite "*default*"
fi
if [ -L /etc/apache2/sites-enabled/invenio.conf ]; then
$sudo /usr/sbin/a2dissite "invenio*"
fi
start_apache_ubuntu_precise
}
drop_apache_vhost_centos6 () {
stop_apache_centos6
if grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf" /etc/httpd/conf/httpd.conf; then
sudo sed -i "s,^Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf,#Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost.conf,g" /etc/httpd/conf/httpd.conf
fi
if grep -q "Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf" /etc/httpd/conf/httpd.conf; then
sudo sed -i "s,^Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf,#Include ${INVENIO_WEB_DSTDIR}/etc/apache/invenio-apache-vhost-ssl.conf,g" /etc/httpd/conf/httpd.conf
fi
sudo sed -i 's,^#Alias /error/,Alias /error/,g' /etc/httpd/conf/httpd.conf
start_apache_centos6
}
drop_symlinks () {
for pythonversion in python2.4 python2.6 python2.7; do
for libversion in lib lib64 local/lib local/lib64; do
for packageversion in site-packages dist-packages; do
if [ -d /usr/$libversion/$pythonversion/$packageversion/ ] && [ ! -L /usr/$libversion/$pythonversion/$packageversion/invenio ]; then
$sudo rm /usr/$libversion/$pythonversion/$packageversion/invenio
fi
done
done
done
}
drop_instance_folder () {
$sudo rm -rf "${INVENIO_WEB_DSTDIR}/var/tmp/ooffice-tmp-files"
- $sudo -u "${INVENIO_WEB_USER}" rm -rf "${INVENIO_WEB_DSTDIR}/*"
+ # shellcheck disable=SC2086
+ $sudo -u "${INVENIO_WEB_USER}" rm -rf ${INVENIO_WEB_DSTDIR}/*
}
drop_instance_tables () {
if [ -e "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" ]; then
$sudo -u "${INVENIO_WEB_USER}" "${INVENIO_WEB_DSTDIR}/bin/inveniocfg" --drop-tables --yes-i-know
fi
}
main () {
# detect OS distribution and release version:
if hash lsb_release 2> /dev/null; then
os_distribution=$(lsb_release -i | cut -f 2)
os_release=$(lsb_release -r | cut -f 2 | grep -oE '[0-9]+\.' | cut -d. -f1 | head -1)
elif [ -e /etc/redhat-release ]; then
os_distribution=$(cut -d ' ' -f 1 /etc/redhat-release)
os_release=$(grep -oE '[0-9]+\.' /etc/redhat-release | cut -d. -f1 | head -1)
else
os_distribution="UNDETECTED"
os_release="UNDETECTED"
fi
# call appropriate provisioning functions:
if [ "$os_distribution" = "Ubuntu" ]; then
if [ "$os_release" = "12" ]; then
stop_apache_ubuntu_precise
drop_instance_tables
start_apache_ubuntu_precise
drop_apache_vhost_ubuntu_precise
drop_instance_folder
drop_symlinks
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
elif [ "$os_distribution" = "CentOS" ]; then
if [ "$os_release" = "6" ]; then
stop_apache_centos6
drop_instance_tables
start_apache_centos6
drop_apache_vhost_centos6
drop_instance_folder
drop_symlinks
else
echo "[ERROR] Sorry, unsupported release ${os_release}."
exit 1
fi
else
echo "[ERROR] Sorry, unsupported distribution ${os_distribution}."
exit 1
fi
}
main