')
h(content)
h('
')
return "\n".join(html)
def tmpl_notification_box(self, message, teaser="Notice:", show_close_btn=True):
'''
Creates a notification box based on the jQuery UI style
@param message: message to display in the box
@type message: string
@param teaser: Teaser text in bold next to icon
@type teaser: string
@param show_close_btn: display close button [x]
@type show_close_btn: boolean
@return: HTML code
@rtype: string
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_error_box(self, message, teaser="Alert:", show_close_btn=True):
'''
Creates an error box based on the jQuery UI style
@param message: message to display in the box
@type message: string
@param teaser: Teaser text in bold next to icon
@type teaser: string
@param show_close_btn: display close button [x]
@type show_close_btn: boolean
@return: HTML code
@rtype: string
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_ticket_box(self, teaser, message, show_close_btn=True):
'''
Creates a semi-permanent box informing about ticket
status notifications
@param message: message to display in the box
@type message: string
@param teaser: Teaser text in bold next to icon
@type teaser: string
@param ticket: The ticket object from the session
@param ticket: list of dict
@param show_close_btn: display close button [x]
@type show_close_btn: boolean
@return: HTML code
@rtype: string
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_search_ticket_box(self, teaser, message, search_ticket, show_close_btn=False):
'''
Creates a box informing about a claim in progress for
the search.
@param message: message to display in the box
@type message: string
@param teaser: Teaser text in bold next to icon
@type teaser: string
@param search_ticket: The search ticket object from the session
@param search_ticket: list of dict
@param show_close_btn: display close button [x]
@type show_close_btn: boolean
@return: HTML code
@rtype: string
'''
html = []
h = html.append
h('')
h('') return "\n".join(html) def tmpl_meta_includes(self, kill_browser_cache=False): ''' Generates HTML code for the header section of the document META tags to kill browser caching Javascript includes CSS definitions @param kill_browser_cache: Do we want to kill the browser cache? @type kill_browser_cache: boolean ''' js_path = "%s/js" % CFG_SITE_URL imgcss_path = "%s/img" % CFG_SITE_URL result = [] # Add browser cache killer, hence some notifications are not displayed # out of the session. if kill_browser_cache: result = [ '', '', '', '', ''] scripts = ["jquery-ui.min.js", "jquery.form.js", "jquery.dataTables.min.js", "bibauthorid.js"] result.append('' % (imgcss_path)) result.append('' % (imgcss_path)) result.append('' % (imgcss_path)) for script in scripts: result.append('' % (js_path, script)) return "\n".join(result) def tmpl_author_confirmed(self, bibref, pid, verbiage_dict={'alt_confirm':'Confirmed.', 'confirm_text':'This record assignment has been confirmed.', 'alt_forget':'Forget decision!', 'forget_text':'Forget assignment decision', 'alt_repeal':'Repeal!', 'repeal_text':'Repeal record assignment', 'to_other_text':'Assign to another person', 'alt_to_other':'To other person!' }, show_reset_button=True): ''' Generate play per-paper links for the table for the status "confirmed" @param bibref: construct of unique ID for this author on this paper @type bibref: string @param pid: the Person ID @type pid: int @param verbiage_dict: language for the link descriptions @type verbiage_dict: dict ''' stri = (' ' '
s/img/aid_check.png)
'
'%(confirm_text)s ') if show_reset_button: stri = stri + ( '' '
s/img/aid_reset_gray.png)
'
'%(forget_text)s') stri = stri + ( '' '
s/img/aid_reject_gray.png)
'
'%(repeal_text)s' '' '
s/img/aid_to_other_gray.png)
'
'%(to_other_text)s ')
return (stri
% ({'url': CFG_SITE_URL, 'ref': bibref, 'pid': pid,
'alt_confirm':verbiage_dict['alt_confirm'],
'confirm_text':verbiage_dict['confirm_text'],
'alt_forget':verbiage_dict['alt_forget'],
'forget_text':verbiage_dict['forget_text'],
'alt_repeal':verbiage_dict['alt_repeal'],
'repeal_text':verbiage_dict['repeal_text'],
'to_other_text':verbiage_dict['to_other_text'],
'alt_to_other':verbiage_dict['alt_to_other']}))
def tmpl_author_repealed(self, bibref, pid, verbiage_dict={'alt_confirm':'Confirm!',
'confirm_text':'Confirm record assignment.',
'alt_forget':'Forget decision!',
'forget_text':'Forget assignment decision',
'alt_repeal':'Rejected!',
'repeal_text':'Repeal this record assignment.',
'to_other_text':'Assign to another person',
'alt_to_other':'To other person!'
},
show_reset_button=True):
'''
Generate play per-paper links for the table for the
status "repealed"
@param bibref: construct of unique ID for this author on this paper
@type bibref: string
@param pid: the Person ID
@type pid: int
@param verbiage_dict: language for the link descriptions
@type verbiage_dict: dict
'''
stri = (' '
's/img/aid_reject.png)
'
'%(repeal_text)s ') if show_reset_button: stri = stri + ( '' '
'
'%(forget_text)s') stri = stri + ( '' '
s/img/aid_check_gray.png)
'
'%(confirm_text)s' '' '
'
'%(to_other_text)s ')
return (stri
% ({'url': CFG_SITE_URL, 'ref': bibref, 'pid': pid,
'alt_confirm':verbiage_dict['alt_confirm'],
'confirm_text':verbiage_dict['confirm_text'],
'alt_forget':verbiage_dict['alt_forget'],
'forget_text':verbiage_dict['forget_text'],
'alt_repeal':verbiage_dict['alt_repeal'],
'repeal_text':verbiage_dict['repeal_text'],
'to_other_text':verbiage_dict['to_other_text'],
'alt_to_other':verbiage_dict['alt_to_other']}))
def tmpl_author_undecided(self, bibref, pid, verbiage_dict={'alt_confirm':'Confirm!',
'confirm_text':'Confirm record assignment.',
'alt_repeal':'Rejected!',
'repeal_text':'This record has been repealed.',
'to_other_text':'Assign to another person',
'alt_to_other':'To other person!'
},
show_reset_button=True):
'''
Generate play per-paper links for the table for the
status "no decision taken yet"
@param bibref: construct of unique ID for this author on this paper
@type bibref: string
@param pid: the Person ID
@type pid: int
@param verbiage_dict: language for the link descriptions
@type verbiage_dict: dict
'''
#batchprocess?mconfirm=True&bibrefs=['100:17,16']&pid=1
str = (' '
''
''
'%(confirm_text)s' '' '
'
'%(repeal_text)s ' '' '
s/img/aid_to_other.png)
'
'%(to_other_text)s ')
return (str
% ({'url': CFG_SITE_URL, 'ref': bibref, 'pid': pid,
'alt_confirm':verbiage_dict['alt_confirm'],
'confirm_text':verbiage_dict['confirm_text'],
'alt_repeal':verbiage_dict['alt_repeal'],
'repeal_text':verbiage_dict['repeal_text'],
'to_other_text':verbiage_dict['to_other_text'],
'alt_to_other':verbiage_dict['alt_to_other']}))
def tmpl_open_claim(self, bibrefs, pid, last_viewed_pid,
search_enabled=True):
'''
Generate entry page for "claim or attribute this paper"
@param bibref: construct of unique ID for this author on this paper
@type bibref: string
@param pid: the Person ID
@type pid: int
@param last_viewed_pid: last ID that had been subject to an action
@type last_viewed_pid: int
'''
t_html = []
h = t_html.append
h(self._('You are about to attribute the following paper'))
if len(bibrefs) > 1:
h('s: ') else: h(':
') h("
- ")
bibs = ''
for paper in bibrefs:
if bibs:
bibs = bibs + '&'
bibs = bibs + 'selection=' + str(paper)
if ',' in paper:
pbibrec = paper.split(',')[1]
else:
pbibrec = paper
h("
- %s " % (format_record(pbibrec, "ha"))) h("
Your options:
') if pid > -1: h((' Claim for yourself') % (CFG_SITE_URL, bibs, str(pid))) if last_viewed_pid: h((' Attribute to %s
') % (CFG_SITE_URL, bibs, str(last_viewed_pid[0]), last_viewed_pid[1])) if search_enabled: h(('' + self._(' Search for a person to attribute the paper to') + '
') % (CFG_SITE_URL, bibs)) return "\n".join(pp_html) def __tmpl_admin_records_table(self, form_id, person_id, bibrecids, verbiage_dict={'no_doc_string':'Sorry, there are currently no documents to be found in this category.', 'b_confirm':'Confirm', 'b_repeal':'Repeal', 'b_to_others':'Assign to other person', 'b_forget':'Forget decision'}, buttons_verbiage_dict={'mass_buttons':{'no_doc_string':'Sorry, there are currently no documents to be found in this category.', 'b_confirm':'Confirm', 'b_repeal':'Repeal', 'b_to_others':'Assign to other person', 'b_forget':'Forget decision'}, 'record_undecided':{'alt_confirm':'Confirm!', 'confirm_text':'Confirm record assignment.', 'alt_repeal':'Rejected!', 'repeal_text':'This record has been repealed.'}, 'record_confirmed':{'alt_confirm':'Confirmed.', 'confirm_text':'This record assignment has been confirmed.', 'alt_forget':'Forget decision!', 'forget_text':'Forget assignment decision', 'alt_repeal':'Repeal!', 'repeal_text':'Repeal record assignment'}, 'record_repealed':{'alt_confirm':'Confirm!', 'confirm_text':'Confirm record assignment.', 'alt_forget':'Forget decision!', 'forget_text':'Forget assignment decision', 'alt_repeal':'Rejected!', 'repeal_text':'Repeal this record assignment.'}}, show_reset_button=True): ''' Generate the big tables for the person overview page @param form_id: name of the form @type form_id: string @param person_id: Person ID @type person_id: int @param bibrecids: List of records to display @type bibrecids: list @param verbiage_dict: language for the elements @type verbiage_dict: dict @param buttons_verbiage_dict: language for the buttons @type buttons_verbiage_dict: dict ''' no_papers_html = ['
']
no_papers_html.append('%s' % verbiage_dict['no_doc_string'])
no_papers_html.append('
')
if not bibrecids or not person_id:
return "\n".join(no_papers_html)
pp_html = []
h = pp_html.append
h('")
return "\n".join(pp_html)
def __tmpl_reviews_table(self, person_id, bibrecids, admin=False):
'''
Generate the table for potential reviews.
@param form_id: name of the form
@type form_id: string
@param person_id: Person ID
@type person_id: int
@param bibrecids: List of records to display
@type bibrecids: list
@param admin: Show admin functions
@type admin: boolean
'''
no_papers_html = ['']
no_papers_html.append(self._('Sorry, there are currently no records to be found in this category.'))
no_papers_html.append('
')
if not bibrecids or not person_id:
return "\n".join(no_papers_html)
pp_html = []
h = pp_html.append
h('')
return "\n".join(pp_html)
def tmpl_admin_person_info_box(self, ln, person_id= -1, names=[]):
'''
Generate the box showing names
@param ln: the language to use
@type ln: string
@param person_id: Person ID
@type person_id: int
@param names: List of names to display
@type names: list
'''
html = []
h = html.append
if not ln:
pass
#class="ui-tabs ui-widget ui-widget-content ui-corner-all">
h('' + self._('Names variants:') + '')
h("
")
return "\n".join(html)
def tmpl_admin_tabs(self, ln=CFG_SITE_LANG, person_id= -1,
rejected_papers=[],
rest_of_papers=[],
review_needed=[],
rt_tickets=[],
open_rt_tickets=[],
show_tabs=['records', 'repealed', 'review', 'comments', 'tickets', 'data'],
show_reset_button=True,
ticket_links=['delete', 'commit', 'del_entry', 'commit_entry'],
verbiage_dict={'confirmed':'Records', 'repealed':'Not this person\'s records',
'review':'Records in need of review',
'tickets':'Open Tickets', 'data':'Data',
'confirmed_ns':'Papers of this Person',
'repealed_ns':'Papers _not_ of this Person',
'review_ns':'Papers in need of review',
'tickets_ns':'Tickets for this Person',
'data_ns':'Additional Data for this Person'},
buttons_verbiage_dict={'mass_buttons':{'no_doc_string':'Sorry, there are currently no documents to be found in this category.',
'b_confirm':'Confirm',
'b_repeal':'Repeal',
'b_to_others':'Assign to other person',
'b_forget':'Forget decision'},
'record_undecided':{'alt_confirm':'Confirm!',
'confirm_text':'Confirm record assignment.',
'alt_repeal':'Rejected!',
'repeal_text':'This record has been repealed.'},
'record_confirmed':{'alt_confirm':'Confirmed.',
'confirm_text':'This record assignment has been confirmed.',
'alt_forget':'Forget decision!',
'forget_text':'Forget assignment decision',
'alt_repeal':'Repeal!',
'repeal_text':'Repeal record assignment'},
'record_repealed':{'alt_confirm':'Confirm!',
'confirm_text':'Confirm record assignment.',
'alt_forget':'Forget decision!',
'forget_text':'Forget assignment decision',
'alt_repeal':'Rejected!',
'repeal_text':'Repeal this record assignment.'}}):
'''
Generate the tabs for the person overview page
@param ln: the language to use
@type ln: string
@param person_id: Person ID
@type person_id: int
@param rejected_papers: list of repealed papers
@type rejected_papers: list
@param rest_of_papers: list of attributed of undecided papers
@type rest_of_papers: list
@param review_needed: list of papers that need a review (choose name)
@type review_needed:list
@param rt_tickets: list of tickets for this Person
@type rt_tickets: list
@param open_rt_tickets: list of open request tickets
@type open_rt_tickets: list
@param show_tabs: list of tabs to display
@type show_tabs: list of strings
@param ticket_links: list of links to display
@type ticket_links: list of strings
@param verbiage_dict: language for the elements
@type verbiage_dict: dict
@param buttons_verbiage_dict: language for the buttons
@type buttons_verbiage_dict: dict
'''
html = []
h = html.append
h('")
h(''
% (person_id, person_id))
for name in names:
# h(("%s "+self._('as appeared on')+" %s"+self._(' records')+"
")
# % (name[0], name[1]))
h(("%s (%s); ")
% (name[0], name[1]))
h("
')
h('
")
return "\n".join(html)
def tmpl_bibref_check(self, bibrefs_auto_assigned, bibrefs_to_confirm):
'''
Generate overview to let user chose the name on the paper that
resembles the person in question.
@param bibrefs_auto_assigned: list of auto-assigned papers
@type bibrefs_auto_assigned: list
@param bibrefs_to_confirm: list of unclear papers and names
@type bibrefs_to_confirm: list
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_invenio_search_box(self):
'''
Generate little search box for missing papers. Links to main invenio
search on start papge.
'''
html = []
h = html.append
h('- ')
if 'records' in show_tabs:
r = verbiage_dict['confirmed']
h('
- %(r)s (%(l)s) ' % ({'r':r, 'l':len(rest_of_papers)})) if 'repealed' in show_tabs: r = verbiage_dict['repealed'] h('
- %(r)s (%(l)s) ' % ({'r':r, 'l':len(rejected_papers)})) if 'review' in show_tabs: r = verbiage_dict['review'] h('
- %(r)s (%(l)s) ' % ({'r':r, 'l':len(review_needed)})) if 'tickets' in show_tabs: r = verbiage_dict['tickets'] h('
- %(r)s (%(l)s) ' % ({'r':r, 'l':len(open_rt_tickets)})) if 'data' in show_tabs: r = verbiage_dict['data'] h('
- %s ' % r) h('
')
r = verbiage_dict['confirmed_ns']
h('' % r)
h(self.__tmpl_admin_records_table("massfunctions",
person_id, rest_of_papers,
verbiage_dict=buttons_verbiage_dict['mass_buttons'],
buttons_verbiage_dict=buttons_verbiage_dict,
show_reset_button=show_reset_button))
h("
")
if 'repealed' in show_tabs:
h(' ')
r = verbiage_dict['repealed_ns']
h('' % r)
h(self._('These records have been marked as not being from this person.'))
h('
' + self._('They will be regarded in the next run of the author ') + self._('disambiguation algorithm and might disappear from this listing.')) h(self.__tmpl_admin_records_table("rmassfunctions", person_id, rejected_papers, verbiage_dict=buttons_verbiage_dict['mass_buttons'], buttons_verbiage_dict=buttons_verbiage_dict, show_reset_button=show_reset_button)) h("
")
if 'review' in show_tabs:
h(' ' + self._('They will be regarded in the next run of the author ') + self._('disambiguation algorithm and might disappear from this listing.')) h(self.__tmpl_admin_records_table("rmassfunctions", person_id, rejected_papers, verbiage_dict=buttons_verbiage_dict['mass_buttons'], buttons_verbiage_dict=buttons_verbiage_dict, show_reset_button=show_reset_button)) h("
')
r = verbiage_dict['review_ns']
h('' % r)
h(self.__tmpl_reviews_table(person_id, review_needed, True))
h('
')
if 'tickets' in show_tabs:
h(' ')
r = verbiage_dict['tickets']
h('' % r)
r = verbiage_dict['tickets_ns']
h('
') % ({'tnum':t[1], 'url':CFG_SITE_URL, 'pid':str(person_id)})) h('')
h('Open from: %s, %s
' % (surname, name)) h('Date: %s
' % date) h('identified by: %s
' % uidip) h('email: %s
' % email) h('comments: %s
' % comments) h('Suggested actions:
') h('')
for a in actions:
bibref, bibrec = a[1].split(',')
pname = get_bibrefrec_name_string(bibref)
title = ""
try:
title = get_fieldvalues(int(bibrec), "245__a")[0]
except IndexError:
title = "No title available"
if 'commit_entry' in ticket_links:
h('%(action)s - %(name)s on %(title)s '
% ({'action': a[0], 'url': CFG_SITE_URL,
'pid': str(person_id), 'bib':a[1],
'name': pname, 'title': title, 'rt': t[1]}))
else:
h('%(action)s - %(name)s on %(title)s'
% ({'action': a[0], 'name': pname, 'title': title}))
if 'del_entry' in ticket_links:
h(' - Delete this entry '
% ({'action': a[0], 'url': CFG_SITE_URL,
'pid': str(person_id), 'bib': a[1], 'rt': t[1]}))
h(' - View record
' % ({'url':CFG_SITE_URL, 'record':str(bibrec)})) h(' ')
h('
') # h(str(open_rt_tickets)) h("
")
if 'data' in show_tabs:
h(' %s:
' % r) if rt_tickets: pass # open_rt_tickets = [a for a in open_rt_tickets if a[1] == rt_tickets] for t in open_rt_tickets: name = self._('Not provided') surname = self._('Not provided') uidip = self._('Not available') comments = self._('No comments') email = self._('Not provided') date = self._('Not Available') actions = [] for info in t[0]: if info[0] == 'firstname': name = info[1] elif info[0] == 'lastname': surname = info[1] elif info[0] == 'uid-ip': uidip = info[1] elif info[0] == 'comments': comments = info[1] elif info[0] == 'email': email = info[1] elif info[0] == 'date': date = info[1] elif info[0] in ['confirm', 'repeal']: actions.append(info) if 'delete' in ticket_links: h(('Ticket number: %(tnum)s ' + self._(' Delete this ticket') + ' ') % ({'tnum':t[1], 'url':CFG_SITE_URL, 'pid':str(person_id)})) if 'commit' in ticket_links: h((' or ' + self._(' Commit this entire ticket') + '') % ({'tnum':t[1], 'url':CFG_SITE_URL, 'pid':str(person_id)})) h('
' % (surname, name)) h('Date: %s
' % date) h('identified by: %s
' % uidip) h('email: %s
' % email) h('comments: %s
' % comments) h('Suggested actions:
') h('
' % ({'url':CFG_SITE_URL, 'record':str(bibrec)})) h('
') # h(str(open_rt_tickets)) h("
')
r = verbiage_dict['data_ns']
h('' % r)
canonical_name = get_canonical_id_from_person_id(person_id)
if '.' in str(canonical_name) and not isinstance(canonical_name, int):
canonical_name = canonical_name[0:canonical_name.rindex('.')]
h('')
- h('
' + self._('... This tab is currently under construction ... ') + '') + userid = get_uid_from_personid(person_id) + h('
') + h(' Internal IDs
') + if userid: + email = get_email(int(userid)) + h('UserID: INSPIRE user %s is associated with this profile with email: %s' % (str(userid), str(email))) + else: + h('UserID: There is no INSPIRE user associated to this profile!') + h('
') + + external_ids = get_personiID_external_ids(person_id) + h('
")
h(" Canonical name setup ')
h('
Current canonical name: %s
')
h(' ' + self._('... This tab is currently under construction ... ') + '') + userid = get_uid_from_personid(person_id) + h('
') + h(' Internal IDs
') + if userid: + email = get_email(int(userid)) + h('UserID: INSPIRE user %s is associated with this profile with email: %s' % (str(userid), str(email))) + else: + h('UserID: There is no INSPIRE user associated to this profile!') + h('
') + + external_ids = get_personiID_external_ids(person_id) + h('
')
+ h(' External IDs
') + for id in external_ids: + for k in external_ids[id]: + h(' %s : %s
' % (id, k)) + h('
')
+
h(" ') + for id in external_ids: + for k in external_ids[id]: + h(' %s : %s
' % (id, k)) + h('
Search for missing papers:
')
return "\n".join(html)
def tmpl_person_menu(self):
'''
Generate the menu bar
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_person_menu_admin(self):
'''
Generate the menu bar
'''
html = []
h = html.append
h('')
return "\n".join(html)
def tmpl_ticket_final_review(self, req, mark_yours=[], mark_not_yours=[],
mark_theirs=[], mark_not_theirs=[]):
'''
Generate final review page. Displaying transactions if they
need confirmation.
@param req: Apache request object
@type req: Apache request object
@param mark_yours: papers marked as 'yours'
@type mark_yours: list
@param mark_not_yours: papers marked as 'not yours'
@type mark_not_yours: list
@param mark_theirs: papers marked as being someone else's
@type mark_theirs: list
@param mark_not_theirs: papers marked as NOT being someone else's
@type mark_not_theirs: list
'''
def html_icon_legend():
html = []
h = html.append
h('')
h("
'
% (CFG_SITE_URL, self._("Everything is shiny, captain!")))
h(self._('The result of this request will be visible immediately'))
h('
') h('') h('
'
% (CFG_SITE_URL, self._("Confirmation needed to continue")))
h(self._('The result of this request will be visible immediately but we need your confirmation to do so for this paper has been manually claimed before'))
h('
') h('') h('
'
% (CFG_SITE_URL, self._("This will create a change request for the operators")))
h(self._("The result of this request will be visible upon confirmation through an operator"))
h("")
h("
")
return "\n".join(html)
def mk_ticket_row(ticket):
recid = -1
rectitle = ""
recauthor = "No Name Found."
personname = "No Name Found."
try:
recid = ticket['bibref'].split(",")[1]
except (ValueError, KeyError, IndexError):
return ""
try:
rectitle = get_fieldvalues(int(recid), "245__a")[0]
except (ValueError, IndexError, TypeError):
rectitle = self._('Error retrieving record title')
if "authorname_rec" in ticket:
recauthor = ticket['authorname_rec']
if "person_name" in ticket:
personname = ticket['person_name']
html = []
h = html.append
# h("Debug: " + str(ticket) + "") h(self._("Symbols legend: ")) h("
") h('') h(''
% (CFG_SITE_URL, self._("Everything is shiny, captain!")))
h(self._('The result of this request will be visible immediately'))
h('') h('') h('
'
% (CFG_SITE_URL, self._("Confirmation needed to continue")))
h(self._('The result of this request will be visible immediately but we need your confirmation to do so for this paper has been manually claimed before'))
h('') h('') h('
'
% (CFG_SITE_URL, self._("This will create a change request for the operators")))
h(self._("The result of this request will be visible upon confirmation through an operator"))
h("")
h("") h('
'
% (CFG_SITE_URL, self._("Everything is shiny, captain!")))
elif ticket['status'] == "warning_granted":
h(''
% (CFG_SITE_URL, self._("Verification needed to continue")))
else:
h(''
% (CFG_SITE_URL, self._("This will create a request for the operators")))
h('')
h("
')
h('" + self._('Please provide your information') + "
") h('") h('')
h('')
if not ulevel == "guest":
h('
')
h("
")
h("" + self._('Mark as your documents') + " | ")
h('||||
| ') h(' | Nothing staged as yours | ') h("|||
" + self._("Mark as _not_ your documents") + " | ")
h('||||
| ') h(' | ' + self._('Nothing staged as not yours') + ' | ') h("|||
" + self._('Mark as their documents') + " | ")
h('||||
| ') h(' | ' + self._('Nothing staged in this category') + ' | ') h("|||
" + self._('Mark as _not_ their documents') + " | ")
h('||||
| ') h(' | ' + self._('Nothing staged in this category') + ' | ') h("|||
")
h(self._(" * You can come back to this page later. Nothing will be lost.
"))
h(self._(" ** Performs all requested changes. Changes subject to permission restrictions "
"will be submitted to an operator for manual review."))
h("
%s
' % fallback_title)
if fallback_message:
h('%s' % fallback_message)
if not results and not query:
h('')
return "\n".join(html)
h("") if query and not results: authemail = CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL h(('' + self._("We do not have a publication list for '%s'." + " Try using a less specific author name, or check" + " back in a few days as attributions are updated " + "frequently. Or you can send us feedback, at ") + "%s.") % (query, authemail, authemail)) h('') if new_person_link: if search_ticket: link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, '-3') for r in search_ticket['bibrefs']: link = link + '&selection=%s' % str(r) else: link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, '-3') h('') return "\n".join(html) # base_color = 100 # row_color = 0 for index, result in enumerate(results): # if len(results) > base_color: # row_color += 1 # else: # row_color = base_color - (base_color - index * # (base_color / len(results))) pid = result[0] names = result[1] papers = result[2] try: total_papers = result[3] if total_papers > 1: papers_string = '(%s Papers)' % str(total_papers) elif total_papers == 1: if (len(papers) == 1 and len(papers[0]) == 1 and papers[0][0] == 'Not retrieved to increase performances.'): papers_string = '' else: papers_string = '(1 Paper)' else: papers_string = '(No papers)' except IndexError: papers_string = '' h('
' % (index % 2))
h('
'
+ self._('Recent Papers') +
'')
% (pid))
else:
h("")
if search_ticket:
link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, pid)
for r in search_ticket['bibrefs']:
link = link + '&selection=%s' % str(r)
h((''
''
'' + self._('YES!') + ''
+ self._(' Attribute Papers To ') +
'%s %s ')
% (link, get_person_redirect_link(pid), papers_string))
else:
h((''
''
+ self._('Publication List ') + '(%s) %s ')
% (CFG_SITE_URL, linktarget,
get_person_redirect_link(pid),
get_person_redirect_link(pid), papers_string))
h('
')
if new_person_link:
if search_ticket:
link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, '-3')
for r in search_ticket['bibrefs']:
link = link + '&selection=%s' % str(r)
else:
link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, '-3')
h('')
return "\n".join(html)
def tmpl_welcome_start(self):
'''
Shadows the behaviour of tmpl_search_pagestart
'''
return '')
# h('%s. '
# % (row_color, row_color, row_color, index + 1))
h('%s. ' % (index + 1))
# for nindex, name in enumerate(names):
# color = row_color + nindex * 35
# color = min(color, base_color)
# h('%s; '
# % (color, color, color, name[0]))
for name in names:
h('%s '
% (name[0]))
h('
')
h('')
if index < bconfig.PERSON_SEARCH_RESULTS_SHOW_PAPERS_PERSON_LIMIT:
h((''
' '
+ self._('Recent Papers') +
'')
% (pid))
else:
h("")
if search_ticket:
link = "%s/person/action?confirm=True&pid=%s" % (CFG_SITE_URL, pid)
for r in search_ticket['bibrefs']:
link = link + '&selection=%s' % str(r)
h((''
''
'' + self._('YES!') + ''
+ self._(' Attribute Papers To ') +
'%s %s ')
% (link, get_person_redirect_link(pid), papers_string))
else:
h((''
''
+ self._('Publication List ') + '(%s) %s ')
% (CFG_SITE_URL, linktarget,
get_person_redirect_link(pid),
get_person_redirect_link(pid), papers_string))
h('' % (pid))
if papers and index < bconfig.PERSON_SEARCH_RESULTS_SHOW_PAPERS_PERSON_LIMIT:
h((self._('Showing the') + ' %d ' + self._('most recent documents:')) % len(papers))
h("
')
h('- ")
for paper in papers:
h("
- %s " % (format_record(paper[0], "ha"))) h("
" + self._('Sorry, there are no documents known for this person') + "
") elif index >= bconfig.PERSON_SEARCH_RESULTS_SHOW_PAPERS_PERSON_LIMIT: h("" + self._('Information not shown to increase performances. Please refine your search.') + "
") h(('' '' + self._('Publication List ') + '(%s) (in a new window or tab)') % (CFG_SITE_URL, linktarget, get_person_redirect_link(pid), get_person_redirect_link(pid))) h(''
def tmpl_welcome_arxiv(self):
'''
SSO landing/welcome page.
'''
html = []
h = html.append
h('
Congratulations! you have now successfully connected to INSPIRE via arXiv.org!
') h('Right now, you can verify your' ' publication records, which will help us to produce better publication lists and' ' citation statistics.' '
') h('We are currently importing your publication list from arXiv.org .' 'When we\'re done, you\'ll see a link to verify your' ' publications below; please claim the papers that are yours ' ' and remove the ones that are not. This information will be automatically processed' ' or be sent to our operator for approval if needed, usually within 24' ' hours.' '
') h('If you have ' 'any questions or encounter any problems please contact us here: ' '%s' % (CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL, CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL)) return "\n".join(html) def tmpl_welcome(self): ''' SSO landing/welcome page. ''' html = [] h = html.append h('Congratulations! you have successfully logged in!
') h('We are currently creating your publication list. When we\'re done, you\'ll see a link to correct your ' 'publications below.
') h('When the link appears we invite you to confirm the papers that are ' 'yours and to reject the ones that you are not author of. If you have ' 'any questions or encounter any problems please contact us here: ' '%s
' % (CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL, CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL)) return "\n".join(html) def tmpl_claim_stub(self, person='-1'): ''' claim stub page ''' html = [] h = html.append h('- Login through arXiv.org ' % bconfig.BIBAUTHORID_CFG_INSPIRE_LOGIN)
- h(' - Use this option if you have an arXiv account.')
+ h(' - Use this option if you have an arXiv account and have claimed your papers in arXiv.')
h('(If you login through arXiv.org, INSPIRE will immediately verify you as an author and process your claimed papers.)
') h(' - Continue as a guest ' % (CFG_SITE_URL, person))
h(' - Use this option if you DON\'T have an arXiv account, or you have not claimed any paper in arXiv.')
h('(If you login as a guest, INSPIRE will need to confirm you as an author before processing your claimed papers.)
') h('If you login through arXiv.org we can verify that you are the author of these papers and accept your claims rapidly, ' 'as well as adding additional claims from arXiv.
If you choose not to login via arXiv your changes will ' 'be publicly visible only after our editors check and confirm them, usually a few days.
' 'Either way, claims made on behalf of another author will go through our staff and may take longer to display. ' 'This applies as well to papers which have been previously claimed, by yourself or someone else.') return "\n".join(html) def tmpl_welcome_link(self): ''' Creates the link for the actual user action. ''' return '' + \ self._('Correct my publication lists!') + \ '' def tmpl_welcome_personid_association(self, pid): """ """ canon_name = get_canonical_id_from_personid(pid) head = "
" if canon_name: body = ("Your arXiv.org account is associated " "with person %s." % canon_name[0][0]) else: body = ("Warning: your arXiv.org account is associated with an empty profile. " "This can happen if it the first time you log in and you do not have any " "paper directly claimen in arXiv.org which we can use to identify you." "If this is the case, you are welcome to search and claim your papers to your" " new profile manually, or please contact us to get help.") body += ("
You are very welcome to contact us shall you need any help or explanation" " about the management of" " your profile page" " in INSPIRE and it's connections with arXiv.org: " ''' authors@inspirehep.net ''') tail = "
" return head + body + tail def tmpl_welcome_arXiv_papers(self, paps): ''' Creates the list of arXiv papers ''' plist = "
" if paps: plist = plist + "We have got and we are about to automatically claim for You the following papers from arXiv.org:
" for p in paps: plist = plist + " " + str(p) + "
" else: plist = "We have got no papers from arXiv.org which we could claim automatically for You.
" return plist def tmpl_welcome_end(self): ''' Shadows the behaviour of tmpl_search_pageend ''' return '
') for t in tickets: h(' %(longname)s - (%(cname)s - PersonID: %(pid)s): %(num)s open tickets.
' % ({'cname':str(t[1]), 'longname':str(t[0]), 'pid':str(t[2]), 'num':str(t[3])})) else: h('There are currently no open tickets.') return "\n".join(html) # pylint: enable=C0301 diff --git a/modules/bibauthorid/lib/bibauthorid_tortoise.py b/modules/bibauthorid/lib/bibauthorid_tortoise.py index f2735ddf4..39ac25a89 100644 --- a/modules/bibauthorid/lib/bibauthorid_tortoise.py +++ b/modules/bibauthorid/lib/bibauthorid_tortoise.py @@ -1,113 +1,201 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. ## Copyright (C) 2011, 2012 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -from operator import itemgetter -from itertools import groupby +import bibauthorid_config as bconfig +from datetime import datetime -from bibauthorid_cluster_set import Cluster_set -from bibauthorid_cluster_set import cluster_sets_from_marktables +from bibauthorid_cluster_set import delayed_cluster_sets_from_marktables +from bibauthorid_cluster_set import delayed_cluster_sets_from_personid from bibauthorid_wedge import wedge from bibauthorid_name_utils import generate_last_name_cluster_str -from bibauthorid_backinterface import get_all_names_from_personid -from bibauthorid_backinterface import in_results +from bibauthorid_backinterface import empty_results_table +from bibauthorid_backinterface import remove_result_cluster from bibauthorid_general_utils import bibauthor_print -from bibauthorid_scheduler import schedule +from bibauthorid_prob_matrix import prepare_matirx +from bibauthorid_scheduler import schedule \ + , Estimator \ + , matrix_coefs \ + , wedge_coefs +''' + There are three main entry points to tortoise + + i) tortoise + Performs disambiguation iteration. + The arguemnt pure indicates whether to use + the claims and the rejections or not. + Use pure=True only to test the accuracy of tortoise. + + ii) tortoise_from_scratch + NOT RECOMMENDED! + Use this function only if you have just + installed invenio and this is your first + disambiguation or if personid is broken. + + iii) tortoise_last_name + Computes the clusters for only one last name + group. Is is primary used for testing. It + may also be used to fix a broken last name + cluster. It does not involve multiprocessing + so it is convinient to debug with pdb. +''' + +# Exit codes: +# The standard ones are not well documented +# so we are using random numbers. def tortoise_from_scratch(): - cluster_sets = ((cs, sum(len(c.bibs) for c in cs.clusters)) - for cs in cluster_sets_from_marktables()) - cluster_sets = sorted(cluster_sets, key=itemgetter(1)) - args = [(x[0], ) for x in cluster_sets] - sizs = map(itemgetter(1), cluster_sets) + bibauthor_print("Preparing cluster sets.") + cluster_sets, lnames, sizes = delayed_cluster_sets_from_marktables() + bibauthor_print("Building all matrices.") + exit_statuses = schedule_create_matrix( + cluster_sets, + sizes, + force=True) + assert len(exit_statuses) == len(cluster_sets) + + empty_results_table() + + bibauthor_print("Preparing cluster sets.") + cluster_sets, lnames, sizes = delayed_cluster_sets_from_marktables() + bibauthor_print("Starting disambiguation.") + exit_statuses = schedule_wedge_and_store( + cluster_sets, + sizes) + assert len(exit_statuses) == len(cluster_sets) + + +def tortoise(pure=False, + force_matrix_creation=False, + skip_matrix_creation=False, + last_run=None): + assert not force_matrix_creation or not skip_matrix_creation + # The computation must be forced in case we want + # to compute pure results + force_matrix_creation = force_matrix_creation or pure + + if not skip_matrix_creation: + bibauthor_print("Preparing cluster sets.") + clusters, lnames, sizes = delayed_cluster_sets_from_personid(pure, last_run) + bibauthor_print("Building all matrices.") + exit_statuses = schedule_create_matrix( + clusters, + sizes, + force=force_matrix_creation) + assert len(exit_statuses) == len(clusters) + + bibauthor_print("Preparing cluster sets.") + clusters, lnames, sizes = delayed_cluster_sets_from_personid(pure, last_run) + bibauthor_print("Starting disambiguation.") + exit_statuses = schedule_wedge_and_store( + clusters, + sizes) + assert len(exit_statuses) == len(clusters) + + +def tortoise_last_name(name, from_mark=False, pure=False): + assert not(from_mark and pure) - schedule(disambiguate, args, sizs) - - -def tortoise(pure=False, only_missing=False): - names = create_lastname_list_from_personid() - names = sorted(names, key=itemgetter(2)) - args = [(x[1], x[0], pure, only_missing) for x in names] - sizs = map(itemgetter(2), names) - - schedule(disambiguate_last_name, args, sizs) - - -def tortoise_last_name(name, pure=False): lname = generate_last_name_cluster_str(name) - names = create_lastname_list_from_personid() - names = filter(lambda x: x[0] == name, names) - - if names: - pids = names[0][1] - bibauthor_print("Found %s(%s), %d pids" % (name, lname, len(pids))) - disambiguate_last_name(pids, lname, pure, False) + if from_mark: + clusters, lnames, sizes = delayed_cluster_sets_from_marktables() else: + clusters, lnames, sizes = delayed_cluster_sets_from_personid(pure) + + try: + idx = lnames.index(lname) + cluster = clusters[idx] + size = sizes[idx] + bibauthor_print("Found, %s(%s). Total number of bibs: %d." % (name, lname, size)) + cluster_set = cluster() + create_matrix(cluster_set, True) + wedge_and_store(cluster_set) + except IndexError: bibauthor_print("Sorry, %s(%s) not found in the last name clusters" % (name, lname)) -def create_lastname_list_from_personid(): - ''' - This function generates a dictionary from a last name - to list of personids which have this lastname. - ''' - # ((personid, [full Name1], Nbibs) ... ) - all_names = get_all_names_from_personid() - - # ((personid, last_name, Nbibs) ... ) - all_names = ((row[0], generate_last_name_cluster_str(iter(row[1]).next()), row[2]) - for row in all_names) - - # { (last_name, [(personid)... ], Nbibs) ... } - all_names = groupby(sorted(all_names, key=itemgetter(1)), key=itemgetter(1)) - all_names = ((key, list(data)) for key, data in all_names) - all_names = ((key, map(itemgetter(0), data), sum(x[2] for x in data)) for key, data in all_names) - - return all_names +def create_matrix(cluster_set, force): + bibs = cluster_set.num_all_bibs + expected = bibs * (bibs - 1) / 2 + bibauthor_print("Start building matrix for %s. Total number of bibs: %d, " + "maximum number of comparisons: %d" + % (cluster_set.last_name, bibs, expected)) + return prepare_matirx(cluster_set, force) -def disambiguate_last_name(personids, last_name, pure, only_missing): - ''' - Creates a cluster_set from personid and calls disambiguate. - ''' - if only_missing and in_results(last_name): - return - cs = Cluster_set() - if pure: - cs.create_pure(personids, last_name) - else: - cs.create_skeleton(personids, last_name) - disambiguate(cs) +def force_create_matrix(cluster_set, force): + bibauthor_print("Building a cluster set.") + return create_matrix(cluster_set(), force) -def disambiguate(cluster_set): - ''' - Updates personid from a list of personids, sharing common - last name, and this last name. - ''' - bibs = sum(len(c.bibs) for c in cluster_set.clusters) +def wedge_and_store(cluster_set): + bibs = cluster_set.num_all_bibs expected = bibs * (bibs - 1) / 2 bibauthor_print("Start working on %s. Total number of bibs: %d, " "maximum number of comparisons: %d" - % (cluster_set.last_name, bibs, expected)) + % (cluster_set.last_name, bibs, expected)) wedge(cluster_set) + remove_result_cluster(cluster_set.last_name) cluster_set.store() - + return True + + +def force_wedge_and_store(cluster_set): + bibauthor_print("Building a cluster set.") + return wedge_and_store(cluster_set()) + + +def schedule_create_matrix(cluster_sets, sizes, force): + def create_job(cluster): + def ret(): + return force_create_matrix(cluster, force) + return ret + + memfile_path = None + if bconfig.DEBUG_PROCESS_PEAK_MEMORY: + tt = datetime.now() + tt = (tt.hour, tt.minute, tt.day, tt.month, tt.year) + memfile_path = ('%smatrix_memory_%d:%d_%d-%d-%d.log' % + ((bconfig.TORTOISE_FILES_PATH,) + tt)) + + return schedule(map(create_job, cluster_sets), + sizes, + Estimator(matrix_coefs), + memfile_path) + + +def schedule_wedge_and_store(cluster_sets, sizes): + def create_job(cluster): + def ret(): + return force_wedge_and_store(cluster) + return ret + + memfile_path = None + if bconfig.DEBUG_PROCESS_PEAK_MEMORY: + tt = datetime.now() + tt = (tt.hour, tt.minute, tt.day, tt.month, tt.year) + memfile_path = ('%swedge_memory_%d:%d_%d-%d-%d.log' % + ((bconfig.TORTOISE_FILES_PATH,) + tt)) + + return schedule(map(create_job, cluster_sets), + sizes, + Estimator(wedge_coefs), + memfile_path) diff --git a/modules/bibauthorid/lib/bibauthorid_webapi.py b/modules/bibauthorid/lib/bibauthorid_webapi.py index 2294c3c19..bc2550a94 100644 --- a/modules/bibauthorid/lib/bibauthorid_webapi.py +++ b/modules/bibauthorid/lib/bibauthorid_webapi.py @@ -1,1393 +1,1409 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. ## Copyright (C) 2011 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. ''' Bibauthorid_webapi Point of access to the documents clustering facility. Provides utilities to safely interact with stored data. ''' import invenio.bibauthorid_config as bconfig import invenio.bibauthorid_frontinterface as dbapi import invenio.bibauthorid_name_utils as nameapi import invenio.search_engine as search_engine from search_engine import perform_request_search from cgi import escape from time import gmtime, strftime, ctime from invenio.dbquery import OperationalError from invenio.access_control_admin import acc_find_user_role_actions from invenio.webuser import collect_user_info, get_session, getUid from invenio.webuser import isUserSuperAdmin from invenio.access_control_engine import acc_authorize_action from invenio.access_control_admin import acc_get_role_id, acc_get_user_roles from invenio.external_authentication_robot import ExternalAuthRobot from invenio.external_authentication_robot import load_robot_keys from invenio.config import CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL from invenio.config import CFG_SITE_URL from invenio.mailutils import send_email +from invenio.bibauthorid_dbinterface import get_personiID_external_ids #export + def get_person_redirect_link(pid): ''' Returns the canonical name of a pid if found, the pid itself otherwise @param pid: int ''' cname = dbapi.get_canonical_id_from_personid(pid) if len(cname) > 0: return str(cname[0][0]) else: return str(pid) def update_person_canonical_name(person_id, canonical_name, userinfo=''): ''' Updates a person's canonical name @param person_id: person id @param canonical_name: string ''' + if userinfo.count('||'): + uid = userinfo.split('||')[0] + else: + uid = '' dbapi.update_personID_canonical_names((person_id,), overwrite=True, suggested=canonical_name) - dbapi.insert_user_log(userinfo, person_id, 'data_update', 'CMPUI_changecanonicalname', '', 'Canonical name manually updated.') + dbapi.insert_user_log(userinfo, person_id, 'data_update', 'CMPUI_changecanonicalname', '', 'Canonical name manually updated.', userid=uid) def get_canonical_id_from_person_id(person_id): ''' Finds the person canonical name from personid (e.g. 1) @param person_id: the canonical ID @type person_id: string @return: result from the request or person_id on failure @rtype: int ''' if not person_id or not (isinstance(person_id, str) or isinstance(person_id, int)): return person_id canonical_name = person_id try: canonical_name = dbapi.get_canonical_id_from_personid(person_id)[0][0] except IndexError: pass return canonical_name def get_person_id_from_canonical_id(canonical_id): ''' Finds the person id from a canonical name (e.g. Ellis_J_R_1) @param canonical_id: the canonical ID @type canonical_id: string @return: result from the request or -1 on failure @rtype: int ''' if not canonical_id or not isinstance(canonical_id, str): return -1 pid = -1 try: pid = dbapi.get_person_id_from_canonical_id(canonical_id)[0][0] except IndexError: pass return pid def get_bibrefs_from_bibrecs(bibreclist): ''' Retrieve all bibrefs for all the recids in the list @param bibreclist: list of record IDs @type bibreclist: list of int @return: a list of record->bibrefs @return: list of lists ''' return [[bibrec, dbapi.get_possible_bibrecref([''], bibrec, always_match=True)] for bibrec in bibreclist] def get_possible_bibrefs_from_pid_bibrec(pid, bibreclist, always_match=False, additional_names=None): ''' Returns for each bibrec a list of bibrefs for which the surname matches. @param pid: person id to gather the names strings from @param bibreclist: list of bibrecs on which to search @param always_match: match all bibrefs no matter the name @param additional_names: [n1,...,nn] names to match other then the one from personid ''' pid = wash_integer_id(pid) pid_names = dbapi.get_person_db_names_set(pid) if additional_names: pid_names += zip(additional_names) lists = [] for bibrec in bibreclist: lists.append([bibrec, dbapi.get_possible_bibrecref([n[0] for n in pid_names], bibrec, always_match)]) return lists def get_pid_from_uid(uid): ''' Return the PID associated with the uid @param uid: the internal ID of a user @type uid: int @return: the Person ID attached to the user or -1 if none found ''' if not isinstance(uid, tuple): uid = ((uid,),) return dbapi.get_personid_from_uid(uid) def get_user_level(uid): ''' Finds and returns the aid-universe-internal numeric user level @param uid: the user's id @type uid: int @return: A numerical representation of the maximum access level of a user @rtype: int ''' actions = [row[1] for row in acc_find_user_role_actions({'uid': uid})] return max([dbapi.resolve_paper_access_right(acc) for acc in actions]) def get_person_id_from_paper(bibref=None): ''' Returns the id of the person who wrote the paper @param bibref: the bibref,bibrec pair that identifies the person @type bibref: str @return: the person id @rtype: int ''' if not is_valid_bibref(bibref): return -1 person_id = -1 db_data = dbapi.get_papers_status(bibref) try: person_id = db_data[0][1] except (IndexError): pass return person_id def get_papers_by_person_id(person_id= -1, rec_status= -2, ext_out=False): ''' Returns all the papers written by the person @param person_id: identifier of the person to retrieve papers from @type person_id: int @param rec_status: minimal flag status a record must have to be displayed @type rec_status: int @param ext_out: Extended output (w/ author aff and date) @type ext_out: boolean @return: list of record ids @rtype: list of int ''' if not isinstance(person_id, int): try: person_id = int(person_id) except (ValueError, TypeError): return [] if person_id < 0: return [] if not isinstance(rec_status, int): return [] records = [] db_data = dbapi.get_person_papers(person_id, rec_status, show_author_name=True, show_title=False, show_rt_status=True, show_affiliations=ext_out, show_date=ext_out, show_experiment=ext_out) if not ext_out: records = [[row["data"].split(",")[1], row["data"], row["flag"], row["authorname"]] for row in db_data] else: for row in db_data: recid = row["data"].split(",")[1] bibref = row["data"] flag = row["flag"] authorname = row["authorname"] rt_status = row['rt_status'] authoraff = ", ".join(row['affiliation']) try: date = sorted(row['date'], key=len)[0] except IndexError: date = "Not available" exp = ", ".join(row['experiment']) #date = "" records.append([recid, bibref, flag, authorname, authoraff, date, rt_status, exp]) return records def get_papers_cluster(bibref): ''' Returns the cluster of documents connected with this one @param bibref: the table:bibref,bibrec pair to look for @type bibref: str @return: a list of record IDs @rtype: list of int ''' papers = [] person_id = get_person_id_from_paper(bibref) if person_id > -1: papers = get_papers_by_person_id(person_id) return papers def get_person_request_ticket(pid= -1, tid=None): ''' Returns the list of request tickets associated to a person. @param pid: person id @param tid: ticket id, to select if want to retrieve only a particular one @return: tickets [[],[]] ''' if pid < 0: return [] else: return dbapi.get_request_ticket(pid, ticket_id=tid) def get_persons_with_open_tickets_list(): ''' Finds all the persons with open tickets and returns pids and count of tickets @return: [[pid,ticket_count]] ''' return dbapi.get_persons_with_open_tickets_list() def get_person_names_from_id(person_id= -1): ''' Finds and returns the names associated with this person along with the frequency of occurrence (i.e. the number of papers) @param person_id: an id to find the names for @type person_id: int @return: name and number of occurrences of the name @rtype: tuple of tuple ''' # #retrieve all rows for the person if (not person_id > -1) or (not isinstance(person_id, int)): return [] return dbapi.get_person_names_count(person_id) def get_person_db_names_from_id(person_id= -1): ''' Finds and returns the names associated with this person as stored in the meta data of the underlying data set along with the frequency of occurrence (i.e. the number of papers) @param person_id: an id to find the names for @type person_id: int @return: name and number of occurrences of the name @rtype: tuple of tuple ''' # #retrieve all rows for the person if (not person_id > -1) or (not isinstance(person_id, int)): return [] return dbapi.get_person_db_names_count(person_id) def get_longest_name_from_pid(person_id= -1): ''' Finds the longest name of a person to be representative for this person. @param person_id: the person ID to look at @type person_id: int @return: returns the longest normalized name of a person @rtype: string ''' if (not person_id > -1) or (not isinstance(person_id, int)): return "This doesn't look like a person ID!" longest_name = "" for name in dbapi.get_person_names_count(person_id): if name and len(name[0]) > len(longest_name): longest_name = name[0] if longest_name: return longest_name else: return "This person does not seem to have a name!" def get_most_frequent_name_from_pid(person_id= -1, allow_none=False): ''' Finds the most frequent name of a person to be representative for this person. @param person_id: the person ID to look at @type person_id: int @return: returns the most frequent normalized name of a person @rtype: string ''' pid = wash_integer_id(person_id) if (not pid > -1) or (not isinstance(pid, int)): if allow_none: return None else: return "'%s' doesn't look like a person ID!" % person_id person_id = pid mf_name = "" try: nn = dbapi.get_person_names_count(person_id) mf_name = sorted(nn, key=lambda k:k[1], reverse=True)[0][0] except IndexError: pass if mf_name: return mf_name else: if allow_none: return None else: return "This person does not seem to have a name!" def get_paper_status(bibref): ''' Finds an returns the status of a bibrec to person assignment @param bibref: the bibref-bibrec pair that unambiguously identifies a paper @type bibref: string ''' db_data = dbapi.get_papers_status(bibref) #data,PersonID,flag status = None try: status = db_data[0][2] except IndexError: status = -10 status = wash_integer_id(status) return status def wash_integer_id(param_id): ''' Creates an int out of either int or string @param param_id: the number to be washed @type param_id: int or string @return: The int representation of the param or -1 @rtype: int ''' pid = -1 try: pid = int(param_id) except (ValueError, TypeError): return (-1) return pid def is_valid_bibref(bibref): ''' Determines if the provided string is a valid bibref-bibrec pair @param bibref: the bibref-bibrec pair that unambiguously identifies a paper @type bibref: string @return: True if it is a bibref-bibrec pair and False if it's not @rtype: boolean ''' if (not isinstance(bibref, str)) or (not bibref): return False if not bibref.count(":"): return False if not bibref.count(","): return False try: table = bibref.split(":")[0] ref = bibref.split(":")[1].split(",")[0] bibrec = bibref.split(":")[1].split(",")[1] except IndexError: return False try: table = int(table) ref = int(ref) bibrec = int(bibrec) except (ValueError, TypeError): return False return True def is_valid_canonical_id(cid): ''' Checks if presented canonical ID is valid in structure Must be of structure: ([Initial|Name]\.)*Lastname\.Number Example of valid cid: J.Ellis.1 @param cid: The canonical ID to check @type cid: string @return: Is it valid? @rtype: boolean ''' if not cid.count("."): return False xcheck = -1 sp = cid.split(".") if not (len(sp) > 1 and sp[-1]): return False try: xcheck = int(sp[-1]) except (ValueError, TypeError, IndexError): return False if xcheck and xcheck > -1: return True else: return False #def confirm_person_bibref_assignments(person_id, bibrefs, uid): # ''' # Confirms a bibref-bibrec assignment to a person. That internally # raises the flag of the entry to 2, which means 'user confirmed' and # sets the user level to the highest level of the user provided as param # # @param person_id: the id of the person to confirm the assignment to # @type person_id: int # @param bibrefs: the bibref-bibrec pairs that unambiguously identify records # @type bibrefs: list of strings # @param uid: the id of the user that arranges the confirmation # @type uid: int # # @return: True if the process ran smoothly, False if there was an error # @rtype: boolean # ''' # pid = wash_integer_id(person_id) # refs = [] # # if pid < 0: # return False # # if not isinstance(bibrefs, list) or not len(bibrefs): # return False # else: # for bibref in bibrefs: # if is_valid_bibref(bibref): # refs.append((bibref,)) # else: # return False # # try: # dbapi.confirm_papers_to_person((pid,), refs, get_user_level(uid)) # except OperationalError: # return False # # return True # # #def repeal_person_bibref_assignments(person_id, bibrefs, uid): # ''' # Repeals a bibref-bibrec assignment from a person. That internally # sets the flag of the entry to -2, which means 'user repealed' and # sets the user level to the highest level of the user provided as param # # @param person_id: the id of the person to repeal the assignment from # @type person_id: int # @param bibrefs: the bibref-bibrec pairs that unambiguously identify records # @type bibrefs: list of strings # @param uid: the id of the user that arranges the repulsion # @type uid: int # # @return: True if the process ran smoothly, False if there was an error # @rtype: boolean # ''' # pid = wash_integer_id(person_id) # refs = [] # # if pid < 0: # return False # # if not isinstance(bibrefs, list) or not len(bibrefs): # return False # else: # for bibref in bibrefs: # if is_valid_bibref(bibref): # refs.append((bibref,)) # else: # return False # # try: # dbapi.reject_papers_from_person((pid,), refs, get_user_level(uid)) # except OperationalError: # return False # # return True # # #def reset_person_bibref_decisions(person_id, bibrefs): # ''' # Resets a bibref-bibrec assignment of a person. That internally # sets the flag of the entry to 0, which means 'no user interaction' and # sets the user level to 0 to give the record free for claiming/curation # # @param person_id: the id of the person to reset the assignment from # @type person_id: int # @param bibrefs: the bibref-bibrec pairs that unambiguously identify records # @type bibrefs: list of strings # # @return: True if the process ran smoothly, False if there was an error # @rtype: boolean # ''' # pid = wash_integer_id(person_id) # refs = [] # # if pid < 0: # return False # # if not isinstance(bibrefs, list) or not len(bibrefs): # return False # else: # for bibref in bibrefs: # if is_valid_bibref(bibref): # refs.append((bibref,)) # else: # return False # # try: # dbapi.reset_papers_flag((person_id,), refs) # except OperationalError: # return False # # return True # # def add_person_comment(person_id, message): ''' Adds a comment to a person after enriching it with meta-data (date+time) @param person_id: person id to assign the comment to @type person_id: int @param message: defines the comment to set @type message: string @return the message incl. the metadata if everything was fine, False on err @rtype: string or boolean ''' msg = "" pid = -1 try: msg = str(message) pid = int(person_id) except (ValueError, TypeError): return False strtimestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime()) msg = escape(msg, quote=True) dbmsg = "%s;;;%s" % (strtimestamp, msg) dbapi.set_person_data(pid, "comment", dbmsg) return dbmsg def get_person_comments(person_id): ''' Get all comments from a person @param person_id: person id to get the comments from @type person_id: int @return the message incl. the metadata if everything was fine, False on err @rtype: string or boolean ''' pid = -1 comments = [] try: pid = int(person_id) except (ValueError, TypeError): return False for row in dbapi.get_person_data(pid, "comment"): comments.append(row[1]) return comments def search_person_ids_by_name(namequery): ''' Prepares the search to search in the database @param namequery: the search query the user enquired @type namequery: string @return: information about the result w/ probability and occurrence @rtype: tuple of tuple ''' query = "" escaped_query = "" try: query = str(namequery) except (ValueError, TypeError): return [] if query: escaped_query = escape(query, quote=True) else: return [] return dbapi.find_personIDs_by_name_string(escaped_query) def insert_log(userinfo, personid, action, tag, value, comment='', transactionid=0): ''' Log an action performed by a user Examples (in the DB): 1 2010-09-30 19:30 admin||10.0.0.1 1 assign paper 1133:4442 'from 23' 1 2010-09-30 19:30 admin||10.0.0.1 1 assign paper 8147:4442 2 2010-09-30 19:35 admin||10.0.0.1 1 reject paper 72:4442 @param userinfo: information about the user [UID|IP] @type userinfo: string @param personid: ID of the person this action is targeting @type personid: int @param action: intended action @type action: string @param tag: A tag to describe the data entered @type tag: string @param value: The value of the action described by the tag @type value: string @param comment: Optional comment to describe the transaction @type comment: string @param transactionid: May group bulk operations together @type transactionid: int @return: Returns the current transactionid @rtype: int ''' userinfo = escape(str(userinfo)) action = escape(str(action)) tag = escape(str(tag)) value = escape(str(value)) comment = escape(str(comment)) if not isinstance(personid, int): try: personid = int(personid) except (ValueError, TypeError): return -1 if not isinstance(transactionid, int): try: transactionid = int(transactionid) except (ValueError, TypeError): return -1 + if userinfo.count('||'): + uid = userinfo.split('||')[0] + else: + uid = '' + return dbapi.insert_user_log(userinfo, personid, action, tag, - value, comment, transactionid) + value, comment, transactionid, userid=uid) def user_can_modify_data(uid, pid): ''' Determines if a user may modify the data of a person @param uid: the id of a user (invenio user id) @type uid: int @param pid: the id of a person @type pid: int @return: True if the user may modify data, False if not @rtype: boolean @raise ValueError: if the supplied parameters are invalid ''' if not isinstance(uid, int): try: uid = int(uid) except (ValueError, TypeError): raise ValueError("User ID has to be a number!") if not isinstance(pid, int): try: pid = int(pid) except (ValueError, TypeError): raise ValueError("Person ID has to be a number!") return dbapi.user_can_modify_data(uid, pid) def user_can_modify_paper(uid, paper): ''' Determines if a user may modify the record assignments of a person @param uid: the id of a user (invenio user id) @type uid: int @param pid: the id of a person @type pid: int @return: True if the user may modify data, False if not @rtype: boolean @raise ValueError: if the supplied parameters are invalid ''' if not isinstance(uid, int): try: uid = int(uid) except (ValueError, TypeError): raise ValueError("User ID has to be a number!") if not paper: raise ValueError("A bibref is expected!") return dbapi.user_can_modify_paper(uid, paper) def person_bibref_is_touched_old(pid, bibref): ''' Determines if an assignment has been touched by a user (i.e. check for the flag of an assignment being 2 or -2) @param pid: the id of the person to check against @type pid: int @param bibref: the bibref-bibrec pair that unambiguously identifies a paper @type bibref: string @raise ValueError: if the supplied parameters are invalid ''' if not isinstance(pid, int): try: pid = int(pid) except (ValueError, TypeError): raise ValueError("Person ID has to be a number!") if not bibref: raise ValueError("A bibref is expected!") return dbapi.person_bibref_is_touched_old(pid, bibref) def get_review_needing_records(pid): ''' Returns list of records associated to pid which are in need of review (only bibrec ma no bibref selected) @param pid: pid ''' pid = wash_integer_id(pid) db_data = dbapi.get_person_papers_to_be_manually_reviewed(pid) return [int(row[1]) for row in db_data if row[1]] def add_review_needing_record(pid, bibrec_id): ''' Add record in need of review to a person @param pid: pid @param bibrec_id: bibrec ''' pid = wash_integer_id(pid) bibrec_id = wash_integer_id(bibrec_id) dbapi.add_person_paper_needs_manual_review(pid, bibrec_id) def del_review_needing_record(pid, bibrec_id): ''' Removes a record in need of review from a person @param pid: personid @param bibrec_id: bibrec ''' pid = wash_integer_id(pid) bibrec_id = wash_integer_id(bibrec_id) dbapi.del_person_papers_needs_manual_review(pid, bibrec_id) def get_processed_external_recids(pid): ''' Get list of records that have been processed from external identifiers @param pid: Person ID to look up the info for @type pid: int @return: list of record IDs @rtype: list of strings ''' list_str = dbapi.get_processed_external_recids(pid) return list_str.split(";") def set_processed_external_recids(pid, recid_list): ''' Set list of records that have been processed from external identifiers @param pid: Person ID to set the info for @type pid: int @param recid_list: list of recids @type recid_list: list of int ''' if isinstance(recid_list, list): recid_list_str = ";".join(recid_list) dbapi.set_processed_external_recids(pid, recid_list_str) def arxiv_login(req): ''' Log in through arxive. If user already associated to a personid, returns the personid. If user has no pid, try to guess which personid to associate based on surname and papers from arxiv. If no compatible person is found, creates a new person. At the end of the process opens a ticket for the user claiming the papers from arxiv. !!! the user will find the open ticket, which will require him to go through the final review before getting committed. @param req: Apache request object @type req: Apache request object @return: Returns the pid resulting in the process @rtype: int ''' def session_bareinit(req): session = get_session(req) try: pinfo = session["personinfo"] if 'ticket' not in pinfo: pinfo["ticket"] = [] except KeyError: pinfo = dict() session['personinfo'] = pinfo pinfo["ticket"] = [] session.save() session_bareinit(req) session = get_session(req) pinfo = session['personinfo'] ticket = session['personinfo']['ticket'] uinfo = collect_user_info(req) pinfo['external_first_entry'] = False try: name = uinfo['external_firstname'] except KeyError: name = '' try: surname = uinfo['external_familyname'] except KeyError: surname = '' if surname: session['personinfo']['arxiv_name'] = nameapi.create_normalized_name( nameapi.split_name_parts(surname + ', ' + name)) else: session['personinfo']['arxiv_name'] = '' session.save() try: arxiv_p_ids = uinfo['external_arxivids'].split(';') except KeyError: arxiv_p_ids = [] #'external_arxivids': 'hep-th/0112017;hep-th/0112020', #'external_familyname': 'Weiler', #'external_firstname': 'Henning', try: found_bibrecs = set(zip(*[perform_request_search(p='037:' + str(arx), of='id', rg=0) for arx in arxiv_p_ids])[0]) except IndexError: found_bibrecs = set() #found_bibrecs = [567700, 567744] uid = getUid(req) pid, pid_found = dbapi.get_personid_from_uid([[uid]]) if not pid_found: pid = dbapi.reclaim_personid_for_new_arXiv_user(found_bibrecs, nameapi.create_normalized_name(nameapi.split_name_parts(surname + ', ' + name)), uid) else: pid = pid[0] pid_bibrecs = set([i[0] for i in dbapi.get_all_personids_recs(pid, claimed_only=True)]) missing_bibrecs = found_bibrecs - pid_bibrecs #present_bibrecs = found_bibrecs.intersection(pid_bibrecs) #assert len(found_bibrecs) == len(missing_bibrecs) + len(present_bibrecs) tempticket = [] #now we have to open the tickets... #person_papers contains the papers which are already assigned to the person and came from arxive, #they can be claimed regardless for bibrec in missing_bibrecs: tempticket.append({'pid':pid, 'bibref':str(bibrec), 'action':'confirm'}) #check if ticket targets (bibref for pid) are already in ticket for t in list(tempticket): for e in list(ticket): if e['pid'] == t['pid'] and e['bibref'] == t['bibref']: ticket.remove(e) ticket.append(t) session.save() return pid def external_user_can_perform_action(uid): ''' Check for SSO user and if external claims will affect the decision wether or not the user may use the Invenio claiming platform @param uid: the user ID to check permissions for @type uid: int @return: is user allowed to perform actions? @rtype: boolean ''' #If no EXTERNAL_CLAIMED_RECORDS_KEY we bypass this check if not bconfig.EXTERNAL_CLAIMED_RECORDS_KEY: return True uinfo = collect_user_info(uid) keys = [] for k in bconfig.EXTERNAL_CLAIMED_RECORDS_KEY: if k in uinfo: keys.append(k) full_key = False for k in keys: if uinfo[k]: full_key = True break return full_key def is_external_user(uid): ''' Check for SSO user and if external claims will affect the decision wether or not the user may use the Invenio claiming platform @param uid: the user ID to check permissions for @type uid: int @return: is user allowed to perform actions? @rtype: boolean ''' #If no EXTERNAL_CLAIMED_RECORDS_KEY we bypass this check if not bconfig.EXTERNAL_CLAIMED_RECORDS_KEY: return False uinfo = collect_user_info(uid) keys = [] for k in bconfig.EXTERNAL_CLAIMED_RECORDS_KEY: if k in uinfo: keys.append(k) full_key = False for k in keys: if uinfo[k]: full_key = True break return full_key def check_transaction_permissions(uid, bibref, pid, action): ''' Check if the user can perform the given action on the given pid,bibrefrec pair. return in: granted, denied, warning_granted, warning_denied @param uid: The internal ID of a user @type uid: int @param bibref: the bibref pair to check permissions for @type bibref: string @param pid: the Person ID to check on @type pid: int @param action: the action that is to be performed @type action: string @return: granted, denied, warning_granted xor warning_denied @rtype: string ''' c_own = True c_override = False is_superadmin = isUserSuperAdmin({'uid': uid}) access_right = _resolve_maximum_acces_rights(uid) bibref_status = dbapi.get_bibref_modification_status(bibref) old_flag = bibref_status[0] if old_flag == 2 or old_flag == -2: if action in ['confirm', 'assign']: new_flag = 2 elif action in ['repeal']: new_flag = -2 elif action in ['reset']: new_flag = 0 if old_flag != new_flag: c_override = True uid_pid = dbapi.get_personid_from_uid([[uid]]) if not uid_pid[1] or pid != uid_pid[0][0]: c_own = False #if we cannot override an already touched bibref, no need to go on checking if c_override: if is_superadmin: return 'warning_granted' if access_right[1] < bibref_status[1]: return "warning_denied" else: if is_superadmin: return 'granted' #let's check if invenio is allowing us the action we want to perform if c_own: action = bconfig.CLAIMPAPER_CLAIM_OWN_PAPERS else: action = bconfig.CLAIMPAPER_CLAIM_OTHERS_PAPERS auth = acc_authorize_action(uid, action) if auth[0] != 0: return "denied" #now we know if claiming for ourselfs, we can ask for external ideas if c_own: action = 'claim_own_paper' else: action = 'claim_other_paper' ext_permission = external_user_can_perform_action(uid) #if we are here invenio is allowing the thing and we are not overwriting a #user with higher privileges, if externals are ok we go on! if ext_permission: if not c_override: return "granted" else: return "warning_granted" return "denied" def delete_request_ticket(pid, ticket): ''' Delete a request ticket associated to a person @param pid: pid (int) @param ticket: ticket id (int) ''' dbapi.delete_request_ticket(pid, ticket) def delete_transaction_from_request_ticket(pid, tid, action, bibref): ''' Deletes a transaction from a ticket. If ticket empty, deletes it. @param pid: pid @param tid: ticket id @param action: action @param bibref: bibref ''' rt = get_person_request_ticket(pid, tid) if len(rt) > 0: # rt_num = rt[0][1] rt = rt[0][0] else: return for t in list(rt): if str(t[0]) == str(action) and str(t[1]) == str(bibref): rt.remove(t) action_present = False for t in rt: if str(t[0]) in ['confirm', 'repeal']: action_present = True if not action_present: delete_request_ticket(pid, tid) return dbapi.update_request_ticket(pid, rt, tid) def create_request_ticket(userinfo, ticket): ''' Creates a request ticket @param usernfo: dictionary of info about user @param ticket: dictionary ticket ''' # write ticket to DB # send eMail to RT udata = [] mailcontent = [] m = mailcontent.append m("A user sent a change request through the web interface.") m("User Information:") for k, v in userinfo.iteritems(): if v: m(" %s: %s" % (k, v)) m("\nLinks to all issued Person-based requests:\n") for i in userinfo: udata.append([i, userinfo[i]]) tic = {} for t in ticket: if not t['action'] in ['confirm', 'assign', 'repeal', 'reset']: return False elif t['pid'] < 0: return False elif not is_valid_bibref(t['bibref']): return False if t['action'] == 'reset': #we ignore reset tickets continue else: if t['pid'] not in tic: tic[t['pid']] = [] if t['action'] == 'assign': t['action'] = 'confirm' tic[t['pid']].append([t['action'], t['bibref']]) for pid in tic: data = [] for i in udata: data.append(i) data.append(['date', ctime()]) for i in tic[pid]: data.append(i) dbapi.update_request_ticket(pid, data) pidlink = get_person_redirect_link(pid) m("%s/person/%s?open_claim=True#tabTickets" % (CFG_SITE_URL, pidlink)) m("\nPlease remember that you have to be logged in " "in order to see the ticket of a person.\n") if ticket and tic and mailcontent: sender = CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL if bconfig.TICKET_SENDING_FROM_USER_EMAIL and userinfo['email']: sender = userinfo['email'] send_email(sender, CFG_BIBAUTHORID_AUTHOR_TICKET_ADMIN_EMAIL, subject="[Author] Change Request", content="\n".join(mailcontent)) return True def send_user_commit_notification_email(userinfo, ticket): ''' Sends commit notification email to RT system ''' # send eMail to RT mailcontent = [] m = mailcontent.append m("A user committed a change through the web interface.") m("User Information:") for k, v in userinfo.iteritems(): if v: m(" %s: %s" % (k, v)) m("\nChanges:\n") for t in ticket: m(" ---
 |
""" % (CFG_SITE_URL) out += """
%s |
| %s |
""" out += """
| %s | %s |
| %s | %s |
| %s | %s |
| %s | %s |
| %s | %s |
""" out += """
| %s | %s |
| %s | %s |
| %s | %s |
| %s | %s |
""" out += """
%s: %s |
|
""" % (CFG_SITE_URL) out += """
%s |
| %s |
""" out += """
| %s | %s | %s | %s | %s | %s | %s |
| %s | %s | %s | %s | %s | %s | %s |
') return """
Note that you HAVE to pre-generate the correspondances journal->ISSN if you want this element to return something (Run
python bfe_issn.py -h to get help).
"""
journal_name = bfo.field(journal_name_tag)
# Here you might want to process journal name
# by doing the same operation that has been
# done when saving the mappings
journal_name = journal_name.lower().strip()
if journal_name.endswith("[online]"):
journal_name = journal_name[:-8].rstrip()
return issns.get(journal_name, '')
def build_issns_from_distant_site(url):
"""
Retrieves the ISSNs from a distant Invenio system.
Store the "journal name -> issn" relation.
Normalize journal names a little bit:
- strip whithespace chars (left and right)
- all lower case
- remove "[Online]" suffix
Print the result as Python dict structure.
@param url: the url to load issn from (in the *exact* form: http://www.mysite.com/)
"""
## Parse the results of the http request:
## http://cdsweb.cern.ch/search?cc=Periodicals&ot=022,210&of=tm&rg=9000
pattern_field = re.compile(r'''
\D*(?PContents
1. Overview2. Configuring BibUpload
3. Running BibUpload
3.1. Inserting new records
3.2. Inserting records into the Holding Pen
3.3. Updating existing records
3.4. Inserting and updating at the same time
3.5. Updating preformatted output formats
3.6. Uploading fulltext files
3.7. Obtaining feedbacks
4. Batch Uploader
4.1. Web interface - Cataloguers
4.1. Web interface - Robots
4.2. Daemon mode
1. Overview
BibUpload enables you to upload bibliographic data in MARCXML format into Invenio bibliographic database. It is also used internally by other Invenio modules as the sole entrance of metadata into the bibliographic databases.
Note that before uploading a MARCXML file, you may want to run
provided /opt/invenio/bin/xmlmarclint on it in order
to verify its correctness.
2. Configuring BibUpload
BibUpload takes a MARCXML file as its input. There is nothing to be configured for these files. If the files have to be coverted into MARCXML from some other format, structured or not, this is usually done beforehand via BibConvert module.
Note that if you are using external system numbers for your records, such as when your records are being synchronized from an external system, then BibUpload knows about the tag 970 as the one containing external system number. (To change this 970 tag into something else, you would have to edit BibUpload config source file.)
Note also that in the similar way BibUpload knows about OAI identifiers, so that it will refuse to insert the same OAI harvested record twice, for example.
3. Running BibUpload
3.1 Inserting new records
Consider that you have an MARCXML file containing new records that is to be uploaded into the Invenio. (For example, it might have been produced by BibConvert.) To finish the upload, you would call the BibUpload script in the insert mode as follows:
In the insert mode, all the records from the file will be treated as new. This means that they should not contain neither 001 tags (holding record IDs) nor 970 tags (holding external system numbers). BibUpload would refuse to upload records having these tags, in order to prevent potential double uploading. If your file does contain 001 or 970, then chances are that you want to update existing records, not re-upload them as new, and so BibUpload will warn you about this and will refuse to continue.$ bibupload -i file.xml
For example, to insert a new record, your file should look like this:
<record>
<datafield tag="100" ind1=" " ind2=" ">
<subfield code="a">Doe, John</subfield>
</datafield>
<datafield tag="245" ind1=" " ind2=" ">
<subfield code="a">On The Foo And Bar</subfield>
</datafield>
</record>
3.2 Inserting records into the Holding Pen
A special mode of BibUpload that is thigthly connected with BibEdit is the Holding Pen mode.
When you insert a record using the holding pen mode such as in the following example:
the records are not actually integrated into the database, but are instead put into an intermediate space called holding pen, where authorized curators can review them, manipulate them and eventually approve them.$ bibupload -o file.xml
The holding pen is integrated with BibEdit.
3.3 Updating existing records
When you want to update existing records, with the new content from your input MARCXML file, then your input file should contain either tags 001 (holding record IDs) or tag 970 (holding external system numbers). BibUpload will try to match existing records via 001 and 970 and if it finds a record in the database that corresponds to a record from the file, it will update its content. Otherwise it will signal an error saying that it could not find the record-to-be-updated.
For example, to update a title of record #123 via correct mode, your input file should contain record ID in the 001 tag and the title in 245 tag as follows:
<record>
<controlfield tag="001">123</controlfield>
<datafield tag="245" ind1=" " ind2=" ">
<subfield code="a">My Newly Updated Title</subfield>
</datafield>
</record>
There are several updating modes:
-r, --replace Replace existing records by those from the XML
MARC file. The original content is wiped out
and fully replaced. Signals error if record
is not found via matching record IDs or system
numbers.
Fields defined in Invenio config variable
CFG_BIBUPLOAD_STRONG_TAGS are not replaced.
Note also that `-r' can be combined with `-i'
into an `-ir' option that would automatically
either insert records as new if they are not
found in the system, or correct existing
records if they are found to exist.
-a, --append Append fields from XML MARC file at the end of
existing records. The original content is
enriched only. Signals error if record is not
found via matching record IDs or system
numbers.
-c, --correct Correct fields of existing records by those
from XML MARC file. The original record
content is modified only on those fields from
the XML MARC file where both the tags and the
indicators match: the original fields are
removed and replaced by those from the XML
MARC file. Fields not present in XML MARC
file are not changed (unlike the -r option).
Fields with "provenance" subfields defined in
'CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS'
are protected against deletion unless the
input MARCXML contains a matching
provenance value.
Signals error if record is not found via
matching record IDs or system numbers.
-d, --delete Delete fields of existing records that are
contained in the XML MARC file. The fields in
the original record that are not present in
the XML MARC file are preserved.
This is incompatible with FFT (see below).
Note that if you are using the --replace mode, and you specify
in the incoming MARCXML a 001 tag with a value representing a record ID that
does not exist, bibupload will not create the record on-the-fly unless the
--force parameter was also passed on the command line. This is done
in order to avoid creating, by mistake, holes in the database list of record
identifiers. When you ask, in fact, to --replace a non-existing record
imposing a record ID with a value of, say, 1 000 000 and, subsequently, you
--insert a new record, this will automatically receive an ID with
the value 1 000 001.
If you combine the --pretend parameter with the above updating modes you can actually test what would be executed without modifying the database or altering the system status.
3.4 Inserting and updating at the same time
Note that the insert/update modes can be combined together. For example, if you have a file that contains a mixture of new records with possibly some records to be updated, then you can run:
In this case BibUpload will try to do an update (for records having either 001 or 970 identifiers), or an insert (for the other ones).$ bibupload -i -r file.xml
3.5 Updating preformatted output formats
BibFormat can use this special upload mode during which metadata will not be updated, only the preformatted output formats for records:
-f, --format Upload only the format (FMT) fields.
The original content is not changed, and neither its modification date.
This is useful for bibreformat daemon only; human
administrators don't need to explicitly know about this mode.
3.6 Uploading fulltext files
The fulltext files can be uploaded and revised via a special FFT ("fulltext file transfer") tag with the following semantic:
FFT $a ... location of the docfile to upload (a filesystem path or a URL)
$d ... docfile description (optional)
$f ... format (optional; if not set, deduced from $a)
$m ... new desired docfile name (optional; used for renaming files)
$n ... docfile name (optional; if not set, deduced from $a)
$o ... flag (repeatable subfield)
$r ... restriction (optional, see below)
$t ... docfile type (e.g. Main, Additional)
$v ... version (used only with REVERT and DELETE-FILE, see below)
$x ... url/path for an icon (optional)
$z ... comment (optional)
For example, to upload a new fulltext file thesis.pdf
associated to record ID 123:
<record>
<controlfield tag="001">123</controlfield>
<datafield tag="FFT" ind1=" " ind2=" ">
<subfield code="a">/tmp/thesis.pdf</subfield>
<subfield code="t">Main</subfield>
<subfield code="d">
This is the fulltext version of my thesis in the PDF format.
Chapter 5 still needs some revision.
</subfield>
</datafield>
</record>
The FFT tag can be repetitive, so one can pass along another FFT tag instance containing a pointer to e.g. the thesis defence slides. The subfields of an FFT tag are non-repetitive.
When more than one FFT tag is specified for the same document (e.g. for adding more than one format at a time), if $t (docfile type), $m (new desired docfile name), $r (restriction), $v (version), $x (url/path for an icon), are specified, they should be identically specified for each single entry of FFT. E.g. if you want to specify an icon for a document with two formats (say .pdf and .doc), you'll write two FFT tags, both containing the same $x subfield.
The bibupload process, when it encounters FFT tags, will
automatically populate fulltext storage space
(/opt/invenio/var/data/files) and metadata record
associated tables (bibrec_bibdoc, bibdoc) as
appropriate. It will also enrich the 856 tags (URL tags) of the MARC
metadata of the record in question with references to the latest
versions of each file.
Note that for $a and $x subfields filesystem paths must be absolute
(e.g. /tmp/icon.gif is valid,
while Destkop/icon.gif is not) and they must be readable
by the user/group of the bibupload process that will handle the FFT.
The bibupload process supports the usual modes correct, append, replace, insert with a semantic that is somewhat similar to the semantic of the metadata upload:
Metadata Fulltext objects being uploaded MARC field instances characterized by tags (010-999) fulltext files characterized by unique file names (FFT $n) insert insert new record; must not exist insert new files; must not exist append append new tag instances for the given tag XXX, regardless of existing tag instances append new files, if filename (i.e. new format) not already present correct correct tag instances for the given tag XXX; delete existing ones and replace with given ones correct files with the given filename; add new revision or delete file; if the docname does not exist the file is added replace replace all tags, whatever XXX are replace all files, whatever filenames are delete delete all existing tag instances not supported
Note, in append and insert mode,
$mis ignored.
In order to rename a document just use the the correct mode specifing in the $n subfield the original docname that should be renamed and in $m the new name.
Special values can be assigned to the $t subfield.
| Value | Meaning |
|---|---|
| PURGE | In order to purge previous file revisions (i.e. in order to keep only the latest file version), please use the correct mode with $n docname and $t PURGE as the special keyword. |
| DELETE | In order to delete all existing versions of a file, making it effectively hidden, please use the correct mode with $n docname and $t DELETE as the special keyword. |
| EXPUNGE | In order to expunge (i.e. remove completely, also from the filesystem) all existing versions of a file, making it effectively disappear, please use the correct mode with $n docname and $t EXPUNGE as the special keyword. |
| FIX-MARC | In order to synchronize MARC to the bibrec/bibdoc structure (e.g. after an update or a tweak in the database), please use the correct mode with $n docname and $t FIX-MARC as the special keyword. |
| FIX-ALL | In order to fix a record (i.e. put all its linked documents in a coherent state) and synchronize the MARC to the table, please use the correct mode with $n docname and $t FIX-ALL as the special keyword. |
| REVERT | In order to revert to a previous file revision (i.e. to create a new revision with the same content as some previous revision had), please use the correct mode with $n docname, $t REVERT as the special keyword and $v the number corresponding to the desired version. |
| DELETE-FILE | In order to delete a particular file added by mistake, please use the correct mode with $n docname, $t DELETE-FILE, specifing $v version and $f format. Note that this operation is not reversible. Note that if you don't spcify a version, the last version will be used. |
In order to preserve previous comments and descriptions when correcting, please use the KEEP-OLD-VALUE special keyword with the desired $d and $z subfield.
The $r subfield can contain a string that can be use to restrict the given document. The same value must be specified for all the format of a given document. By default the keyword will be used as the status parameter for the "viewrestrdoc" action, which can be used to give access right/restriction to desired user. e.g. if you set the keyword "thesis", you can the connect the "thesisviewer" to the action "viewrestrdoc" with parameter "status" set to "thesis". Then all the user which are linked with the "thesisviewer" role will be able to download the document. Instead any other user which are not considered as authors for the given record will not be allowed. Note, if you use the keyword "KEEP-OLD-VALUE" the previous restrictions if applicable will be kept.
More advanced document-level restriction is indeed possible. If the value contains infact:
- email: john.doe@example.org: then only the user having john.doe@example.org as email address will be authorized to access the given document.
- group: example: then only users belonging to the local/external group example will be authorized to access the given document.
- role: example: then only the users belonging to the WebAccess role example will be authorized to access the given document.
- firerole: allow .../deny...: then only the users implicitly matched by the given firewall like role definition will be authorized to access the given document.
- status: example: then only the users belonging to roles having an authorization for the WebAccess action viewrestrdoc with parameter status set to example will be authorized (that is exactly like setting $r to example).
Some special flags might be set via FFT and associated with the current document by using the $o subfield. This feature is experimental. Currently only two flags are actively considered:
- HIDDEN: used to specify that the file that is currently added (via revision or append) must be hidden, i.e. must not be visible to the world but only known by the system (e.g. to allow for fulltext indexing). This flag is permanently associated with the specific revision and format of the file being added.
- PERFORM_HIDE_PREVIOUS: used to specify that, although the current file should be visible (unless the HIDDEN flag is also specified), any other previous revision of the document should receive the HIDDEN flag, and should thus be hidden to the world.
Note that each time bibupload is called on a record, the 8564 tags pointing to locally stored files are recreated on the basis of the full-text files connected to the record. Thus, if you whish to update some 8564 tag pointing to a locally managed file, the only way to perform this is through the FFT tag, not by editing 8564 directly.
3.7 Obtaining feedbacks
Sometimes, to implement a particular workflow or policy in a digital repository, it might be nice to receive an automatic machine friendly feedback that aknowledges the outcome of a bibupload execution. To this aim the --callback-url command line parameter can be used. This parameter expects a URL to be specified to which a JSON-serialized response will POSTed.
Say, you have an external service reachable via the URL http://www.example.org/accept_feedback. If the argument:
--callback-url http://www.example.org/accept_feedbackis added to the usual bibupload call, at the end of the execution of the corresponding bibupload task, an HTTP POST request will be performed, if possible to the given URL, reporting the outcome of the bibupload execution as a JSON-serialized response with the following structure:
- a JSON object with the following string -- value mapping:
- string: results -- value: a JSON array whose values
are all JSON objects with the following string -- value mapping:
- recid: an integer number, representing the described record identifier (-1 if no record identifier can be retrieved)
- success: either true or false depending on the success of the elaboration of the corresponding MARCXML
- error_message: a string containing a human-friendly description of the error that caused the MARCXML elaboration to fail (in case success was having false value)
- marcxml: in case of success, this contains the final MARCXML representation of the record
- url: in case of success, this contains the final URL where the detailde representation of the record can be fetched (i.e. its canonical URL)
- string: results -- value: a JSON array whose values
are all JSON objects with the following string -- value mapping:
For example, a possible JSON response posted to a specified URL can look like:
{
"results": [
{
"recid": -1,
"error_message": "ERROR: can not retrieve the record identifier",
"success": false
},
{
"recid": 1000,
"error_message": "",
"success": true,
"marcxml": "1000 ...
Note that, currently, in case the specified URL can not be reached at the time of the POST request, the whole bibupload task will fail.
+If you use the same callback URL to receive the feedback from more than one bibupload request +you might want to be able to correctly identify each bibupload call with the corresponding +feedback. For this reason you can pass to the bibupload call an additional argument: +
+--nonce VALUE ++where value can be any string you wish. Such string will be then added to the JSON structure, as in +(supposing you specified --nonce 1234): +
+{
+ "nonce": "1234",
+ "results": [
+ {
+ "recid": -1,
+ "error_message": "ERROR: can not retrieve the record identifier",
+ "success": false
+ },
+ {
+ "recid": 1000,
+ "error_message": "",
+ "success": true,
+ "marcxml": "1000 ...
+
4. Batch Uploader
4.1 Web interface - Cataloguers
The batchuploader web interface can be used either to upload metadata files or documents. Opposed to daemon mode, actions will be executed only once.
The available upload history displays metadata and document uploads using the web interface, not daemon mode.
4.2 Web interface - Robots
If it is needed to use the batch upload function from within command line, this can be achieved with a curl call, like:
$ curl -F 'file=@localfile.xml' -F 'mode=-i' http://cdsweb.cern.ch/batchuploader/robotupload [-F 'callback_url=http://...'] -A invenio_webupload
This service provides (client, file) checking to assure the records are put into a collection the client has rights to.
To configure this permissions, check CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS variable in the configuration file.
The allowed user agents can also be defined using the CFG_BATCHUPLOADER_WEB_ROBOT_AGENT variable.
Note that you can receive machine-friendly feedbacks from the corresponding bibupload task that is launched by a given batchuploader request, by adding the optional POST field callback_url with the same semantic of the --callback-url command line parameter of bibupload (see the previous paragraph Obtaining feedbacks).
+A second more RESTful interface is also available: it will suffice to append to the URL the specific mode (among "insert", +"append", "correct", "delete", "replace"), as in: +
+http://cdsweb.cern.ch/batchuploader/robotupload/insert ++ +
The callback_url argument can be put in query part of the URL as in: +
+http://cdsweb.cern.ch/batchuploader/robotupload/insert?callback_url=http://myhandler ++
+ +
The MARCXML content should then be specified as the body of the request. With curl this can be implemented as in: +
+$ curl -T localfile.xml http://cdsweb.cern.ch/batchuploader/robotupload/insert?callback_url=http://... -A invenio_webupload -H "Content-Type: application/marcxml+xml" ++ +
The nonce argument that can be passed to BibUpload as described in the previous paragraph can also be specified with both robotupload interfaces. E.g.: +
+$ curl -F 'file=@localfile.xml' -F 'nonce=1234' -F 'mode=-i' http://cdsweb.cern.ch/batchuploader/robotupload -F 'callback_url=http://...' -A invenio_webupload ++and +
+$ curl -T localfile.xml http://cdsweb.cern.ch/batchuploader/robotupload/insert?nonce=1234&callback_url=http://... -A invenio_webupload -H "Content-Type: application/marcxml+xml" ++ +
4.2 Daemon mode
The batchuploader daemon mode is intended to be a bibsched task for document or metadata upload. The parent directory where the daemon will look for folders metadata and documents must be specified in the invenio configuration file.
An example of how directories should be arranged, considering that invenio was installed in folder /opt/invenio would be:
/opt/invenio/var/batchupload
/opt/invenio/var/batchupload/documents
/opt/invenio/var/batchupload/documents/append
/opt/invenio/var/batchupload/documents/revise
/opt/invenio/var/batchupload/metadata
/opt/invenio/var/batchupload/metadata/append
/opt/invenio/var/batchupload/metadata/correct
/opt/invenio/var/batchupload/metadata/insert
/opt/invenio/var/batchupload/metadata/replace
When running the batchuploader daemon there are two possible execution modes:
-m, --metadata Look for metadata files in folders insert, append, correct and replace.
All files are uploaded and then moved to the corresponding DONE folder.
-d, --documents Look for documents in folders append and revise. Uploaded files are then
moved to DONE folders if possible.
By default, metadata mode is used.
An example of invocation would be:
It is possible to program batch uploader to run periodically. Read the Howto-run guide to see how. diff --git a/modules/bibupload/lib/Makefile.am b/modules/bibupload/lib/Makefile.am index 46118e4f1..d46d299a0 100644 --- a/modules/bibupload/lib/Makefile.am +++ b/modules/bibupload/lib/Makefile.am @@ -1,30 +1,31 @@ ## This file is part of Invenio. ## Copyright (C) 2006, 2007, 2008, 2010, 2011 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. pylibdir = $(libdir)/python/invenio pylib_DATA = bibupload_config.py \ bibupload.py \ bibupload_regression_tests.py \ batchuploader_webinterface.py \ batchuploader_engine.py \ batchuploader_templates.py \ - batchuploader.py + batchuploader.py \ + batchuploader_regression_tests.py EXTRA_DIST = $(pylib_DATA) CLEANFILES = *~ *.tmp *.pyc diff --git a/modules/bibupload/lib/batchuploader_engine.py b/modules/bibupload/lib/batchuploader_engine.py index cdbc85d9a..29eb119e0 100644 --- a/modules/bibupload/lib/batchuploader_engine.py +++ b/modules/bibupload/lib/batchuploader_engine.py @@ -1,695 +1,711 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. ## Copyright (C) 2010, 2011 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Batch Uploader core functions. Uploading metadata and documents. """ import os import pwd import grp import sys import time import tempfile import cgi import re import calendar from invenio.dbquery import run_sql, Error from invenio.access_control_engine import acc_authorize_action from invenio.webuser import collect_user_info, page_not_authorized from invenio.config import CFG_BINDIR, CFG_TMPSHAREDDIR, CFG_LOGDIR, \ CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG, \ CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG, \ CFG_OAI_ID_FIELD, CFG_BATCHUPLOADER_DAEMON_DIR, \ CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS, \ - CFG_BATCHUPLOADER_WEB_ROBOT_AGENT, \ + CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS, \ CFG_PREFIX, CFG_SITE_LANG from invenio.textutils import encode_for_xml from invenio.bibtask import task_low_level_submission from invenio.messages import gettext_set_language from invenio.textmarc2xmlmarc import transform_file from invenio.shellutils import run_shell_command from invenio.bibupload import xml_marc_to_records, bibupload import invenio.bibupload as bibupload_module from invenio.bibrecord import create_records, \ record_strip_empty_volatile_subfields, \ record_strip_empty_fields try: from cStringIO import StringIO except ImportError: from StringIO import StringIO PERMITTED_MODES = ['-i', '-r', '-c', '-a', '-ir', '--insert', '--replace', '--correct', '--append'] +_CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS_RE = re.compile(CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS) + def cli_allocate_record(req): req.content_type = "text/plain" req.send_http_header() # check IP and useragent: if not _check_client_ip(req): msg = "[ERROR] Sorry, client IP %s cannot use the service." % _get_client_ip(req) _log(msg) return _write(req, msg) if not _check_client_useragent(req): msg = "[ERROR] Sorry, this useragent cannot use the service." _log(msg) return _write(req, msg) recid = run_sql("insert into bibrec (creation_date,modification_date) values(NOW(),NOW())") return recid -def cli_upload(req, file_content=None, mode=None, callback_url=None): +def cli_upload(req, file_content=None, mode=None, callback_url=None, nonce=None): """ Robot interface for uploading MARC files """ req.content_type = "text/plain" req.send_http_header() # check IP and useragent: if not _check_client_ip(req): msg = "[ERROR] Sorry, client IP %s cannot use the service." % _get_client_ip(req) _log(msg) return _write(req, msg) if not _check_client_useragent(req): - msg = "[ERROR] Sorry, this useragent cannot use the service." + msg = "[ERROR] Sorry, the %s useragent cannot use the service." % _get_useragent(req) _log(msg) return _write(req, msg) - arg_file = file_content arg_mode = mode - if not arg_file: - msg = "[ERROR] Please specify file body to input." - _log(msg) - return _write(req, msg) if not arg_mode: msg = "[ERROR] Please specify upload mode to use." _log(msg) return _write(req, msg) if not arg_mode in PERMITTED_MODES: msg = "[ERROR] Invalid upload mode." _log(msg) return _write(req, msg) - if hasattr(arg_file, "filename"): - arg_file = arg_file.value + + arg_file = file_content + if hasattr(arg_file, 'read'): + ## We've been passed a readable file, e.g. req + arg_file = arg_file.read() + if not arg_file: + msg = "[ERROR] Please provide a body to your request." + _log(msg) + return _write(req, msg) else: - msg = "[ERROR] 'file' parameter must be a (single) file" - _log(msg) - return _write(req, msg) + if not arg_file: + msg = "[ERROR] Please specify file body to input." + _log(msg) + return _write(req, msg) + if hasattr(arg_file, "filename"): + arg_file = arg_file.value + else: + msg = "[ERROR] 'file' parameter must be a (single) file" + _log(msg) + return _write(req, msg) # write temporary file: tempfile.tempdir = CFG_TMPSHAREDDIR filename = tempfile.mktemp(prefix="batchupload_" + \ time.strftime("%Y%m%d%H%M%S", time.localtime()) + "_") filedesc = open(filename, 'w') filedesc.write(arg_file) filedesc.close() # check if this client can run this file: client_ip = _get_client_ip(req) permitted_dbcollids = CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS[client_ip] if permitted_dbcollids != ['*']: # wildcard allow = _check_client_can_submit_file(client_ip, filename, req, 0) if not allow: msg = "[ERROR] Cannot submit such a file from this IP. (Wrong collection.)" _log(msg) return _write(req, msg) # check validity of marcxml xmlmarclint_path = CFG_BINDIR + '/xmlmarclint' xmlmarclint_output, dummy1, dummy2 = run_shell_command('%s %s' % (xmlmarclint_path, filename)) if xmlmarclint_output != 0: msg = "[ERROR] MARCXML is not valid." _log(msg) return _write(req, msg) + args = ['bibupload', "batchupload", arg_mode, filename] # run upload command if callback_url: - task_low_level_submission('bibupload', "batchupload", arg_mode, filename, "--callback-url", callback_url) - msg = "[INFO] %s %s %s %s %s" % ('bibupload', arg_mode, filename, "--callback-url", callback_url) - else: - task_low_level_submission('bibupload', "batchupload", arg_mode, filename) - msg = "[INFO] %s %s %s" % ('bibupload', arg_mode, filename) + args += ["--callback-url", callback_url] + if nonce: + args += ["--nonce", nonce] + task_low_level_submission(*args) + msg = "[INFO] %s" % ' '.join(args) _log(msg) return _write(req, msg) def metadata_upload(req, metafile=None, filetype=None, mode=None, exec_date=None, exec_time=None, metafilename=None, ln=CFG_SITE_LANG, priority="1"): """ Metadata web upload service. Get upload parameters and exec bibupload for the given file. Finally, write upload history. @return: tuple (error code, message) error code: code that indicates if an error ocurred message: message describing the error """ # start output: req.content_type = "text/html" req.send_http_header() error_codes = {'not_authorized': 1, 'invalid_marc': 2} # write temporary file: if filetype == 'marcxml': metafile = metafile.value else: metafile = _transform_input_to_marcxml(file_input=metafile.value) user_info = collect_user_info(req) tempfile.tempdir = CFG_TMPSHAREDDIR filename = tempfile.mktemp(prefix="batchupload_" + \ user_info['nickname'] + "_" + time.strftime("%Y%m%d%H%M%S", time.localtime()) + "_") filedesc = open(filename, 'w') filedesc.write(metafile) filedesc.close() # check if this client can run this file: if req is not None: allow = _check_client_can_submit_file(req=req, metafile=metafile, webupload=1, ln=ln) if allow[0] != 0: return (error_codes['not_authorized'], allow[1]) # check MARCXML validity if filetype == 'marcxml': # check validity of marcxml xmlmarclint_path = CFG_BINDIR + '/xmlmarclint' xmlmarclint_output, dummy1, dummy2 = run_shell_command('%s %s' % (xmlmarclint_path, filename)) if xmlmarclint_output != 0: msg = "[ERROR] MARCXML is not valid." return (error_codes['invalid_marc'], msg) # run upload command: if exec_date: date = exec_date if exec_time: date += ' ' + exec_time jobid = task_low_level_submission('bibupload', user_info['nickname'], mode, "--name=" + metafilename, "--priority=" + priority, "-t", date, filename) else: jobid = task_low_level_submission('bibupload', user_info['nickname'], mode, "--name=" + metafilename, "--priority=" + priority, filename) # write batch upload history run_sql("""INSERT INTO hstBATCHUPLOAD (user, submitdate, filename, execdate, id_schTASK, batch_mode) VALUES (%s, NOW(), %s, %s, %s, "metadata")""", (user_info['nickname'], metafilename, exec_date != "" and (exec_date + ' ' + exec_time) or time.strftime("%Y-%m-%d %H:%M:%S"), str(jobid), )) return (0, "Task %s queued" % str(jobid)) def document_upload(req=None, folder="", matching="", mode="", exec_date="", exec_time="", ln=CFG_SITE_LANG, priority="1"): """ Take files from the given directory and upload them with the appropiate mode. @parameters: + folder: Folder where the files to upload are stored + matching: How to match file names with record fields (report number, barcode,...) + mode: Upload mode (append, revise, replace) @return: tuple (file, error code) file: file name causing the error to notify the user error code: 1 - More than one possible recID, ambiguous behaviour 2 - No records match that file name 3 - File already exists """ import sys if sys.hexversion < 0x2060000: from md5 import md5 else: from hashlib import md5 from invenio.bibdocfile import BibRecDocs, file_strip_ext import shutil from invenio.search_engine import perform_request_search, \ search_pattern, \ guess_collection_of_a_record _ = gettext_set_language(ln) errors = [] info = [0, []] # Number of files read, name of the files try: files = os.listdir(folder) except OSError, error: errors.append(("", error)) return errors, info err_desc = {1: _("More than one possible recID, ambiguous behaviour"), 2: _("No records match that file name"), 3: _("File already exists"), 4: _("A file with the same name and format already exists"), 5: _("No rights to upload to collection '%s'")} # Create directory DONE/ if doesn't exist folder = (folder[-1] == "/") and folder or (folder + "/") files_done_dir = folder + "DONE/" try: os.mkdir(files_done_dir) except OSError: # Directory exists or no write permission pass for docfile in files: if os.path.isfile(os.path.join(folder, docfile)): info[0] += 1 identifier = file_strip_ext(docfile) extension = docfile[len(identifier):] rec_id = None if identifier: rec_id = search_pattern(p=identifier, f=matching, m='e') if not rec_id: errors.append((docfile, err_desc[2])) continue elif len(rec_id) > 1: errors.append((docfile, err_desc[1])) continue else: rec_id = str(list(rec_id)[0]) rec_info = BibRecDocs(rec_id) if rec_info.bibdocs: for bibdoc in rec_info.bibdocs: attached_files = bibdoc.list_all_files() file_md5 = md5(open(os.path.join(folder, docfile), "rb").read()).hexdigest() num_errors = len(errors) for attached_file in attached_files: if attached_file.checksum == file_md5: errors.append((docfile, err_desc[3])) break elif attached_file.fullname == docfile: errors.append((docfile, err_desc[4])) break if len(errors) > num_errors: continue # Check if user has rights to upload file if req is not None: file_collection = guess_collection_of_a_record(int(rec_id)) auth_code, auth_message = acc_authorize_action(req, 'runbatchuploader', collection=file_collection) if auth_code != 0: error_msg = err_desc[5] % file_collection errors.append((docfile, error_msg)) continue tempfile.tempdir = CFG_TMPSHAREDDIR # Move document to be uploaded to temporary folder tmp_file = tempfile.mktemp(prefix=identifier + "_" + time.strftime("%Y%m%d%H%M%S", time.localtime()) + "_", suffix=extension) shutil.copy(os.path.join(folder, docfile), tmp_file) # Create MARC temporary file with FFT tag and call bibupload filename = tempfile.mktemp(prefix=identifier + '_') filedesc = open(filename, 'w') marc_content = """$ batchuploader --documents
)?[0-9]{9}\s001__\s\d+($|\n)', '', hmbuffer)
def compare_hmbuffers(hmbuffer1, hmbuffer2):
"""Compare two HM (HTML MARC) buffers by removing whitespaces
before testing.
"""
hmbuffer1 = hmbuffer1.strip()
hmbuffer2 = hmbuffer2.strip()
# remove eventual ...
formatting:
hmbuffer1 = re.sub(r'^', '', hmbuffer1)
hmbuffer2 = re.sub(r'^', '', hmbuffer2)
hmbuffer1 = re.sub(r'$', '', hmbuffer1)
hmbuffer2 = re.sub(r'$', '', hmbuffer2)
# remove 005 revision numbers:
hmbuffer1 = re.sub(r'(^|\n)[0-9]{9}\s005.*($|\n)', '\n', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)[0-9]{9}\s005.*($|\n)', '\n', hmbuffer2)
hmbuffer1 = hmbuffer1.strip()
hmbuffer2 = hmbuffer2.strip()
# remove leading recid, leaving only field values:
hmbuffer1 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)[0-9]{9}\s', '', hmbuffer2)
# remove leading whitespace:
hmbuffer1 = re.sub(r'(^|\n)\s+', '', hmbuffer1)
hmbuffer2 = re.sub(r'(^|\n)\s+', '', hmbuffer2)
compare_hmbuffers = hmbuffer1 == hmbuffer2
if not compare_hmbuffers:
return "\n=" + hmbuffer1 + "=\n" + '!=' + "\n=" + hmbuffer2 + "=\n"
return ''
def wipe_out_record_from_all_tables(recid):
"""
Wipe out completely the record and all its traces of RECID from
the database (bibrec, bibrec_bibxxx, bibxxx, bibfmt). Useful for
the time being for test cases.
"""
# delete all the linked bibdocs
for bibdoc in BibRecDocs(recid).list_bibdocs():
bibdoc.expunge()
# delete from bibrec:
run_sql("DELETE FROM bibrec WHERE id=%s", (recid,))
# delete from bibrec_bibxxx:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE FROM %(bibrec_bibxxx)s WHERE id_bibrec=%%s" % \
{'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)},
(recid,))
# delete all unused bibxxx values:
for i in range(0, 10):
for j in range(0, 10):
run_sql("DELETE %(bibxxx)s FROM %(bibxxx)s " \
" LEFT JOIN %(bibrec_bibxxx)s " \
" ON %(bibxxx)s.id=%(bibrec_bibxxx)s.id_bibxxx " \
" WHERE %(bibrec_bibxxx)s.id_bibrec IS NULL" % \
{'bibxxx': "bib%i%ix" % (i, j),
'bibrec_bibxxx': "bibrec_bib%i%ix" % (i, j)})
# delete from bibfmt:
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s", (recid,))
# delete from bibrec_bibdoc:
run_sql("DELETE FROM bibrec_bibdoc WHERE id_bibrec=%s", (recid,))
def try_url_download(url):
"""Try to download a given URL"""
try:
open_url = urlopen(url)
open_url.read()
except Exception, e:
raise StandardError("Downloading %s is impossible because of %s"
% (url, str(e)))
return True
def force_webcoll(recid):
from invenio import bibindex_engine
reload(bibindex_engine)
from invenio import websearch_webcoll
reload(websearch_webcoll)
index_id, index_name, index_tags = bibindex_engine.get_word_tables("collection")[0]
bibindex_engine.WordTable(index_name, index_id, index_tags, "idxWORD%02dF", default_get_words_fnc=bibindex_engine.get_words_from_phrase, tag_to_words_fnc_map={'8564_u': bibindex_engine.get_words_from_fulltext}).add_recIDs([[recid, recid]], 1)
c = websearch_webcoll.Collection()
c.calculate_reclist()
c.update_reclist()
class GenericBibUploadTest(unittest.TestCase):
"""Generic BibUpload testing class with predefined
setUp and tearDown methods.
"""
def setUp(self):
self.verbose = 0
setup_loggers()
task_set_task_param('verbose', self.verbose)
self.last_recid = run_sql("SELECT MAX(id) FROM bibrec")[0][0]
def tearDown(self):
for recid in run_sql("SELECT id FROM bibrec WHERE id>%s", (self.last_recid,)):
wipe_out_record_from_all_tables(recid[0])
class BibUploadCallbackURLTest(GenericBibUploadTest):
"""Testing usage of CLI callback_url"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.test = """
something
Tester, J Y
MIT
Tester, K J
CERN2
Tester, G
CERN3
test11
test31
test12
test32
test13
test33
test21
test41
test22
test42
test14
test51
test52
Tester, T
CERN
"""
self.testfile_path = os.path.join(CFG_TMPDIR, 'bibupload_regression_test_input.xml')
open(self.testfile_path, "w").write(self.test)
self.resultfile_path = os.path.join(CFG_TMPDIR, 'bibupload_regression_test_result.json')
if CFG_DEVEL_SITE:
def test_simple_insert_callback_url(self):
"""bibupload - --callback-url with simple insert"""
taskid = task_low_level_submission('bibupload', 'test', '-i', self.testfile_path, '--callback-url', CFG_SITE_URL + '/httptest/post2?%s' % urlencode({"save": self.resultfile_path}), '-v0')
- os.system(CFG_BINDIR + '/bibupload' + " %s" % taskid)
+ run_shell_command(CFG_BINDIR + '/bibupload %s', [taskid])
results = json.loads(open(self.resultfile_path).read())
self.failUnless('results' in results)
self.assertEqual(len(results['results']), 1)
self.failUnless(results['results'][0]['success'])
self.failUnless(results['results'][0]['recid'] > 0)
self.failUnless("""Tester, J Y """ in results['results'][0]['marcxml'], results['results'][0]['marcxml'])
class BibUploadInsertModeTest(GenericBibUploadTest):
"""Testing insert mode."""
def setUp(self):
# pylint: disable=C0103
"""Initialise the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test = """
something
Tester, J Y
MIT
Tester, K J
CERN2
Tester, G
CERN3
test11
test31
test12
test32
test13
test33
test21
test41
test22
test42
test14
test51
test52
Tester, T
CERN
"""
self.test_hm = """
100__ $$aTester, T$$uCERN
111__ $$atest11$$ctest31
111__ $$atest12$$ctest32
111__ $$atest13$$ctest33
111__ $$btest21$$dtest41
111__ $$btest22$$dtest42
111__ $$atest14
111__ $$etest51
111__ $$etest52
245__ $$asomething
700__ $$aTester, J Y$$uMIT
700__ $$aTester, K J$$uCERN2
700__ $$aTester, G$$uCERN3
"""
def test_create_record_id(self):
"""bibupload - insert mode, trying to create a new record ID in the database"""
rec_id = bibupload.create_new_record()
self.assertNotEqual(None, rec_id)
def test_create_specific_record_id(self):
"""bibupload - insert mode, trying to create a new specifc record ID in the database"""
expected_rec_id = run_sql("SELECT MAX(id) FROM bibrec")[0][0] + 1
rec_id = bibupload.create_new_record(expected_rec_id)
self.assertEqual(rec_id, expected_rec_id)
def test_no_retrieve_record_id(self):
"""bibupload - insert mode, detection of record ID in the input file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'insert')
# We compare the value found with None
self.assertEqual(None, rec_id)
def test_insert_complete_xmlmarc(self):
"""bibupload - insert mode, trying to insert complete MARCXML file"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test)
# We call the main function with the record as a parameter
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# We retrieve the inserted xml
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.test), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.test_hm), '')
def test_retrieve_005_tag(self):
"""bibupload - insert mode, verifying insertion of 005 control field for record """
# Convert marc xml into record structure
recs = bibupload.xml_marc_to_records(self.test)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# Retrive the inserted record based on the record id
rec = get_record(recid)
# We retrieve the creationdate date from the database
query = """SELECT DATE_FORMAT(creation_date,'%%Y%%m%%d%%H%%i%%s') FROM bibrec where id = %s"""
res = run_sql(query % recid)
self.assertEqual(record_has_field(rec,'005'),True)
self.assertEqual(str(res[0][0])+'.0',record_get_field_value(rec,'005','',''))
class BibUploadAppendModeTest(GenericBibUploadTest):
"""Testing append mode."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_existing = """
123456789
Tester, T
DESY
0003719PHOPHO
"""
self.test_to_append = """
123456789
Tester, U
CERN
0003719PHOPHO
"""
self.test_expected_xm = """
123456789
Tester, T
DESY
Tester, U
CERN
0003719PHOPHO
"""
self.test_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uDESY
100__ $$aTester, U$$uCERN
970__ $$a0003719PHOPHO
"""
# insert test record:
test_to_upload = self.test_existing.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
self.test_recid = recid
# replace test buffers with real recid of inserted test record:
self.test_existing = self.test_existing.replace('123456789',
str(self.test_recid))
self.test_to_append = self.test_to_append.replace('123456789',
str(self.test_recid))
self.test_expected_xm = self.test_expected_xm.replace('123456789',
str(self.test_recid))
self.test_expected_hm = self.test_expected_hm.replace('123456789',
str(self.test_recid))
def test_retrieve_record_id(self):
"""bibupload - append mode, the input file should contain a record ID"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], 'append')
# We compare the value found with None
self.assertEqual(self.test_recid, rec_id)
# clean up after ourselves:
def test_update_modification_record_date(self):
"""bibupload - append mode, checking the update of the modification date"""
# Initialize the global variable
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_existing)
# We call the function which should retrieve the record id
rec_id = bibupload.retrieve_rec_id(recs[0], opt_mode='append')
# Retrieve current localtime
now = time.localtime()
# We update the modification date
bibupload.update_bibrec_modif_date(convert_datestruct_to_datetext(now), rec_id)
# We retrieve the modification date from the database
query = """SELECT DATE_FORMAT(modification_date,'%%Y-%%m-%%d %%H:%%i:%%s') FROM bibrec where id = %s"""
res = run_sql(query % rec_id)
# We compare the two results
self.assertEqual(res[0][0], convert_datestruct_to_datetext(now))
# clean up after ourselves:
def test_append_complete_xml_marc(self):
"""bibupload - append mode, appending complete MARCXML file"""
# Now we append a datafield
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_to_append)
# We call the main function with the record as a parameter
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
# We retrieve the inserted xm
after_append_xm = print_record(recid, 'xm')
after_append_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(after_append_xm, self.test_expected_xm), '')
self.assertEqual(compare_hmbuffers(after_append_hm, self.test_expected_hm), '')
# clean up after ourselves:
def test_retrieve_updated_005_tag(self):
"""bibupload - append mode, updating 005 control tag after modifiction """
recs = bibupload.xml_marc_to_records(self.test_to_append)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
rec = get_record(recid)
query = """SELECT DATE_FORMAT(modification_date,'%%Y%%m%%d%%H%%i%%s') FROM bibrec where id = %s"""
res = run_sql(query % recid)
self.assertEqual(str(res[0][0])+'.0',record_get_field_value(rec,'005','',''))
class BibUploadCorrectModeTest(GenericBibUploadTest):
"""
Testing correcting a record containing similar tags (identical
tag, different indicators). Currently Invenio replaces only
those tags that have matching indicators too, unlike ALEPH500 that
does not pay attention to indicators, it corrects all fields with
the same tag, regardless of the indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
self.testrec1_xm_to_correct = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Cool
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10048 $$aCool
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_record_correction(self):
"""bibupload - correct mode, similar MARCXML tags/indicators"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
err, self.recid, msg = bibupload.bibupload(recs[0], opt_mode='correct')
corrected_xm = print_record(self.recid, 'xm')
corrected_hm = print_record(self.recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# clean up after ourselves:
return
class BibUploadDeleteModeTest(GenericBibUploadTest):
"""
Testing deleting specific tags from a record while keeping anything else
untouched. Currently Invenio deletes only those tags that have
matching indicators too, unlike ALEPH500 that does not pay attention to
indicators, it corrects all fields with the same tag, regardless of the
indicator values.
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
dumb text
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
888__ $$adumb text
"""
self.testrec1_xm_to_delete = """
123456789
Test, Jane
Test Institute
Test, Johnson
Test University
Cool
dumb text
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, John
Test University
Test, Jim
Test Laboratory
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
10047 $$aTest, John$$uTest University
10047 $$aTest, Jim$$uTest Laboratory
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_delete = self.testrec1_xm_to_delete.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
# Checking dumb text is in bibxxx
self.failUnless(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
def test_record_tags_deletion(self):
"""bibupload - delete mode, deleting specific tags"""
# correct some tags:
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_delete)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='delete')
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
# Checking dumb text is no more in bibxxx
self.failIf(run_sql("SELECT id_bibrec from bibrec_bib88x WHERE id_bibrec=%s", (recid, )))
# clean up after ourselves:
class BibUploadReplaceModeTest(GenericBibUploadTest):
"""Testing replace mode."""
def test_record_replace(self):
"""bibupload - replace mode, similar MARCXML tags/indicators"""
# replace some tags:
testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
"""
testrec1_xm_to_replace = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
testrec1_replaced_xm = """
123456789
Test, Joseph
Test Academy
Test2, Joseph
Test2 Academy
"""
testrec1_replaced_hm = """
001__ 123456789
10047 $$aTest, Joseph$$uTest Academy
10047 $$aTest2, Joseph$$uTest2 Academy
"""
# insert test record:
test_record_xm = testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
testrec1_xm = testrec1_xm.replace('123456789', str(recid))
testrec1_hm = testrec1_hm.replace('123456789', str(recid))
testrec1_xm_to_replace = testrec1_xm_to_replace.replace('123456789', str(recid))
testrec1_replaced_xm = testrec1_replaced_xm.replace('123456789', str(recid))
testrec1_replaced_hm = testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, testrec1_hm), '')
recs = bibupload.xml_marc_to_records(testrec1_xm_to_replace)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, testrec1_replaced_hm), '')
def test_record_replace_force_non_existing(self):
"""bibupload - replace mode, force non existing recid"""
# replace some tags:
the_recid = self.last_recid + 1
testrec1_xm = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
""" % the_recid
testrec1_hm = """
001__ %s
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
""" % the_recid
recs = bibupload.xml_marc_to_records(testrec1_xm)
task_set_option('force', True)
try:
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
finally:
task_set_option('force', False)
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, testrec1_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, testrec1_hm), '')
self.assertEqual(recid, the_recid)
def test_record_replace_non_existing(self):
"""bibupload - replace mode, non existing recid"""
# replace some tags:
the_recid = self.last_recid + 1
testrec1_xm = """
%s
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
""" % the_recid
testrec1_hm = """
001__ %s
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
10047 $$aTest, John$$uTest University
10048 $$aCool
10047 $$aTest, Jim$$uTest Laboratory
""" % the_recid
recs = bibupload.xml_marc_to_records(testrec1_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual((err, recid), (1, -1))
def test_record_replace_two_recids(self):
"""bibupload - replace mode, two recids"""
# replace some tags:
testrec1_xm = """
300
305
SzGeCERN
Test, Jane
Test Institute
Test, John
Test University
Cool
Test, Jim
Test Laboratory
"""
recs = bibupload.xml_marc_to_records(testrec1_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
# did it work?
self.assertEqual((err, recid), (1, -1))
class BibUploadReferencesModeTest(GenericBibUploadTest):
"""Testing references mode."""
def setUp(self):
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.test_insert = """
123456789
Tester, T
CERN
"""
self.test_reference = """
123456789
M. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,
J. High Energy Phys. 07 (2004) 014
"""
self.test_reference_expected_xm = """
123456789
Tester, T
CERN
M. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,
J. High Energy Phys. 07 (2004) 014
"""
self.test_insert_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
"""
self.test_reference_expected_hm = """
001__ 123456789
100__ $$aTester, T$$uCERN
%(reference_tag)sC5 $$mM. Lüscher and P. Weisz, String excitation energies in SU(N) gauge theories beyond the free-string approximation,$$sJ. High Energy Phys. 07 (2004) 014
""" % {'reference_tag': bibupload.CFG_BIBUPLOAD_REFERENCE_TAG}
# insert test record:
test_insert = self.test_insert.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_insert)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.test_insert = self.test_insert.replace('123456789', str(recid))
self.test_insert_hm = self.test_insert_hm.replace('123456789', str(recid))
self.test_reference = self.test_reference.replace('123456789', str(recid))
self.test_reference_expected_xm = self.test_reference_expected_xm.replace('123456789', str(recid))
self.test_reference_expected_hm = self.test_reference_expected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.test_insert), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.test_insert_hm), '')
self.test_recid = recid
def test_reference_complete_xml_marc(self):
"""bibupload - reference mode, inserting references MARCXML file"""
# We create create the record out of the xml marc
recs = bibupload.xml_marc_to_records(self.test_reference)
# We call the main function with the record as a parameter
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='reference')
# We retrieve the inserted xml
reference_xm = print_record(recid, 'xm')
reference_hm = print_record(recid, 'hm')
# Compare if the two MARCXML are the same
self.assertEqual(compare_xmbuffers(reference_xm, self.test_reference_expected_xm), '')
self.assertEqual(compare_hmbuffers(reference_hm, self.test_reference_expected_hm), '')
class BibUploadFMTModeTest(GenericBibUploadTest):
"""Testing FMT mode."""
def setUp(self):
"""Initialize the MARCXML variable"""
GenericBibUploadTest.setUp(self)
self.new_xm_with_fmt = """
SzGeCERN
HB
Test. Okay.
2008-03-14 15:14:00
Bar, Baz
Foo
On the quux and huux
"""
self.expected_xm_after_inserting_new_xm_with_fmt = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux
"""
self.expected_hm_after_inserting_new_xm_with_fmt = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux
"""
self.recid76_xm_before_all_the_tests = print_record(76, 'xm')
self.recid76_hm_before_all_the_tests = print_record(76, 'hm')
self.recid76_fmts = run_sql("""SELECT last_updated, value, format FROM bibfmt WHERE id_bibrec=76""")
self.recid76_xm_with_fmt = """
76
SzGeCERN
HB
Test. Here is some format value.
Doe, John
CERN
On the foos and bars
"""
self.recid76_xm_with_fmt_only_first = """
76
HB
Test. Let us see if this gets inserted well.
"""
self.recid76_xm_with_fmt_only_second = """
76
HB
Test. Yet another test, to be run after the first one.
HD
Test. Let's see what will be stored in the detailed format field.
"""
def tearDown(self):
"""Helper function that restores recID 76 MARCXML, using the
value saved before all the tests started to execute.
(see self.recid76_xm_before_all_the_tests).
Does not restore HB and HD formats.
"""
recs = bibupload.xml_marc_to_records(self.recid76_xm_before_all_the_tests)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
for (last_updated, value, format) in self.recid76_fmts:
run_sql("""UPDATE bibfmt SET last_updated=%s, value=%s WHERE id_bibrec=76 AND format=%s""", (last_updated, value, format))
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.recid76_xm_before_all_the_tests), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.recid76_hm_before_all_the_tests), '')
def test_inserting_new_record_containing_fmt_tag(self):
"""bibupload - FMT tag, inserting new record containing FMT tag"""
recs = bibupload.xml_marc_to_records(self.new_xm_with_fmt)
dummy, new_recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
xm_after = print_record(new_recid, 'xm')
hm_after = print_record(new_recid, 'hm')
hb_after = print_record(new_recid, 'hb')
self.assertEqual(compare_xmbuffers(xm_after,
self.expected_xm_after_inserting_new_xm_with_fmt.replace('123456789', str(new_recid))), '')
self.assertEqual(compare_hmbuffers(hm_after,
self.expected_hm_after_inserting_new_xm_with_fmt.replace('123456789', str(new_recid))), '')
self.assertEqual(run_sql('SELECT last_updated from bibfmt WHERE id_bibrec=%s', (new_recid, ))[0][0], datetime.datetime(2008, 3, 14, 15, 14))
self.failUnless(hb_after.startswith("Test. Okay."))
def test_updating_existing_record_formats_in_format_mode(self):
"""bibupload - FMT tag, updating existing record via format mode"""
xm_before = print_record(76, 'xm')
hm_before = print_record(76, 'hm')
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='format')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Let us see if this gets inserted well."))
# now insert another format value and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_second)
bibupload.bibupload(recs[0], opt_mode='format')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(xm_after, xm_before)
self.assertEqual(hm_after, hm_before)
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
def test_updating_existing_record_formats_in_correct_mode(self):
"""bibupload - FMT tag, updating existing record via correct mode"""
xm_before = print_record(76, 'xm')
hm_before = print_record(76, 'hm')
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='correct')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(compare_xmbuffers(xm_after, xm_before), '')
self.assertEqual(compare_hmbuffers(hm_after, hm_before), '')
self.failUnless(hb_after.startswith("Test. Let us see if this gets inserted well."))
# now insert another format value and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_second)
bibupload.bibupload(recs[0], opt_mode='correct')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(compare_xmbuffers(xm_after, xm_before), '')
self.assertEqual(compare_hmbuffers(hm_after, hm_before), '')
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
def test_updating_existing_record_formats_in_replace_mode(self):
"""bibupload - FMT tag, updating existing record via replace mode"""
# insert first format value:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt_only_first)
bibupload.bibupload(recs[0], opt_mode='replace')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
self.assertEqual(compare_xmbuffers(xm_after,
'76
76
"""), '')
self.assertEqual(compare_hmbuffers(hm_after, '000000076 001__ 76'), '')
self.failUnless(hb_after.startswith("Test. Yet another test, to be run after the first one."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
# final insertion and recheck:
recs = bibupload.xml_marc_to_records(self.recid76_xm_with_fmt)
bibupload.bibupload(recs[0], opt_mode='replace')
xm_after = print_record(76, 'xm')
hm_after = print_record(76, 'hm')
hb_after = print_record(76, 'hb')
hd_after = print_record(76, 'hd')
self.assertEqual(compare_xmbuffers(xm_after, """
76
SzGeCERN
Doe, John
CERN
On the foos and bars
"""), '')
self.assertEqual(compare_hmbuffers(hm_after, """
001__ 76
003__ SzGeCERN
100__ $$aDoe, John$$uCERN
245__ $$aOn the foos and bars
"""), '')
self.failUnless(hb_after.startswith("Test. Here is some format value."))
self.failUnless(hd_after.startswith("Test. Let's see what will be stored in the detailed format field."))
class BibUploadRecordsWithSYSNOTest(GenericBibUploadTest):
"""Testing uploading of records that have external SYSNO present."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that SYSNO fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
sysno1
sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno1
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno2
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
Bar, Baz
Foo
On the quux and huux 2
sysno2
sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4] or " ",
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5] or " ",
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$%(sysnosubfieldcode)ssysno2
%(sysnotag)s%(sysnoind1)s%(sysnoind2)s $$0sysno1
""" % {'sysnotag': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[0:3],
'sysnoind1': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[3:4],
'sysnoind2': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[4:5],
'sysnosubfieldcode': CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG[5:6],
}
def test_insert_the_same_sysno_record(self):
"""bibupload - SYSNO tag, refuse to insert the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_sysno_record() finished"
def test_insert_or_replace_the_same_sysno_record(self):
"""bibupload - SYSNO tag, allow to insert or replace the same SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0],
opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_sysno_record() finished"
def test_replace_nonexisting_sysno_record(self):
"""bibupload - SYSNO tag, refuse to replace non-existing SYSNO record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_sysno_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_sysno_record() finished"
class BibUploadRecordsWithEXTOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have external EXTOAIID present."""
def setUp(self):
# pylint: disable=C0103
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that EXTOAIID fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
self.xm_testrec1 = """
123456789
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
extoaiid1
extoaisrc1
extoaiid2
Bar, Baz
Foo
On the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid1
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid2
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
extoaiid2
extoaisrc1
extoaiid1
Bar, Baz
Foo
On the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$%(extoaisrcsubfieldcode)sextoaisrc1$$%(extoaiidsubfieldcode)sextoaiid2
%(extoaiidtag)s%(extoaiidind1)s%(extoaiidind2)s $$0extoaiid1
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4],
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5],
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'extoaisrcsubfieldcode' : CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG[5:6],
}
def test_insert_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to insert the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_the_same_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
if self.verbose:
print "test_insert_the_same_extoaiid_record() finished"
def test_insert_or_replace_the_same_extoaiid_record(self):
"""bibupload - EXTOAIID tag, allow to insert or replace the same EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() started"
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
if self.verbose:
print "test_insert_or_replace_the_same_extoaiid_record() finished"
def test_replace_nonexisting_extoaiid_record(self):
"""bibupload - EXTOAIID tag, refuse to replace non-existing EXTOAIID record"""
# initialize bibupload mode:
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() started"
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
if self.verbose:
print "test_replace_nonexisting_extoaiid_record() finished"
class BibUploadRecordsWithOAIIDTest(GenericBibUploadTest):
"""Testing uploading of records that have OAI ID present."""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
# Note that OAI fields are repeated but with different
# subfields, this is to test whether bibupload would not
# mistakenly pick up wrong values.
GenericBibUploadTest.setUp(self)
self.xm_testrec1 = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1 = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_to_update = """
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec1_updated = """
123456789
SzGeCERN
Bar, Baz
Foo
On the quux and huux 1 Updated
oai:foo:1
oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec1_updated = """
001__ 123456789
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 1 Updated
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:1
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:2
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.xm_testrec2 = """
987654321
SzGeCERN
Bar, Baz
Foo
On the quux and huux 2
oai:foo:2
oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4] != "_" and \
CFG_OAI_ID_FIELD[3:4] or " ",
'oaiind2': CFG_OAI_ID_FIELD[4:5] != "_" and \
CFG_OAI_ID_FIELD[4:5] or " ",
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
self.hm_testrec2 = """
001__ 987654321
003__ SzGeCERN
100__ $$aBar, Baz$$uFoo
245__ $$aOn the quux and huux 2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$%(oaisubfieldcode)soai:foo:2
%(oaitag)s%(oaiind1)s%(oaiind2)s $$0oai:foo:1
""" % {'oaitag': CFG_OAI_ID_FIELD[0:3],
'oaiind1': CFG_OAI_ID_FIELD[3:4],
'oaiind2': CFG_OAI_ID_FIELD[4:5],
'oaisubfieldcode': CFG_OAI_ID_FIELD[5:6],
}
def test_insert_the_same_oai_record(self):
"""bibupload - OAIID tag, refuse to insert the same OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# insert record 2 first time:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid2, 'xm')
inserted_hm = print_record(recid2, 'hm')
# use real recID when comparing whether it worked:
self.xm_testrec2 = self.xm_testrec2.replace('987654321', str(recid2))
self.hm_testrec2 = self.hm_testrec2.replace('987654321', str(recid2))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec2), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec2), '')
# try to insert updated record 1, it should fail:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='insert')
self.assertEqual(-1, recid1_updated)
def test_insert_or_replace_the_same_oai_record(self):
"""bibupload - OAIID tag, allow to insert or replace the same OAI record"""
# initialize bibupload mode:
# insert/replace record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to insert/replace updated record 1, it should be okay:
recs = bibupload.xml_marc_to_records(self.xm_testrec1_to_update)
err1_updated, recid1_updated, msg1_updated = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1_updated, 'xm')
inserted_hm = print_record(recid1_updated, 'hm')
self.assertEqual(recid1, recid1_updated)
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1_updated = self.xm_testrec1_updated.replace('123456789', str(recid1))
self.hm_testrec1_updated = self.hm_testrec1_updated.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1_updated), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1_updated), '')
def test_replace_nonexisting_oai_record(self):
"""bibupload - OAIID tag, refuse to replace non-existing OAI record"""
# insert record 1 first time:
testrec_to_insert_first = self.xm_testrec1.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='replace_or_insert')
inserted_xm = print_record(recid1, 'xm')
inserted_hm = print_record(recid1, 'hm')
# use real recID in test buffers when comparing whether it worked:
self.xm_testrec1 = self.xm_testrec1.replace('123456789', str(recid1))
self.hm_testrec1 = self.hm_testrec1.replace('123456789', str(recid1))
self.assertEqual(compare_xmbuffers(inserted_xm,
self.xm_testrec1), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
self.hm_testrec1), '')
# try to replace record 2 it should fail:
testrec_to_insert_first = self.xm_testrec2.replace('987654321 ',
'')
recs = bibupload.xml_marc_to_records(testrec_to_insert_first)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='replace')
self.assertEqual(-1, recid2)
class BibUploadIndicatorsTest(GenericBibUploadTest):
"""
Testing uploading of a MARCXML record with indicators having
either blank space (as per MARC schema) or empty string value (old
behaviour).
"""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
def test_record_with_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec1_hm), '')
def test_record_with_no_spaces_in_indicators(self):
"""bibupload - inserting MARCXML with no spaces in indicators"""
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(inserted_hm),
self.testrec2_hm), '')
class BibUploadUpperLowerCaseTest(GenericBibUploadTest):
"""
Testing treatment of similar records with only upper and lower
case value differences in the bibxxx table.
"""
def setUp(self):
"""Initialize the MARCXML test records."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
SzGeCERN
Test, John
Test University
"""
self.testrec1_hm = """
003__ SzGeCERN
100__ $$aTest, John$$uTest University
"""
self.testrec2_xm = """
SzGeCERN
TeSt, JoHn
Test UniVeRsity
"""
self.testrec2_hm = """
003__ SzGeCERN
100__ $$aTeSt, JoHn$$uTest UniVeRsity
"""
def test_record_with_upper_lower_case_letters(self):
"""bibupload - inserting similar MARCXML records with upper/lower case"""
# insert test record #1:
recs = bibupload.xml_marc_to_records(self.testrec1_xm)
err1, recid1, msg1 = bibupload.bibupload(recs[0], opt_mode='insert')
recid1_inserted_xm = print_record(recid1, 'xm')
recid1_inserted_hm = print_record(recid1, 'hm')
# insert test record #2:
recs = bibupload.xml_marc_to_records(self.testrec2_xm)
err2, recid2, msg2 = bibupload.bibupload(recs[0], opt_mode='insert')
recid2_inserted_xm = print_record(recid2, 'xm')
recid2_inserted_hm = print_record(recid2, 'hm')
# let us compare stuff now:
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid1_inserted_xm),
self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid1_inserted_hm),
self.testrec1_hm), '')
self.assertEqual(compare_xmbuffers(remove_tag_001_from_xmbuffer(recid2_inserted_xm),
self.testrec2_xm), '')
self.assertEqual(compare_hmbuffers(remove_tag_001_from_hmbuffer(recid2_inserted_hm),
self.testrec2_hm), '')
class BibUploadControlledProvenanceTest(GenericBibUploadTest):
"""Testing treatment of tags under controlled provenance in the correct mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
blabla
sam
blublu
sim
human
"""
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$9sim$$ablublu
6531_ $$ahuman
"""
self.testrec1_xm_to_correct = """
123456789
bleble
sim
bloblo
som
"""
self.testrec1_corrected_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
blabla
sam
human
bleble
sim
bloblo
som
"""
self.testrec1_corrected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
6531_ $$9sam$$ablabla
6531_ $$ahuman
6531_ $$9sim$$ableble
6531_ $$9som$$abloblo
"""
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_correct = self.testrec1_xm_to_correct.replace('123456789', str(recid))
self.testrec1_corrected_xm = self.testrec1_corrected_xm.replace('123456789', str(recid))
self.testrec1_corrected_hm = self.testrec1_corrected_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_controlled_provenance_persistence(self):
"""bibupload - correct mode, tags with controlled provenance"""
# correct metadata tags; will the protected tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_correct)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='correct')
corrected_xm = print_record(recid, 'xm')
corrected_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(corrected_xm, self.testrec1_corrected_xm), '')
self.assertEqual(compare_hmbuffers(corrected_hm, self.testrec1_corrected_hm), '')
class BibUploadStrongTagsTest(GenericBibUploadTest):
"""Testing treatment of strong tags and the replace mode."""
def setUp(self):
"""Initialize the MARCXML test record."""
GenericBibUploadTest.setUp(self)
self.testrec1_xm = """
123456789
SzGeCERN
Test, Jane
Test Institute
Test title
A value
Another value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, Jane$$uTest Institute
245__ $$aTest title
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_xm_to_replace = """
123456789
Test, Joseph
Test Academy
"""
self.testrec1_replaced_xm = """
123456789
Test, Joseph
Test Academy
A value
Another value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
self.testrec1_replaced_hm = """
001__ 123456789
100__ $$aTest, Joseph$$uTest Academy
%(strong_tag)s__ $$aA value$$bAnother value
""" % {'strong_tag': bibupload.CFG_BIBUPLOAD_STRONG_TAGS[0]}
# insert test record:
test_record_xm = self.testrec1_xm.replace('123456789 ',
'')
recs = bibupload.xml_marc_to_records(test_record_xm)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recID:
self.testrec1_xm = self.testrec1_xm.replace('123456789', str(recid))
self.testrec1_hm = self.testrec1_hm.replace('123456789', str(recid))
self.testrec1_xm_to_replace = self.testrec1_xm_to_replace.replace('123456789', str(recid))
self.testrec1_replaced_xm = self.testrec1_replaced_xm.replace('123456789', str(recid))
self.testrec1_replaced_hm = self.testrec1_replaced_hm.replace('123456789', str(recid))
# test of the inserted record:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm, self.testrec1_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm, self.testrec1_hm), '')
def test_strong_tags_persistence(self):
"""bibupload - strong tags, persistence in replace mode"""
# replace all metadata tags; will the strong tags be kept?
recs = bibupload.xml_marc_to_records(self.testrec1_xm_to_replace)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='replace')
replaced_xm = print_record(recid, 'xm')
replaced_hm = print_record(recid, 'hm')
# did it work?
self.assertEqual(compare_xmbuffers(replaced_xm, self.testrec1_replaced_xm), '')
self.assertEqual(compare_hmbuffers(replaced_hm, self.testrec1_replaced_hm), '')
class BibUploadPretendTest(GenericBibUploadTest):
"""
Testing bibupload --pretend correctness.
"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.demo_data = bibupload.xml_marc_to_records(open(os.path.join(CFG_TMPDIR, 'demobibdata.xml')).read())[0]
self.before = self._get_tables_fingerprint()
task_set_task_param('pretend', True)
def tearDown(self):
task_set_task_param('pretend', False)
def _get_tables_fingerprint():
"""
Take lenght and last modification time of all the tables that
might be touched by bibupload and return them in a nice structure.
"""
fingerprint = {}
tables = ['bibrec', 'bibdoc', 'bibrec_bibdoc', 'bibdoc_bibdoc', 'bibfmt', 'hstDOCUMENT', 'hstRECORD']
for i in xrange(100):
tables.append('bib%02dx' % i)
tables.append('bibrec_bib%02dx' % i)
for table in tables:
fingerprint[table] = get_table_status_info(table)
return fingerprint
_get_tables_fingerprint = staticmethod(_get_tables_fingerprint)
def _checks_tables_fingerprints(before, after):
"""
Checks differences in table_fingerprints.
"""
err = True
for table in before.keys():
if before[table] != after[table]:
print >> sys.stderr, "Table %s has been modified: before was [%s], after was [%s]" % (table, pprint.pformat(before[table]), pprint.pformat(after[table]))
err = False
return err
_checks_tables_fingerprints = staticmethod(_checks_tables_fingerprints)
def test_pretend_insert(self):
"""bibupload - pretend insert"""
bibupload.bibupload(self.demo_data, opt_mode='insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_correct(self):
"""bibupload - pretend correct"""
bibupload.bibupload(self.demo_data, opt_mode='correct', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace(self):
"""bibupload - pretend replace"""
bibupload.bibupload(self.demo_data, opt_mode='replace', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_append(self):
"""bibupload - pretend append"""
bibupload.bibupload(self.demo_data, opt_mode='append', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_replace_or_insert(self):
"""bibupload - pretend replace or insert"""
bibupload.bibupload(self.demo_data, opt_mode='replace_or_insert', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_holdingpen(self):
"""bibupload - pretend holdingpen"""
bibupload.bibupload(self.demo_data, opt_mode='holdingpen', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_delete(self):
"""bibupload - pretend delete"""
bibupload.bibupload(self.demo_data, opt_mode='delete', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
def test_pretend_reference(self):
"""bibupload - pretend reference"""
bibupload.bibupload(self.demo_data, opt_mode='reference', pretend=True)
self.failUnless(self._checks_tables_fingerprints(self.before, self._get_tables_fingerprint()))
class BibUploadHoldingPenTest(GenericBibUploadTest):
"""
Testing the Holding Pen usage.
"""
def setUp(self):
GenericBibUploadTest.setUp(self)
self.verbose = 9
setup_loggers()
task_set_task_param('verbose', self.verbose)
self.recid = 10
self.oai_id = "oai:cds.cern.ch:CERN-EP-2001-094"
def test_holding_pen_upload_with_recid(self):
"""bibupload - holding pen upload with recid"""
test_to_upload = """
%s
Kleefeld, F
Newcomer, Y
Rupp, G
Scadron, M D
""" % self.recid
recs = bibupload.xml_marc_to_records(test_to_upload)
bibupload.insert_record_into_holding_pen(recs[0], "")
res = run_sql("SELECT changeset_xml FROM bibHOLDINGPEN WHERE id_bibrec=%s", (self.recid, ))
self.failUnless("Rupp, G" in res[0][0])
def test_holding_pen_upload_with_oai_id(self):
"""bibupload - holding pen upload with oai_id"""
test_to_upload = """
Kleefeld, F
Newcomer, Y
Rupp, G
Scadron, M D
%(value)s
""" % {'extoaiidtag': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[0:3],
'extoaiidind1': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[3:4] or " ",
'extoaiidind2': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] != "_" and \
CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[4:5] or " ",
'extoaiidsubfieldcode': CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG[5:6],
'value': self.oai_id
}
recs = bibupload.xml_marc_to_records(test_to_upload)
bibupload.insert_record_into_holding_pen(recs[0], self.oai_id)
res = run_sql("SELECT changeset_xml FROM bibHOLDINGPEN WHERE id_bibrec=%s AND oai_id=%s", (self.recid, self.oai_id))
self.failUnless("Rupp, G" in res[0][0])
def tearDown(self):
GenericBibUploadTest.tearDown(self)
run_sql("DELETE FROM bibHOLDINGPEN WHERE id_bibrec=%s", (self.recid, ))
class BibUploadFFTModeTest(GenericBibUploadTest):
"""
Testing treatment of fulltext file transfer import mode.
"""
def _test_bibdoc_status(self, recid, docname, status):
res = run_sql('SELECT bd.status FROM bibrec_bibdoc as bb JOIN bibdoc as bd ON bb.id_bibdoc = bd.id WHERE bb.id_bibrec = %s AND bd.docname = %s', (recid, docname))
self.failUnless(res)
self.assertEqual(status, res[0][0])
def test_writing_rights(self):
"""bibupload - FFT has writing rights"""
self.failUnless(bibupload.writing_rights_p())
def test_simple_fft_insert(self):
"""bibupload - simple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url))
def test_fft_insert_with_valid_embargo(self):
"""bibupload - FFT insert with valid embargo"""
# define the test case:
import time
future_date = time.strftime('%Y-%m-%d', time.gmtime(time.time() + 24 * 3600 * 2))
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
firerole: deny until '%(future_date)s'
allow any
""" % {
'future_date': future_date,
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
result = urlopen(testrec_expected_url).read()
self.failUnless("This file is restricted." in result, result)
def test_fft_insert_with_expired_embargo(self):
"""bibupload - FFT insert with expired embargo"""
# define the test case:
import time
past_date = time.strftime('%Y-%m-%d', time.gmtime(time.time() - 24 * 3600 * 2))
test_to_upload = """
SzGeCERN
Test, John
Test University
ARTICLE
%(siteurl)s/img/site_logo.gif
firerole: deny until '%(past_date)s'
allow any
""" % {
'past_date': past_date,
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
ARTICLE
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
980__ $$aARTICLE
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
result = urlopen(testrec_expected_url).read()
self.failIf("If you already have an account, please login using the form below." in result, result)
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text=urlopen("%(siteurl)s/img/site_logo.gif" % {
'siteurl': CFG_SITE_URL
}).read()), [])
def test_exotic_format_fft_append(self):
"""bibupload - exotic format FFT append"""
# define the test case:
testfile = os.path.join(CFG_TMPDIR, 'test.ps.Z')
open(testfile, 'w').write('TEST')
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
""" % {
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_to_append = """
123456789
%s
""" % testfile
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
%(email_tag)s%(email_ind1)s%(email_ind2)s $$%(email_code)sjekyll@cds.cern.ch
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == ' ' and '_' or email_ind1,
'email_ind2': email_ind2 == ' ' and '_' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test.ps.Z" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/test?format=ps.Z" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_to_append = testrec_to_append.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url.replace('123456789',
str(recid))
recs = bibupload.xml_marc_to_records(testrec_to_append)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='append')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.assertEqual(test_web_page_content(testrec_expected_url, 'jekyll', 'j123ekyll', expected_text='TEST'), [])
self.assertEqual(test_web_page_content(testrec_expected_url2, 'jekyll', 'j123ekyll', expected_text='TEST'), [])
def test_fft_check_md5_through_bibrecdoc_str(self):
"""bibupload - simple FFT insert, check md5 through BibRecDocs.str()"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%s/img/head.gif
""" % CFG_SITE_URL
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
original_md5 = md5(urlopen('%s/img/head.gif' % CFG_SITE_URL).read()).hexdigest()
bibrec_str = str(BibRecDocs(int(recid)))
md5_found = False
for row in bibrec_str.split('\n'):
if 'checksum' in row:
if original_md5 in row:
md5_found = True
self.failUnless(md5_found)
def test_detailed_fft_insert(self):
"""bibupload - detailed FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
SuperMain
This is a description
This is a comment
CIDIESSE
%(siteurl)s/img/rss.png
SuperMain
.jpeg
This is a description
This is a second comment
CIDIESSE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif
This is a description
This is a comment
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg
This is a description
This is a second comment
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif$$yThis is a description$$zThis is a comment
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg$$yThis is a description$$zThis is a second comment
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url1 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.gif" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/CIDIESSE.jpeg" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url1 = testrec_expected_url1.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url1.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url1))
self.failUnless(try_url_download(testrec_expected_url2))
def test_simple_fft_insert_with_restriction(self):
"""bibupload - simple FFT insert with restriction"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
ARTICLE
%(siteurl)s/img/site_logo.gif
thesis
%(siteurl)s/img/sb.gif
""" % {'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code,
'siteurl': CFG_SITE_URL}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
ARTICLE
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
%(email_tag)s%(email_ind1)s%(email_ind2)s $$%(email_code)sjekyll@cds.cern.ch
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon$$xicon
980__ $$aARTICLE
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == ' ' and '_' or email_ind1,
'email_ind2': email_ind2 == ' ' and '_' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_icon = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.assertEqual(test_web_page_content(testrec_expected_icon, 'jekyll', 'j123ekyll', expected_text=urlopen('%(siteurl)s/img/sb.gif' % {
'siteurl': CFG_SITE_URL
}).read()), [])
self.assertEqual(test_web_page_content(testrec_expected_icon, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
force_webcoll(recid)
self.assertEqual(test_web_page_content(testrec_expected_icon, 'hyde', 'h123yde', expected_text=urlopen('%(siteurl)s/img/restricted.gif' % {'siteurl': CFG_SITE_URL}).read()), [])
self.failUnless("HTTP Error 401: Unauthorized" in test_web_page_content(testrec_expected_url, 'hyde', 'h123yde')[0])
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
def test_simple_fft_insert_with_icon(self):
"""bibupload - simple FFT insert with icon"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/sb.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon$$xicon
""" % {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_icon = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_icon = testrec_expected_icon.replace('123456789',
str(recid))
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_icon))
def test_multiple_fft_insert(self):
"""bibupload - multiple FFT insert"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/head.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/95/files/9809057.pdf
%(prefix)s/var/tmp/demobibdata.xml
""" % {
'prefix': CFG_PREFIX,
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/9809057.pdf
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/demobibdata.xml
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/9809057.pdf
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/demobibdata.xml
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
testrec_expected_urls = []
for files in ('site_logo.gif', 'head.gif', '9809057.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'files' : files})
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_urls = []
for files in ('site_logo.gif', 'head.gif', '9809057.pdf', 'demobibdata.xml'):
testrec_expected_urls.append('%(siteurl)s/%(CFG_SITE_RECORD)s/%(recid)s/files/%(files)s' % {'siteurl' : CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD, 'files' : files, 'recid' : recid})
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
for url in testrec_expected_urls:
self.failUnless(try_url_download(url))
self._test_bibdoc_status(recid, 'head', '')
self._test_bibdoc_status(recid, '9809057', '')
self._test_bibdoc_status(recid, 'site_logo', '')
self._test_bibdoc_status(recid, 'demobibdata', '')
def test_simple_fft_correct(self):
"""bibupload - simple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/sb.gif
site_logo
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'site_logo', '')
def test_fft_correct_already_exists(self):
"""bibupload - FFT correct with already identical existing file"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
a description
%(siteurl)s/img/help.png
site_logo
another description
%(siteurl)s/img/rss.png
%(siteurl)s/img/line.gif
%(siteurl)s/img/merge.png
line
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/site_logo.gif
a second description
%(siteurl)s/img/help.png
site_logo
another second description
%(siteurl)s/img/refresh.png
rss
%(siteurl)s/img/line.gif
%(siteurl)s/img/merge-small.png
line
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a second description
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif$$ya second description
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png$$yanother second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/rss.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url3 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url4 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url5 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/line.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
testrec_expected_url2 = testrec_expected_url2.replace('123456789',
str(recid))
testrec_expected_url3 = testrec_expected_url3.replace('123456789',
str(recid))
testrec_expected_url4 = testrec_expected_url4.replace('123456789',
str(recid))
testrec_expected_url5 = testrec_expected_url5.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_url2))
self.failUnless(try_url_download(testrec_expected_url3))
self.failUnless(try_url_download(testrec_expected_url4))
self.failUnless(try_url_download(testrec_expected_url5))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
bibrecdocs = BibRecDocs(recid)
self.failUnless(bibrecdocs.get_bibdoc('rss').list_versions(), [1, 2])
self.failUnless(bibrecdocs.get_bibdoc('site_logo').list_versions(), [1])
self.failUnless(bibrecdocs.get_bibdoc('line').list_versions(), [1, 2])
def test_fft_append_already_exists(self):
"""bibupload - FFT append with already identical existing file"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
a description
""" % {
'siteurl': CFG_SITE_URL
}
test_to_append = """
123456789
%(siteurl)s/img/site_logo.gif
a second description
%(siteurl)s/img/help.png
site_logo
another second description
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
a description
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png
another second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif$$ya description
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png$$yanother second description
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url2 = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.png" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_append = test_to_append.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_append)
bibupload.bibupload(recs[0], opt_mode='append')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.failUnless(try_url_download(testrec_expected_url2))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
bibrecdocs = BibRecDocs(recid)
def test_fft_implicit_fix_marc(self):
"""bibupload - FFT implicit FIX-MARC"""
test_to_upload = """
SzGeCERN
Test, John
Test University
foo@bar.com
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
foo@bar.com
%(siteurl)s/img/site_logo.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
foo@bar.com
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8560_ $$ffoo@bar.com
8564_ $$u%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
test_to_correct = test_to_correct.replace('123456789',
str(recid))
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
# correct test record with implicit FIX-MARC:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
def test_fft_vs_bibedit(self):
"""bibupload - FFT Vs. BibEdit compatibility"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_replace = """
123456789
SzGeCERN
Test, John
Test University
http://www.google.com/
BibEdit Comment
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
BibEdit Description
01
http://cern.ch/
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_xm = str(test_to_replace)
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$uhttp://www.google.com/
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif$$x01$$yBibEdit Description$$zBibEdit Comment
8564_ $$uhttp://cern.ch/
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload(recs[0], opt_mode='replace')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'site_logo', '')
bibrecdocs = BibRecDocs(recid)
bibdoc = bibrecdocs.get_bibdoc('site_logo')
self.assertEqual(bibdoc.get_description('.gif'), 'BibEdit Description')
def test_detailed_fft_correct(self):
"""bibupload - detailed FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/head.gif
site_logo
patata
Next Try
KEEP-OLD-VALUE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Next Try
Comment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif$$yNext Try$$zComment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', '')
def test_no_url_fft_correct(self):
"""bibupload - no_url FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
site_logo
patata
.gif
KEEP-OLD-VALUE
Next Comment
"""
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
Try
Next Comment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif$$yTry$$zNext Comment
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', '')
def test_new_icon_fft_append(self):
"""bibupload - new icon FFT append"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
"""
test_to_correct = """
123456789
site_logo
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon
icon
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon$$xicon
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif?subformat=icon" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='append')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'site_logo', '')
def test_multiple_fft_correct(self):
"""bibupload - multiple FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
Try
Comment
Restricted
%(siteurl)s/img/okay.gif
site_logo
.jpeg
Try jpeg
Comment jpeg
Restricted
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/loading.gif
site_logo
patata
.gif
New restricted
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/patata.gif" \
% {'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless("This file is restricted." in urlopen(testrec_expected_url).read())
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'patata', 'New restricted')
def test_purge_fft_correct(self):
"""bibupload - purge FFT correct"""
# define the test case:
test_to_upload = """
SzGeCERN
Test, John
Test University
%(siteurl)s/img/site_logo.gif
%(siteurl)s/img/head.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_correct = """
123456789
%(siteurl)s/img/site_logo.gif
""" % {
'siteurl': CFG_SITE_URL
}
test_to_purge = """
123456789
%(siteurl)s/img/site_logo.gif
PURGE
""" % {
'siteurl': CFG_SITE_URL
}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_purge = test_to_purge.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# purge test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_purge)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'site_logo', '')
self._test_bibdoc_status(recid, 'head', '')
def test_revert_fft_correct(self):
"""bibupload - revert FFT correct"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/iconpen.gif
site_logo
""" % {
'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
test_to_correct = """
123456789
%s/img/head.gif
site_logo
""" % CFG_SITE_URL
test_to_revert = """
123456789
site_logo
REVERT
1
"""
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
%(email_tag)s%(email_ind1)s%(email_ind2)s $$%(email_code)sjekyll@cds.cern.ch
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif
""" % {'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == ' ' and '_' or email_ind1,
'email_ind2': email_ind2 == ' ' and '_' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/site_logo.gif" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_correct = test_to_correct.replace('123456789',
str(recid))
test_to_revert = test_to_revert.replace('123456789',
str(recid))
# correct test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_correct)
bibupload.bibupload(recs[0], opt_mode='correct')
# revert test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_revert)
bibupload.bibupload(recs[0], opt_mode='correct')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
self._test_bibdoc_status(recid, 'site_logo', '')
expected_content_version1 = urlopen('%s/img/iconpen.gif' % CFG_SITE_URL).read()
expected_content_version2 = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
expected_content_version3 = expected_content_version1
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=1' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version1), [])
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=2' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version2), [])
self.assertEqual(test_web_page_content('%s/%s/%s/files/site_logo.gif?version=3' % (CFG_SITE_URL, CFG_SITE_RECORD, recid), 'jekyll', 'j123ekyll', expected_content_version3), [])
def test_simple_fft_replace(self):
"""bibupload - simple FFT replace"""
# define the test case:
email_tag = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][0:3]
email_ind1 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][3]
email_ind2 = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][4]
email_code = CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS[0][5]
test_to_upload = """
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/iconpen.gif
site_logo
""" % {'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
test_to_replace = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/img/head.gif
""" % {'siteurl': CFG_SITE_URL,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_xm = """
123456789
SzGeCERN
Test, John
Test University
jekyll@cds.cern.ch
%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
""" % {
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == '_' and ' ' or email_ind1,
'email_ind2': email_ind2 == '_' and ' ' or email_ind2,
'email_code': email_code}
testrec_expected_hm = """
001__ 123456789
003__ SzGeCERN
100__ $$aTest, John$$uTest University
%(email_tag)s%(email_ind1)s%(email_ind2)s $$%(email_code)sjekyll@cds.cern.ch
8564_ $$u%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif
""" % {
'siteurl': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'email_tag': email_tag,
'email_ind1': email_ind1 == ' ' and '_' or email_ind1,
'email_ind2': email_ind2 == ' ' and '_' or email_ind2,
'email_code': email_code}
testrec_expected_url = "%(siteurl)s/%(CFG_SITE_RECORD)s/123456789/files/head.gif" % { 'siteurl': CFG_SITE_URL, 'CFG_SITE_RECORD': CFG_SITE_RECORD}
# insert test record:
recs = bibupload.xml_marc_to_records(test_to_upload)
err, recid, msg = bibupload.bibupload(recs[0], opt_mode='insert')
# replace test buffers with real recid of inserted test record:
testrec_expected_xm = testrec_expected_xm.replace('123456789',
str(recid))
testrec_expected_hm = testrec_expected_hm.replace('123456789',
str(recid))
testrec_expected_url = testrec_expected_url.replace('123456789',
str(recid))
test_to_replace = test_to_replace.replace('123456789',
str(recid))
# replace test record with new FFT:
recs = bibupload.xml_marc_to_records(test_to_replace)
bibupload.bibupload(recs[0], opt_mode='replace')
# compare expected results:
inserted_xm = print_record(recid, 'xm')
inserted_hm = print_record(recid, 'hm')
self.failUnless(try_url_download(testrec_expected_url))
self.assertEqual(compare_xmbuffers(inserted_xm,
testrec_expected_xm), '')
self.assertEqual(compare_hmbuffers(inserted_hm,
testrec_expected_hm), '')
expected_content_version = urlopen('%s/img/head.gif' % CFG_SITE_URL).read()
self.assertEqual(test_web_page_content(testrec_expected_url, 'hyde', 'h123yde', expected_text='Authorization failure'), [])
self.assertEqual(test_web_page_content(testrec_expected_url, 'jekyll', 'j123ekyll', expected_text=expected_content_version), [])
TEST_SUITE = make_test_suite(BibUploadHoldingPenTest,
BibUploadInsertModeTest,
BibUploadAppendModeTest,
BibUploadCorrectModeTest,
BibUploadDeleteModeTest,
BibUploadReplaceModeTest,
BibUploadReferencesModeTest,
BibUploadRecordsWithSYSNOTest,
BibUploadRecordsWithEXTOAIIDTest,
BibUploadRecordsWithOAIIDTest,
BibUploadFMTModeTest,
BibUploadIndicatorsTest,
BibUploadUpperLowerCaseTest,
BibUploadControlledProvenanceTest,
BibUploadStrongTagsTest,
BibUploadFFTModeTest,
BibUploadPretendTest,
BibUploadCallbackURLTest
)
if __name__ == "__main__":
run_test_suite(TEST_SUITE, warn_user=True)
diff --git a/modules/miscutil/lib/Makefile.am b/modules/miscutil/lib/Makefile.am
index 47ca2dccb..a75b89080 100644
--- a/modules/miscutil/lib/Makefile.am
+++ b/modules/miscutil/lib/Makefile.am
@@ -1,103 +1,105 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
pylibdir = $(libdir)/python/invenio
pylib_DATA = __init__.py \
errorlib.py \
errorlib_tests.py \
errorlib_webinterface.py \
errorlib_regression_tests.py \
data_cacher.py \
dbdump.py \
+ web_api_key.py \
+ web_api_key_tests.py \
dbquery.py \
dbquery_tests.py \
dbquery_regression_tests.py \
logicutils.py \
logicutils_tests.py \
mailutils.py \
miscutil_config.py \
messages.py \
messages_tests.py \
textutils.py \
textutils_tests.py \
dateutils.py \
dateutils_tests.py \
htmlutils.py \
htmlutils_tests.py \
testutils.py \
testutils_regression_tests.py \
invenio_connector.py \
invenio_connector_regression_tests.py \
urlutils.py \
urlutils_tests.py \
w3c_validator.py \
intbitset_tests.py \
intbitset_helper.py \
inveniocfg.py \
shellutils.py \
shellutils_tests.py \
pluginutils.py \
pluginutils_tests.py \
plotextractor.py \
plotextractor_converter.py \
plotextractor_getter.py \
plotextractor_output_utils.py \
plotextractor_tests.py \
plotextractor_regression_tests.py \
plotextractor_config.py \
solrutils.py \
remote_debugger.py \
remote_debugger_config.py \
remote_debugger_wsgi_reload.py \
jsonutils.py \
- sequtils_cnum.py \
- sequtils.py \
- sequtils_regression_tests.py
+ sequtils_cnum.py \
+ sequtils.py \
+ sequtils_regression_tests.py
noinst_DATA = testimport.py \
kwalitee.py \
pep8.py
tmpdir = $(prefix)/var/tmp
tmp_DATA = intbitset_example.int
EXTRA_DIST = $(pylib_DATA) \
testimport.py \
kwalitee.py \
pep8.py \
intbitset.pyx \
intbitset.c \
intbitset.h \
intbitset_impl.c \
intbitset_setup.py \
intbitset.pyx \
$(tmp_DATA)
all:
$(PYTHON) $(srcdir)/intbitset_setup.py build_ext
install-data-hook:
$(PYTHON) $(srcdir)/testimport.py ${prefix}
@find ${srcdir} -name intbitset.so -exec cp {} ${pylibdir} \;
CLEANFILES = *~ *.tmp *.pyc
clean-local:
rm -rf build
diff --git a/modules/miscutil/lib/invenio_connector.py b/modules/miscutil/lib/invenio_connector.py
index 18babba1e..a673b563f 100644
--- a/modules/miscutil/lib/invenio_connector.py
+++ b/modules/miscutil/lib/invenio_connector.py
@@ -1,607 +1,610 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Tools to connect to distant Invenio servers using Invenio APIs.
Example of use:
from InvenioConnector import *
cds = InvenioConnector("http://cdsweb.cern.ch")
results = cds.search("higgs")
for record in results:
print record["245__a"][0]
print record["520__b"][0]
for author in record["100__"]:
print author["a"][0], author["u"][0]
FIXME:
- implement cache expiration
- exceptions handling
- parsing of
- better checking of input parameters (especially InvenioConnector.__init__ "url")
- improve behaviour when running locally (perform_request_search *requiring* "req" object)
"""
import urllib
import urllib2
import xml.sax
import re
import tempfile
import os
import time
import sys
MECHANIZE_CLIENTFORM_VERSION_CHANGE = (0, 2, 0)
try:
import mechanize
if mechanize.__version__ < MECHANIZE_CLIENTFORM_VERSION_CHANGE:
OLD_MECHANIZE_VERSION = True
import ClientForm
else:
OLD_MECHANIZE_VERSION = False
MECHANIZE_AVAILABLE = True
except ImportError:
MECHANIZE_AVAILABLE = False
try:
# if we are running locally, we can optimize :-)
from invenio.config import CFG_SITE_URL, CFG_SITE_SECURE_URL, CFG_SITE_RECORD, CFG_CERN_SITE
from invenio.bibtask import task_low_level_submission
from invenio.search_engine import perform_request_search, collection_restricted_p
from invenio.bibformat import format_records
+ from invenio.urlutils import make_user_agent_string
LOCAL_SITE_URLS = [CFG_SITE_URL, CFG_SITE_SECURE_URL]
+ CFG_USER_AGENT = make_user_agent_string("invenio_connector")
except ImportError:
LOCAL_SITE_URLS = None
CFG_CERN_SITE = 0
+ CFG_USER_AGENT = "invenio_connector"
CFG_CDS_URL = "http://cdsweb.cern.ch/"
class InvenioConnectorAuthError(Exception):
"""
This exception is called by InvenioConnector when authentication fails during
remote or local connections.
"""
def __init__(self, value):
"""
Set the internal "value" attribute to that of the passed "value" parameter.
@param value: an error string to display to the user.
@type value: string
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""
Return oneself as a string (actually, return the contents of self.value).
@return: representation of error
@rtype: string
"""
return str(self.value)
class InvenioConnectorServerError(Exception):
"""
This exception is called by InvenioConnector when using it on a machine with no
Invenio installed and no remote server URL is given during instantiation.
"""
def __init__(self, value):
"""
Set the internal "value" attribute to that of the passed "value" parameter.
@param value: an error string to display to the user.
@type value: string
"""
Exception.__init__(self)
self.value = value
def __str__(self):
"""
Return oneself as a string (actually, return the contents of self.value).
@return: representation of error
@rtype: string
"""
return str(self.value)
class InvenioConnector(object):
"""
Creates an connector to a server running Invenio
"""
def __init__(self, url=None, user="", password="", login_method="Local", local_import_path="invenio"):
"""
Initialize a new instance of the server at given URL.
If the server happens to be running on the local machine, the
access will be done directly using the Python APIs. In that case
you can choose from which base path to import the necessary file
specifying the local_import_path parameter.
@param url: the url to which this instance will be connected.
Defaults to CFG_SITE_URL, if available.
@type url: string
@param user: the optional username for interacting with the Invenio
instance in an authenticated way.
@type user: string
@param password: the corresponding password.
@type password: string
@param login_method: the name of the login method the Invenio instance
is expecting for this user (in case there is more than one).
@type login_method: string
@param local_import_path: the base path from which the connector should
try to load the local connector, if available. Eg "invenio" will
lead to "import invenio.dbquery"
@type local_import_path: string
@raise InvenioConnectorAuthError: if no secure URL is given for authentication
@raise InvenioConnectorServerError: if no URL is given on a machine without Invenio installed
"""
if url == None and LOCAL_SITE_URLS != None:
self.server_url = LOCAL_SITE_URLS[0] # Default to CFG_SITE_URL
elif url == None:
raise InvenioConnectorServerError("You do not seem to have Invenio installed and no remote URL is given")
else:
self.server_url = url
self.local = LOCAL_SITE_URLS and self.server_url in LOCAL_SITE_URLS
self.cached_queries = {}
self.cached_records = {}
self.cached_baskets = {}
self.user = user
self.password = password
self.login_method = login_method
self.browser = None
if self.user:
if not self.server_url.startswith('https://'):
raise InvenioConnectorAuthError("You have to use a secure URL (HTTPS) to login")
if MECHANIZE_AVAILABLE:
self._init_browser()
self._check_credentials()
else:
self.user = None
raise InvenioConnectorAuthError("The Python module Mechanize (and ClientForm" \
" if Mechanize version < 0.2.0) must" \
" be installed to perform authenticated requests.")
def _init_browser(self):
"""
Ovveride this method with the appropriate way to prepare a logged in
browser.
"""
self.browser = mechanize.Browser()
self.browser.set_handle_robots(False)
self.browser.open(self.server_url + "/youraccount/login")
self.browser.select_form(nr=0)
self.browser['p_un'] = self.user
self.browser['p_pw'] = self.password
# Set login_method to be writable
self.browser.form.find_control('login_method').readonly = False
self.browser['login_method'] = self.login_method
self.browser.submit()
def _check_credentials(self):
if not 'youraccount/logout' in self.browser.response().read():
raise InvenioConnectorAuthError("It was not possible to successfully login with the provided credentials")
def search(self, read_cache=True, **kwparams):
"""
Returns records corresponding to the given search query.
See docstring of invenio.search_engine.perform_request_search()
for an overview of available parameters.
@raise InvenioConnectorAuthError: if authentication fails
"""
parse_results = False
of = kwparams.get('of', "")
if of == "":
parse_results = True
of = "xm"
kwparams['of'] = of
params = urllib.urlencode(kwparams, doseq=1)
# Are we running locally? If so, better directly access the
# search engine directly
if self.local and of != 't':
# See if user tries to search any restricted collection
c = kwparams.get('c', "")
if c != "":
if type(c) is list:
colls = c
else:
colls = [c]
for collection in colls:
if collection_restricted_p(collection):
if self.user:
self._check_credentials()
continue
raise InvenioConnectorAuthError("You are trying to search a restricted collection. Please authenticate yourself.\n")
kwparams['of'] = 'id'
results = perform_request_search(**kwparams)
if of.lower() != 'id':
results = format_records(results, of)
else:
if not self.cached_queries.has_key(params + str(parse_results)) or not read_cache:
if self.user:
results = self.browser.open(self.server_url + "/search?" + params)
else:
results = urllib2.urlopen(self.server_url + "/search?" + params)
if 'youraccount/login' in results.geturl():
# Current user not able to search collection
raise InvenioConnectorAuthError("You are trying to search a restricted collection. Please authenticate yourself.\n")
else:
return self.cached_queries[params + str(parse_results)]
if parse_results:
# FIXME: we should not try to parse if results is string
parsed_records = self._parse_results(results, self.cached_records)
self.cached_queries[params + str(parse_results)] = parsed_records
return parsed_records
else:
# pylint: disable=E1103
# The whole point of the following code is to make sure we can
# handle two types of variable.
try:
res = results.read()
except AttributeError:
res = results
# pylint: enable=E1103
if of == "id":
try:
if type(res) is str:
# Transform to list
res = [int(recid.strip()) for recid in \
res.strip("[]").split(",") if recid.strip() != ""]
res.reverse()
except (ValueError, AttributeError):
res = []
self.cached_queries[params + str(parse_results)] = res
return self.cached_queries[params + str(parse_results)]
def search_with_retry(self, sleeptime=3.0, retrycount=3, **params):
"""
This function performs a search given a dictionary of search(..)
parameters. It accounts for server timeouts as necessary and
will retry some number of times.
@param sleeptime: number of seconds to sleep between retries
@type sleeptime: float
@param retrycount: number of times to retry given search
@type retrycount: int
@param params: search parameters
@type params: **kwds
@rtype: list
@return: returns records in given format
"""
results = []
count = 0
while count < retrycount:
try:
results = self.search(**params)
break
except urllib2.URLError:
sys.stderr.write("Timeout while searching...Retrying\n")
time.sleep(sleeptime)
count += 1
else:
sys.stderr.write("Aborting search after %d attempts.\n" % (retrycount,))
return results
def search_similar_records(self, recid):
"""
Returns the records similar to the given one
"""
return self.search(p="recid:" + str(recid), rm="wrd")
def search_records_cited_by(self, recid):
"""
Returns records cited by the given one
"""
return self.search(p="recid:" + str(recid), rm="citation")
def get_records_from_basket(self, bskid, group_basket=False, read_cache=True):
"""
Returns the records from the (public) basket with given bskid
"""
if not self.cached_baskets.has_key(bskid) or not read_cache:
if self.user:
if group_basket:
group_basket = '&category=G'
else:
group_basket = ''
results = self.browser.open(self.server_url + \
"/yourbaskets/display?of=xm&bskid=" + str(bskid) + group_basket)
else:
results = urllib2.urlopen(self.server_url + \
"/yourbaskets/display_public?of=xm&bskid=" + str(bskid))
else:
return self.cached_baskets[bskid]
parsed_records = self._parse_results(results, self.cached_records)
self.cached_baskets[bskid] = parsed_records
return parsed_records
def get_record(self, recid, read_cache=True):
"""
Returns the record with given recid
"""
if self.cached_records.has_key(recid) or not read_cache:
return self.cached_records[recid]
else:
return self.search(p="recid:" + str(recid))
def upload_marcxml(self, marcxml, mode):
"""
Uploads a record to the server
Parameters:
marcxml - *str* the XML to upload.
mode - *str* the mode to use for the upload.
"-i" insert new records
"-r" replace existing records
"-c" correct fields of records
"-a" append fields to records
"-ir" insert record or replace if it exists
"""
if mode not in ["-i", "-r", "-c", "-a", "-ir"]:
raise NameError, "Incorrect mode " + str(mode)
# Are we running locally? If so, submit directly
if self.local:
(code, marcxml_filepath) = tempfile.mkstemp(prefix="upload_%s" % \
time.strftime("%Y%m%d_%H%M%S_",
time.localtime()))
marcxml_file_d = os.fdopen(code, "w")
marcxml_file_d.write(marcxml)
marcxml_file_d.close()
return task_low_level_submission("bibupload", "", mode, marcxml_filepath)
else:
params = urllib.urlencode({'file': marcxml,
'mode': mode})
## We don't use self.browser as batchuploader is protected by IP
opener = urllib2.build_opener()
- opener.addheaders = [('User-agent', 'invenio_webupload')]
+ opener.addheaders = [('User-Agent', CFG_USER_AGENT)]
return opener.open(self.server_url + "/batchuploader/robotupload", params,)
def _parse_results(self, results, cached_records):
"""
Parses the given results (in MARCXML format).
The given "cached_records" list is a pool of
already existing parsed records (in order to
avoid keeping several times the same records in memory)
"""
parser = xml.sax.make_parser()
handler = RecordsHandler(cached_records)
parser.setContentHandler(handler)
parser.parse(results)
return handler.records
class Record(dict):
"""
Represents a Invenio record
"""
def __init__(self, recid=None, marcxml=None, server_url=None):
#dict.__init__(self)
self.recid = recid
self.marcxml = ""
if marcxml is not None:
self.marcxml = marcxml
#self.record = {}
self.server_url = server_url
def __setitem__(self, item, value):
tag, ind1, ind2, subcode = decompose_code(item)
if subcode is not None:
#if not dict.has_key(self, tag + ind1 + ind2):
# dict.__setitem__(self, tag + ind1 + ind2, [])
dict.__setitem__(self, tag + ind1 + ind2, [{subcode: [value]}])
else:
dict.__setitem__(self, tag + ind1 + ind2, value)
def __getitem__(self, item):
tag, ind1, ind2, subcode = decompose_code(item)
datafields = dict.__getitem__(self, tag + ind1 + ind2)
if subcode is not None:
subfields = []
for datafield in datafields:
if datafield.has_key(subcode):
subfields.extend(datafield[subcode])
return subfields
else:
return datafields
def __contains__(self, item):
return dict.__contains__(item)
def __repr__(self):
return "Record(" + dict.__repr__(self) + ")"
def __str__(self):
return self.marcxml
def export(self, of="marcxml"):
"""
Returns the record in chosen format
"""
return self.marcxml
def url(self):
"""
Returns the URL to this record.
Returns None if not known
"""
if self.server_url is not None and \
self.recid is not None:
return self.server_url + "/"+ CFG_SITE_RECORD +"/" + str(self.recid)
else:
return None
class _SGMLParserFactory(mechanize.DefaultFactory):
"""
Black magic to be able to interact with CERN SSO forms.
"""
def __init__(self, i_want_broken_xhtml_support=False):
if OLD_MECHANIZE_VERSION:
forms_factory = mechanize.FormsFactory(
form_parser_class=ClientForm.XHTMLCompatibleFormParser)
else:
forms_factory = mechanize.FormsFactory(
form_parser_class=mechanize.XHTMLCompatibleFormParser)
mechanize.Factory.__init__(
self,
forms_factory=forms_factory,
links_factory=mechanize.LinksFactory(),
title_factory=mechanize.TitleFactory(),
response_type_finder=mechanize._html.ResponseTypeFinder(
allow_xhtml=i_want_broken_xhtml_support),
)
class CDSInvenioConnector(InvenioConnector):
def __init__(self, user="", password="", local_import_path="invenio"):
"""
This is a specialized InvenioConnector class suitable to connecte
to the CERN Document Server (CDS), which uses centralize SSO.
"""
super(CDSInvenioConnector, self).__init__(CFG_CDS_URL, user, password, local_import_path=local_import_path)
def _init_browser(self):
"""
@note: update this everytime the CERN SSO login form is refactored.
"""
self.browser = mechanize.Browser(factory=_SGMLParserFactory(i_want_broken_xhtml_support=True))
self.browser.set_handle_robots(False)
self.browser.open(self.server_url)
self.browser.follow_link(text_regex="login")
self.browser.select_form(nr=0)
self.browser.form['ctl00$ContentPlaceHolder1$txtFormsLogin'] = self.user
self.browser.form['ctl00$ContentPlaceHolder1$txtFormsPassword'] = self.password
self.browser.submit()
self.browser.select_form(nr=0)
self.browser.submit()
class RecordsHandler(xml.sax.handler.ContentHandler):
"MARCXML Parser"
def __init__(self, records):
"""
Parameters:
records - *dict* a dictionary with an already existing cache of records
"""
self.cached_records = records
self.records = []
self.in_record = False
self.in_controlfield = False
self.in_datafield = False
self.in_subfield = False
self.cur_tag = None
self.cur_subfield = None
self.cur_controlfield = None
self.cur_datafield = None
self.cur_record = None
self.recid = 0
self.buffer = ""
self.counts = 0
def startElement(self, name, attributes):
if name == "record":
self.cur_record = Record()
self.in_record = True
elif name == "controlfield":
tag = attributes["tag"]
self.cur_datafield = ""
self.cur_tag = tag
self.cur_controlfield = []
if not self.cur_record.has_key(tag):
self.cur_record[tag] = self.cur_controlfield
self.in_controlfield = True
elif name == "datafield":
tag = attributes["tag"]
self.cur_tag = tag
ind1 = attributes["ind1"]
if ind1 == " ": ind1 = "_"
ind2 = attributes["ind2"]
if ind2 == " ": ind2 = "_"
if not self.cur_record.has_key(tag + ind1 + ind2):
self.cur_record[tag + ind1 + ind2] = []
self.cur_datafield = {}
self.cur_record[tag + ind1 + ind2].append(self.cur_datafield)
self.in_datafield = True
elif name == "subfield":
subcode = attributes["code"]
if not self.cur_datafield.has_key(subcode):
self.cur_subfield = []
self.cur_datafield[subcode] = self.cur_subfield
else:
self.cur_subfield = self.cur_datafield[subcode]
self.in_subfield = True
def characters(self, data):
if self.in_subfield:
self.buffer += data
elif self.in_controlfield:
self.buffer += data
elif "Search-Engine-Total-Number-Of-Results:" in data:
print data
match_obj = re.search("\d+", data)
if match_obj:
print int(match_obj.group())
self.counts = int(match_obj.group())
def endElement(self, name):
if name == "record":
self.in_record = False
elif name == "controlfield":
if self.cur_tag == "001":
self.recid = int(self.buffer)
if self.cached_records.has_key(self.recid):
# Record has already been parsed, no need to add
pass
else:
# Add record to the global cache
self.cached_records[self.recid] = self.cur_record
# Add record to the ordered list of results
self.records.append(self.cached_records[self.recid])
self.cur_controlfield.append(self.buffer)
self.in_controlfield = False
self.buffer = ""
elif name == "datafield":
self.in_datafield = False
elif name == "subfield":
self.in_subfield = False
self.cur_subfield.append(self.buffer)
self.buffer = ""
def decompose_code(code):
"""
Decomposes a MARC "code" into tag, ind1, ind2, subcode
"""
code = "%-6s" % code
ind1 = code[3:4]
if ind1 == " ": ind1 = "_"
ind2 = code[4:5]
if ind2 == " ": ind2 = "_"
subcode = code[5:6]
if subcode == " ": subcode = None
return (code[0:3], ind1, ind2, subcode)
diff --git a/modules/miscutil/lib/inveniocfg.py b/modules/miscutil/lib/inveniocfg.py
index f310a9f7b..1d355a7e2 100644
--- a/modules/miscutil/lib/inveniocfg.py
+++ b/modules/miscutil/lib/inveniocfg.py
@@ -1,1350 +1,1361 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Invenio configuration and administration CLI tool.
Usage: inveniocfg [options]
General options:
-h, --help print this help
-V, --version print version number
Options to finish your installation:
--create-apache-conf create Apache configuration files
--create-tables create DB tables for Invenio
--load-webstat-conf load the WebStat configuration
--drop-tables drop DB tables of Invenio
--check-openoffice check for correctly set up of openoffice temporary directory
Options to set up and test a demo site:
--create-demo-site create demo site
--load-demo-records load demo records
--remove-demo-records remove demo records, keeping demo site
--drop-demo-site drop demo site configurations too
--run-unit-tests run unit test suite (needs demo site)
--run-regression-tests run regression test suite (needs demo site)
--run-web-tests run web tests in a browser (needs demo site, Firefox, Selenium IDE)
Options to update config files in situ:
--update-all perform all the update options
--update-config-py update config.py file from invenio.conf file
--update-dbquery-py update dbquery.py with DB credentials from invenio.conf
--update-dbexec update dbexec with DB credentials from invenio.conf
--update-bibconvert-tpl update bibconvert templates with CFG_SITE_URL from invenio.conf
--update-web-tests update web test cases with CFG_SITE_URL from invenio.conf
Options to update DB tables:
--reset-all perform all the reset options
--reset-sitename reset tables to take account of new CFG_SITE_NAME*
--reset-siteadminemail reset tables to take account of new CFG_SITE_ADMIN_EMAIL
--reset-fieldnames reset tables to take account of new I18N names from PO files
--reset-recstruct-cache reset record structure cache according to CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE
Options to help the work:
--list print names and values of all options from conf files
--get get value of a given option from conf files
--conf-dir path to directory where invenio*.conf files are [optional]
--detect-system-details print system details such as Apache/Python/MySQL versions
"""
__revision__ = "$Id$"
from ConfigParser import ConfigParser
import os
import re
import shutil
import socket
import sys
def print_usage():
"""Print help."""
print __doc__
def print_version():
"""Print version information."""
print __revision__
def convert_conf_option(option_name, option_value):
"""
Convert conf option into Python config.py line, converting
values to ints or strings as appropriate.
"""
## 1) convert option name to uppercase:
option_name = option_name.upper()
## 2) convert option value to int or string:
if option_name in ['CFG_BIBUPLOAD_REFERENCE_TAG',
'CFG_BIBUPLOAD_EXTERNAL_SYSNO_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_TAG',
'CFG_BIBUPLOAD_EXTERNAL_OAIID_PROVENANCE_TAG',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS',]:
# some options are supposed be string even when they look like
# numeric
option_value = '"' + option_value + '"'
else:
try:
option_value = int(option_value)
except ValueError:
option_value = '"' + option_value + '"'
## 3a) special cases: chars regexps
if option_name in ['CFG_BIBINDEX_CHARS_ALPHANUMERIC_SEPARATORS',
'CFG_BIBINDEX_CHARS_PUNCTUATION']:
option_value = 'r"[' + option_value[1:-1] + ']"'
## 3abis) special cases: real regexps
- if option_name in ['CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES']:
+ if option_name in ['CFG_BIBINDEX_PERFORM_OCR_ON_DOCNAMES',
+ 'CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS']:
option_value = 'r"' + option_value[1:-1] + '"'
## 3b) special cases: True, False, None
if option_value in ['"True"', '"False"', '"None"']:
option_value = option_value[1:-1]
- ## 3c) special cases: dicts
+ ## 3c) special cases: dicts and real pythonic lists
if option_name in ['CFG_WEBSEARCH_FIELDS_CONVERT',
'CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS',
'CFG_SITE_EMERGENCY_EMAIL_ADDRESSES',
'CFG_BIBMATCH_FUZZY_WORDLIMITS',
'CFG_BIBMATCH_QUERY_TEMPLATES',
'CFG_WEBSEARCH_SYNONYM_KBRS',
'CFG_BIBINDEX_SYNONYM_KBRS',
'CFG_WEBCOMMENT_EMAIL_REPLIES_TO',
'CFG_WEBCOMMENT_RESTRICTION_DATAFIELD',
'CFG_WEBCOMMENT_ROUND_DATAFIELD',
'CFG_BIBUPLOAD_FFT_ALLOWED_EXTERNAL_URLS',
'CFG_BIBSCHED_NODE_TASKS',
'CFG_BIBEDIT_EXTEND_RECORD_WITH_COLLECTION_TEMPLATE',
'CFG_OAI_METADATA_FORMATS',
'CFG_WEBSUBMIT_DESIRED_CONVERSIONS',
- 'CFG_BIBDOCFILE_BEST_FORMATS_TO_EXTRACT_TEXT_FROM',]:
+ 'CFG_BIBDOCFILE_BEST_FORMATS_TO_EXTRACT_TEXT_FROM',
+ 'CFG_WEB_API_KEY_ALLOWED_URL',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_MISC',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_DOCTYPES',
+ 'CFG_WEBSUBMIT_DOCUMENT_FILE_MANAGER_RESTRICTIONS']:
option_value = option_value[1:-1]
## 3cbis) very special cases: dicts with backward compatible string
if option_name in ['CFG_BIBINDEX_SPLASH_PAGES']:
if option_value.startswith('"{') and option_value.endswith('}"'):
option_value = option_value[1:-1]
else:
option_value = """{%s: ".*"}""" % option_value
## 3d) special cases: comma-separated lists
if option_name in ['CFG_SITE_LANGS',
'CFG_WEBSUBMIT_ADDITIONAL_KNOWN_FILE_EXTENSIONS',
'CFG_WEBSEARCH_USE_MATHJAX_FOR_FORMATS',
'CFG_BIBUPLOAD_STRONG_TAGS',
'CFG_BIBFORMAT_HIDDEN_TAGS',
'CFG_BIBSCHED_GC_TASKS_TO_REMOVE',
'CFG_BIBSCHED_GC_TASKS_TO_ARCHIVE',
'CFG_BIBUPLOAD_FFT_ALLOWED_LOCAL_PATHS',
'CFG_BIBUPLOAD_CONTROLLED_PROVENANCE_TAGS',
'CFG_BIBUPLOAD_DELETE_FORMATS',
'CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES',
'CFG_WEBSTYLE_HTTP_STATUS_ALERT_LIST',
'CFG_WEBSEARCH_RSS_I18N_COLLECTIONS',
'CFG_BATCHUPLOADER_FILENAME_MATCHING_POLICY',
- 'CFG_BATCHUPLOADER_WEB_ROBOT_AGENT',
'CFG_BIBAUTHORID_EXTERNAL_CLAIMED_RECORDS_KEY',
'CFG_BIBCIRCULATION_ITEM_STATUS_OPTIONAL',
'CFG_PLOTEXTRACTOR_DISALLOWED_TEX',
'CFG_OAI_FRIENDS',
'CFG_WEBSTYLE_REVERSE_PROXY_IPS',
'CFG_BIBEDIT_AUTOCOMPLETE_INSTITUTIONS_FIELDS',
'CFG_BIBFORMAT_DISABLE_I18N_FOR_CACHED_FORMATS',
'CFG_BIBFORMAT_HIDDEN_FILE_FORMATS',]:
out = "["
for elem in option_value[1:-1].split(","):
if elem:
if option_name in ['CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES']:
# 3d1) integer values
out += "%i, " % int(elem)
else:
# 3d2) string values
out += "'%s', " % elem
out += "]"
option_value = out
## 3e) special cases: multiline
if option_name == 'CFG_OAI_IDENTIFY_DESCRIPTION':
# make triple quotes
option_value = '""' + option_value + '""'
## 3f) ignore some options:
if option_name.startswith('CFG_SITE_NAME_INTL'):
# treated elsewhere
return
## 3g) special cases: float
if option_name in ['CFG_BIBDOCFILE_MD5_CHECK_PROBABILITY',
'CFG_BIBMATCH_LOCAL_SLEEPTIME',
'CFG_BIBMATCH_REMOTE_SLEEPTIME',
'CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT',
'CFG_BIBMATCH_FUZZY_MATCH_VALIDATION_LIMIT']:
option_value = float(option_value[1:-1])
## 3h) special cases: bibmatch validation list
if option_name in ['CFG_BIBMATCH_MATCH_VALIDATION_RULESETS']:
option_value = option_value[1:-1]
- ## 4) finally, return output line:
+ ## 4a) dropped variables
+ if option_name in ['CFG_BATCHUPLOADER_WEB_ROBOT_AGENT']:
+ print >> sys.stderr, ("""ERROR: CFG_BATCHUPLOADER_WEB_ROBOT_AGENT has been dropped in favour of
+CFG_BATCHUPLOADER_WEB_ROBOT_AGENTS.
+Please, update your invenio-local.conf file accordingly.""")
+ option_value = option_value[1:-1]
+
+ ## 5) finally, return output line:
return '%s = %s' % (option_name, option_value)
def cli_cmd_update_config_py(conf):
"""
Update new config.py from conf options, keeping previous
config.py in a backup copy.
"""
print ">>> Going to update config.py..."
## location where config.py is:
configpyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'config.py'
## backup current config.py file:
if os.path.exists(configpyfile):
shutil.copy(configpyfile, configpyfile + '.OLD')
## here we go:
fdesc = open(configpyfile, 'w')
## generate preamble:
fdesc.write("# -*- coding: utf-8 -*-\n")
fdesc.write("# DO NOT EDIT THIS FILE! IT WAS AUTOMATICALLY GENERATED\n")
fdesc.write("# FROM INVENIO.CONF BY EXECUTING:\n")
fdesc.write("# " + " ".join(sys.argv) + "\n")
## special treatment for CFG_SITE_NAME_INTL options:
fdesc.write("CFG_SITE_NAME_INTL = {}\n")
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
fdesc.write("CFG_SITE_NAME_INTL['%s'] = \"%s\"\n" % (lang, conf.get("Invenio",
"CFG_SITE_NAME_INTL_" + lang)))
## special treatment for CFG_SITE_SECURE_URL that may be empty, in
## which case it should be put equal to CFG_SITE_URL:
if not conf.get("Invenio", "CFG_SITE_SECURE_URL"):
conf.set("Invenio", "CFG_SITE_SECURE_URL",
conf.get("Invenio", "CFG_SITE_URL"))
## process all the options normally:
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
if not option.startswith('CFG_DATABASE_'):
# put all options except for db credentials into config.py
line_out = convert_conf_option(option, conf.get(section, option))
if line_out:
fdesc.write(line_out + "\n")
## FIXME: special treatment for experimental variables
## CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES and CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE
## (not offering them in invenio.conf since they will be refactored)
fdesc.write("CFG_WEBSEARCH_DEFAULT_SEARCH_INTERFACE = 0\n")
fdesc.write("CFG_WEBSEARCH_ENABLED_SEARCH_INTERFACES = [0, 1,]\n")
## generate postamble:
fdesc.write("")
fdesc.write("# END OF GENERATED FILE")
## we are done:
fdesc.close()
print "You may want to restart Apache now."
print ">>> config.py updated successfully."
def cli_cmd_update_dbquery_py(conf):
"""
Update lib/dbquery.py file with DB parameters read from conf file.
Note: this edits dbquery.py in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbquery.py..."
## location where dbquery.py is:
dbquerypyfile = conf.get("Invenio", "CFG_PYLIBDIR") + \
os.sep + 'invenio' + os.sep + 'dbquery.py'
## backup current dbquery.py file:
if os.path.exists(dbquerypyfile):
shutil.copy(dbquerypyfile, dbquerypyfile + '.OLD')
## replace db parameters:
out = ''
for line in open(dbquerypyfile, 'r').readlines():
match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS|SLAVE)(\s*=\s*)\'.*\'$', line)
if match:
dbparam = 'CFG_DATABASE_' + match.group(1)
out += "%s%s'%s'\n" % (dbparam, match.group(2),
conf.get('Invenio', dbparam))
else:
out += line
fdesc = open(dbquerypyfile, 'w')
fdesc.write(out)
fdesc.close()
print "You may want to restart Apache now."
print ">>> dbquery.py updated successfully."
def cli_cmd_update_dbexec(conf):
"""
Update bin/dbexec file with DB parameters read from conf file.
Note: this edits dbexec in situ, taking a backup first.
Use only when you know what you are doing.
"""
print ">>> Going to update dbexec..."
## location where dbexec is:
dbexecfile = conf.get("Invenio", "CFG_BINDIR") + \
os.sep + 'dbexec'
## backup current dbexec file:
if os.path.exists(dbexecfile):
shutil.copy(dbexecfile, dbexecfile + '.OLD')
## replace db parameters via sed:
out = ''
for line in open(dbexecfile, 'r').readlines():
match = re.search(r'^CFG_DATABASE_(HOST|PORT|NAME|USER|PASS|SLAVE)(\s*=\s*)\'.*\'$', line)
if match:
dbparam = 'CFG_DATABASE_' + match.group(1)
out += "%s%s'%s'\n" % (dbparam, match.group(2),
conf.get("Invenio", dbparam))
else:
out += line
fdesc = open(dbexecfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> dbexec updated successfully."
def cli_cmd_update_bibconvert_tpl(conf):
"""
Update bibconvert/config/*.tpl files looking for 856
http://.../CFG_SITE_RECORD lines, replacing URL with CFG_SITE_URL taken
from conf file. Note: this edits tpl files in situ, taking a
backup first. Use only when you know what you are doing.
"""
print ">>> Going to update bibconvert templates..."
## location where bibconvert/config/*.tpl are:
tpldir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'bibconvert' + os.sep + 'config'
## find all *.tpl files:
for tplfilename in os.listdir(tpldir):
if tplfilename.endswith(".tpl"):
## change tpl file:
tplfile = tpldir + os.sep + tplfilename
shutil.copy(tplfile, tplfile + '.OLD')
out = ''
for line in open(tplfile, 'r').readlines():
match = re.search(r'^(.*)http://.*?/%s/(.*)$' % conf.get("Invenio", 'CFG_SITE_RECORD'), line)
if match:
out += "%s%s/%s/%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_SITE_URL'),
conf.get("Invenio", 'CFG_SITE_RECORD'),
match.group(2))
else:
out += line
fdesc = open(tplfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> bibconvert templates updated successfully."
def cli_cmd_update_web_tests(conf):
"""
Update web test cases lib/webtest/test_*.html looking for
http://.+?[>> Going to update web tests..."
## location where test_*.html files are:
testdir = conf.get("Invenio", 'CFG_PREFIX') + os.sep + \
'lib' + os.sep + 'webtest' + os.sep + 'invenio'
## find all test_*.html files:
for testfilename in os.listdir(testdir):
if testfilename.startswith("test_") and \
testfilename.endswith(".html"):
## change test file:
testfile = testdir + os.sep + testfilename
shutil.copy(testfile, testfile + '.OLD')
out = ''
for line in open(testfile, 'r').readlines():
match = re.search(r'^(.* )http://.+?([)/opt/invenio(.*)$', line)
if match:
out += "%s%s%s\n" % (match.group(1),
conf.get("Invenio", 'CFG_PREFIX'),
match.group(2))
else:
out += line
fdesc = open(testfile, 'w')
fdesc.write(out)
fdesc.close()
print ">>> web tests updated successfully."
def cli_cmd_reset_sitename(conf):
"""
Reset collection-related tables with new CFG_SITE_NAME and
CFG_SITE_NAME_INTL* read from conf files.
"""
print ">>> Going to reset CFG_SITE_NAME and CFG_SITE_NAME_INTL..."
from invenio.dbquery import run_sql, IntegrityError
# reset CFG_SITE_NAME:
sitename = conf.get("Invenio", "CFG_SITE_NAME")
try:
run_sql("""INSERT INTO collection (id, name, dbquery, reclist) VALUES
(1,%s,NULL,NULL)""", (sitename,))
except IntegrityError:
run_sql("""UPDATE collection SET name=%s WHERE id=1""", (sitename,))
# reset CFG_SITE_NAME_INTL:
for lang in conf.get("Invenio", "CFG_SITE_LANGS").split(","):
sitename_lang = conf.get("Invenio", "CFG_SITE_NAME_INTL_" + lang)
try:
run_sql("""INSERT INTO collectionname (id_collection, ln, type, value) VALUES
(%s,%s,%s,%s)""", (1, lang, 'ln', sitename_lang))
except IntegrityError:
run_sql("""UPDATE collectionname SET value=%s
WHERE ln=%s AND id_collection=1 AND type='ln'""",
(sitename_lang, lang))
print "You may want to restart Apache now."
print ">>> CFG_SITE_NAME and CFG_SITE_NAME_INTL* reset successfully."
def cli_cmd_reset_recstruct_cache(conf):
"""If CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE is changed, this function
will adapt the database to either store or not store the recstruct
format."""
from invenio.intbitset import intbitset
from invenio.dbquery import run_sql, serialize_via_marshal
from invenio.search_engine import get_record
from invenio.bibsched import server_pid, pidfile
enable_recstruct_cache = conf.get("Invenio", "CFG_BIBUPLOAD_SERIALIZE_RECORD_STRUCTURE")
enable_recstruct_cache = enable_recstruct_cache in ('True', '1')
pid = server_pid(ping_the_process=False)
if pid:
print >> sys.stderr, "ERROR: bibsched seems to run with pid %d, according to %s." % (pid, pidfile)
print >> sys.stderr, " Please stop bibsched before running this procedure."
sys.exit(1)
if enable_recstruct_cache:
print ">>> Searching records which need recstruct cache resetting; this may take a while..."
all_recids = intbitset(run_sql("SELECT id FROM bibrec"))
good_recids = intbitset(run_sql("SELECT bibrec.id FROM bibrec JOIN bibfmt ON bibrec.id = bibfmt.id_bibrec WHERE format='recstruct' AND modification_date < last_updated"))
recids = all_recids - good_recids
print ">>> Generating recstruct cache..."
tot = len(recids)
count = 0
for recid in recids:
value = serialize_via_marshal(get_record(recid))
run_sql("DELETE FROM bibfmt WHERE id_bibrec=%s AND format='recstruct'", (recid, ))
run_sql("INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, 'recstruct', NOW(), %s)", (recid, value))
count += 1
if count % 1000 == 0:
print " ... done records %s/%s" % (count, tot)
if count % 1000 != 0:
print " ... done records %s/%s" % (count, tot)
print ">>> recstruct cache generated successfully."
else:
print ">>> Cleaning recstruct cache..."
run_sql("DELETE FROM bibfmt WHERE format='recstruct'")
def cli_cmd_reset_siteadminemail(conf):
"""
Reset user-related tables with new CFG_SITE_ADMIN_EMAIL read from conf files.
"""
print ">>> Going to reset CFG_SITE_ADMIN_EMAIL..."
from invenio.dbquery import run_sql
siteadminemail = conf.get("Invenio", "CFG_SITE_ADMIN_EMAIL")
run_sql("DELETE FROM user WHERE id=1")
run_sql("""INSERT INTO user (id, email, password, note, nickname) VALUES
(1, %s, AES_ENCRYPT(email, ''), 1, 'admin')""",
(siteadminemail,))
print "You may want to restart Apache now."
print ">>> CFG_SITE_ADMIN_EMAIL reset successfully."
def cli_cmd_reset_fieldnames(conf):
"""
Reset I18N field names such as author, title, etc and other I18N
ranking method names such as word similarity. Their translations
are taken from the PO files.
"""
print ">>> Going to reset I18N field names..."
from invenio.messages import gettext_set_language, language_list_long
from invenio.dbquery import run_sql, IntegrityError
## get field id and name list:
field_id_name_list = run_sql("SELECT id, name FROM field")
## get rankmethod id and name list:
rankmethod_id_name_list = run_sql("SELECT id, name FROM rnkMETHOD")
## update names for every language:
for lang, dummy in language_list_long():
_ = gettext_set_language(lang)
## this list is put here in order for PO system to pick names
## suitable for translation
field_name_names = {"any field": _("any field"),
"title": _("title"),
"author": _("author"),
"abstract": _("abstract"),
"keyword": _("keyword"),
"report number": _("report number"),
"subject": _("subject"),
"reference": _("reference"),
"fulltext": _("fulltext"),
"collection": _("collection"),
"division": _("division"),
"year": _("year"),
"journal": _("journal"),
"experiment": _("experiment"),
"record ID": _("record ID")}
## update I18N names for every language:
for (field_id, field_name) in field_id_name_list:
if field_name_names.has_key(field_name):
try:
run_sql("""INSERT INTO fieldname (id_field,ln,type,value) VALUES
(%s,%s,%s,%s)""", (field_id, lang, 'ln',
field_name_names[field_name]))
except IntegrityError:
run_sql("""UPDATE fieldname SET value=%s
WHERE id_field=%s AND ln=%s AND type=%s""",
(field_name_names[field_name], field_id, lang, 'ln',))
## ditto for rank methods:
rankmethod_name_names = {"wrd": _("word similarity"),
"demo_jif": _("journal impact factor"),
"citation": _("times cited"),
"citerank_citation_t": _("time-decay cite count"),
"citerank_pagerank_c": _("all-time-best cite rank"),
"citerank_pagerank_t": _("time-decay cite rank"),}
for (rankmethod_id, rankmethod_name) in rankmethod_id_name_list:
if rankmethod_name_names.has_key(rankmethod_name):
try:
run_sql("""INSERT INTO rnkMETHODNAME (id_rnkMETHOD,ln,type,value) VALUES
(%s,%s,%s,%s)""", (rankmethod_id, lang, 'ln',
rankmethod_name_names[rankmethod_name]))
except IntegrityError:
run_sql("""UPDATE rnkMETHODNAME SET value=%s
WHERE id_rnkMETHOD=%s AND ln=%s AND type=%s""",
(rankmethod_name_names[rankmethod_name], rankmethod_id, lang, 'ln',))
print ">>> I18N field names reset successfully."
def cli_check_openoffice(conf):
"""
If OpenOffice.org integration is enabled, checks whether the system is
properly configured.
"""
from invenio.bibtask import check_running_process_user
from invenio.websubmit_file_converter import can_unoconv, get_file_converter_logger
logger = get_file_converter_logger()
for handler in logger.handlers:
logger.removeHandler(handler)
check_running_process_user()
print ">>> Checking if Libre/OpenOffice.org is correctly integrated...",
sys.stdout.flush()
if can_unoconv(True):
print "ok"
else:
sys.exit(1)
def test_db_connection():
"""
Test DB connection, and if fails, advise user how to set it up.
Useful to be called during table creation.
"""
print "Testing DB connection...",
from invenio.textutils import wrap_text_in_a_box
from invenio.dbquery import run_sql, Error
## first, test connection to the DB server:
try:
run_sql("SHOW TABLES")
except Error, err:
from invenio.dbquery import CFG_DATABASE_HOST, CFG_DATABASE_PORT, \
CFG_DATABASE_NAME, CFG_DATABASE_USER, CFG_DATABASE_PASS
print wrap_text_in_a_box("""\
DATABASE CONNECTIVITY ERROR %(errno)d: %(errmsg)s.\n
Perhaps you need to set up database and connection rights?
If yes, then please login as MySQL admin user and run the
following commands now:
$ mysql -h %(dbhost)s -P %(dbport)s -u root -p mysql
mysql> CREATE DATABASE %(dbname)s DEFAULT CHARACTER SET utf8;
mysql> GRANT ALL PRIVILEGES ON %(dbname)s.*
TO %(dbuser)s@%(webhost)s IDENTIFIED BY '%(dbpass)s';
mysql> QUIT
The values printed above were detected from your
configuration. If they are not right, then please edit your
invenio-local.conf file and rerun 'inveniocfg --update-all' first.
If the problem is of different nature, then please inspect
the above error message and fix the problem before continuing.""" % \
{'errno': err.args[0],
'errmsg': err.args[1],
'dbname': CFG_DATABASE_NAME,
'dbhost': CFG_DATABASE_HOST,
'dbport': CFG_DATABASE_PORT,
'dbuser': CFG_DATABASE_USER,
'dbpass': CFG_DATABASE_PASS,
'webhost': CFG_DATABASE_HOST == 'localhost' and 'localhost' or os.popen('hostname -f', 'r').read().strip(),
})
sys.exit(1)
print "ok"
## second, test insert/select of a Unicode string to detect
## possible Python/MySQL/MySQLdb mis-setup:
print "Testing Python/MySQL/MySQLdb UTF-8 chain...",
try:
beta_in_utf8 = "β" # Greek beta in UTF-8 is 0xCEB2
run_sql("CREATE TEMPORARY TABLE test__invenio__utf8 (x char(1), y varbinary(2)) DEFAULT CHARACTER SET utf8")
run_sql("INSERT INTO test__invenio__utf8 (x, y) VALUES (%s, %s)", (beta_in_utf8, beta_in_utf8))
res = run_sql("SELECT x,y,HEX(x),HEX(y),LENGTH(x),LENGTH(y),CHAR_LENGTH(x),CHAR_LENGTH(y) FROM test__invenio__utf8")
assert res[0] == ('\xce\xb2', '\xce\xb2', 'CEB2', 'CEB2', 2L, 2L, 1L, 2L)
run_sql("DROP TEMPORARY TABLE test__invenio__utf8")
except Exception, err:
print wrap_text_in_a_box("""\
DATABASE RELATED ERROR %s\n
A problem was detected with the UTF-8 treatment in the chain
between the Python application, the MySQLdb connector, and
the MySQL database. You may perhaps have installed older
versions of some prerequisite packages?\n
Please check the INSTALL file and please fix this problem
before continuing.""" % err)
sys.exit(1)
print "ok"
def cli_cmd_create_tables(conf):
"""Create and fill Invenio DB tables. Useful for the installation process."""
print ">>> Going to create and fill tables..."
from invenio.config import CFG_PREFIX
test_db_connection()
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabcreate.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/dbexec < %s/lib/sql/invenio/tabfill.sql" % (CFG_PREFIX, CFG_PREFIX)]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
for cmd in ["%s/bin/webaccessadmin -u admin -c -a" % CFG_PREFIX]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables created and filled successfully."
def cli_cmd_load_webstat_conf(conf):
print ">>> Going to load WebStat config..."
from invenio.config import CFG_PREFIX
cmd = "%s/bin/webstatadmin --load-config" % CFG_PREFIX
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> WebStat config load successfully."
def cli_cmd_drop_tables(conf):
"""Drop Invenio DB tables. Useful for the uninstallation process."""
print ">>> Going to drop tables..."
from invenio.config import CFG_PREFIX
from invenio.textutils import wrap_text_in_a_box, wait_for_user
from invenio.webstat import destroy_customevents
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your database tables!"""))
msg = destroy_customevents()
if msg:
print msg
cmd = "%s/bin/dbexec < %s/lib/sql/invenio/tabdrop.sql" % (CFG_PREFIX, CFG_PREFIX)
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Tables dropped successfully."
def cli_cmd_create_demo_site(conf):
"""Create demo site. Useful for testing purposes."""
print ">>> Going to create demo site..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
run_sql("TRUNCATE schTASK")
run_sql("TRUNCATE session")
run_sql("DELETE FROM user WHERE email=''")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/democfgdata.sql" % \
(CFG_PREFIX, CFG_PREFIX),]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
cli_cmd_reset_fieldnames(conf) # needed for I18N demo ranking method names
for cmd in ["%s/bin/webaccessadmin -u admin -c -r -D" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,
"%s/bin/bibsort -u admin --load-config" % CFG_PREFIX,
"%s/bin/bibsort 2" % CFG_PREFIX, ]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo site created successfully."
def cli_cmd_load_demo_records(conf):
"""Load demo records. Useful for testing purposes."""
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
print ">>> Going to load demo records..."
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/bibupload -u admin -i %s/var/tmp/demobibdata.xml" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/bibupload 1" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --with-ocr --recid 97" % CFG_PREFIX,
"%s/bin/bibdocfile --textify --all" % CFG_PREFIX,
"%s/bin/bibindex -u admin" % CFG_PREFIX,
"%s/bin/bibindex 2" % CFG_PREFIX,
"%s/bin/bibreformat -u admin -o HB" % CFG_PREFIX,
"%s/bin/bibreformat 3" % CFG_PREFIX,
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 4" % CFG_PREFIX,
"%s/bin/bibrank -u admin" % CFG_PREFIX,
"%s/bin/bibrank 5" % CFG_PREFIX,
"%s/bin/bibsort -u admin -R" % CFG_PREFIX,
"%s/bin/bibsort 6" % CFG_PREFIX,
"%s/bin/oairepositoryupdater -u admin" % CFG_PREFIX,
"%s/bin/oairepositoryupdater 7" % CFG_PREFIX,
"%s/bin/bibupload 8" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records loaded successfully."
def cli_cmd_remove_demo_records(conf):
"""Remove demo records. Useful when you are finished testing."""
print ">>> Going to remove demo records..."
from invenio.config import CFG_PREFIX
from invenio.dbquery import run_sql
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your records and documents!"""))
if os.path.exists(CFG_PREFIX + os.sep + 'var' + os.sep + 'data'):
shutil.rmtree(CFG_PREFIX + os.sep + 'var' + os.sep + 'data')
run_sql("TRUNCATE schTASK")
for cmd in ["%s/bin/dbexec < %s/lib/sql/invenio/tabbibclean.sql" % (CFG_PREFIX, CFG_PREFIX),
"%s/bin/webcoll -u admin" % CFG_PREFIX,
"%s/bin/webcoll 1" % CFG_PREFIX,]:
if os.system(cmd):
print "ERROR: failed execution of", cmd
sys.exit(1)
print ">>> Demo records removed successfully."
def cli_cmd_drop_demo_site(conf):
"""Drop demo site completely. Useful when you are finished testing."""
print ">>> Going to drop demo site..."
from invenio.textutils import wrap_text_in_a_box, wait_for_user
wait_for_user(wrap_text_in_a_box("""WARNING: You are going to destroy
your site and documents!"""))
cli_cmd_drop_tables(conf)
cli_cmd_create_tables(conf)
cli_cmd_remove_demo_records(conf)
print ">>> Demo site dropped successfully."
def cli_cmd_run_unit_tests(conf):
"""Run unit tests, usually on the working demo site."""
from invenio.testutils import build_and_run_unit_test_suite
if not build_and_run_unit_test_suite():
sys.exit(1)
def cli_cmd_run_regression_tests(conf):
"""Run regression tests, usually on the working demo site."""
from invenio.testutils import build_and_run_regression_test_suite
if not build_and_run_regression_test_suite():
sys.exit(1)
def cli_cmd_run_web_tests(conf):
"""Run web tests in a browser. Requires Firefox with Selenium."""
from invenio.testutils import build_and_run_web_test_suite
if not build_and_run_web_test_suite():
sys.exit(1)
def _detect_ip_address():
"""Detect IP address of this computer. Useful for creating Apache
vhost conf snippet on RHEL like machines.
@return: IP address, or '*' if cannot detect
@rtype: string
@note: creates socket for real in order to detect real IP address,
not the loopback one.
"""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('invenio-software.org', 0))
return s.getsockname()[0]
except:
return '*'
def cli_cmd_create_apache_conf(conf):
"""
Create Apache conf files for this site, keeping previous
files in a backup copy.
"""
print ">>> Going to create Apache conf files..."
from invenio.textutils import wrap_text_in_a_box
from invenio.access_control_config import CFG_EXTERNAL_AUTH_USING_SSO
apache_conf_dir = conf.get("Invenio", 'CFG_ETCDIR') + \
os.sep + 'apache'
## Preparation of XSendFile directive
xsendfile_directive_needed = int(conf.get("Invenio", 'CFG_BIBDOCFILE_USE_XSENDFILE')) != 0
if xsendfile_directive_needed:
xsendfile_directive = "XSendFile On\n"
else:
xsendfile_directive = "#XSendFile On\n"
for path in (conf.get('Invenio', 'CFG_WEBSUBMIT_FILEDIR'), # BibDocFile
conf.get('Invenio', 'CFG_WEBDIR'),
conf.get('Invenio', 'CFG_WEBSUBMIT_STORAGEDIR'), # WebSubmit
conf.get('Invenio', 'CFG_TMPDIR'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'tmp', 'attachfile'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'comments'),
os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'data', 'baskets', 'comments'),
'/tmp'): # BibExport
if xsendfile_directive_needed:
xsendfile_directive += ' XSendFilePath %s\n' % path
else:
xsendfile_directive += ' #XSendFilePath %s\n' % path
xsendfile_directive = xsendfile_directive.strip()
## Preparation of deflate directive
deflate_directive_needed = int(conf.get("Invenio", 'CFG_WEBSTYLE_HTTP_USE_COMPRESSION')) != 0
if deflate_directive_needed:
deflate_directive = r"""
## Configuration snippet taken from:
##
SetOutputFilter DEFLATE
# Netscape 4.x has some problems...
BrowserMatch ^Mozilla/4 gzip-only-text/html
# Netscape 4.06-4.08 have some more problems
BrowserMatch ^Mozilla/4\.0[678] no-gzip
# MSIE masquerades as Netscape, but it is fine
# BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
# NOTE: Due to a bug in mod_setenvif up to Apache 2.0.48
# the above regex won't work. You can use the following
# workaround to get the desired effect:
BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
# Don't compress images
SetEnvIfNoCase Request_URI \
\.(?:gif|jpe?g|png)$ no-gzip dont-vary
# Make sure proxies don't deliver the wrong content
Header append Vary User-Agent env=!dont-vary
"""
else:
deflate_directive = ""
if CFG_EXTERNAL_AUTH_USING_SSO:
shibboleth_directive = r"""
SSLRequireSSL # The modules only work using HTTPS
AuthType shibboleth
ShibRequireSession On
ShibRequireAll On
ShibExportAssertion Off
require valid-user
"""
else:
shibboleth_directive = ""
## Apache vhost conf file is distro specific, so analyze needs:
# Gentoo (and generic defaults):
listen_directive_needed = True
ssl_pem_directive_needed = False
ssl_pem_path = '/etc/apache2/ssl/apache.pem'
ssl_crt_path = '/etc/apache2/ssl/server.crt'
ssl_key_path = '/etc/apache2/ssl/server.key'
vhost_ip_address_needed = False
wsgi_socket_directive_needed = False
# Debian:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'debian_version'):
listen_directive_needed = False
ssl_pem_directive_needed = True
# RHEL/SLC:
if os.path.exists(os.path.sep + 'etc' + os.path.sep + 'redhat-release'):
listen_directive_needed = False
ssl_crt_path = '/etc/pki/tls/certs/localhost.crt'
ssl_key_path = '/etc/pki/tls/private/localhost.key'
vhost_ip_address_needed = True
wsgi_socket_directive_needed = True
# maybe we are using non-standard ports?
vhost_site_url = conf.get('Invenio', 'CFG_SITE_URL').replace("http://", "")
if vhost_site_url.startswith("https://"):
## The installation is configured to require HTTPS for any connection
vhost_site_url = vhost_site_url.replace("https://", "")
vhost_site_url_port = '80'
vhost_site_secure_url = conf.get('Invenio', 'CFG_SITE_SECURE_URL').replace("https://", "")
vhost_site_secure_url_port = '443'
if ':' in vhost_site_url:
vhost_site_url, vhost_site_url_port = vhost_site_url.split(':', 1)
if ':' in vhost_site_secure_url:
vhost_site_secure_url, vhost_site_secure_url_port = vhost_site_secure_url.split(':', 1)
if vhost_site_url_port != '80' or vhost_site_secure_url_port != '443':
listen_directive_needed = True
## OK, let's create Apache vhost files:
if not os.path.exists(apache_conf_dir):
os.mkdir(apache_conf_dir)
apache_vhost_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost.conf'
apache_vhost_ssl_file = apache_conf_dir + os.sep + \
'invenio-apache-vhost-ssl.conf'
apache_vhost_body = """\
AddDefaultCharset UTF-8
ServerSignature Off
ServerTokens Prod
NameVirtualHost %(vhost_ip_address)s:%(vhost_site_url_port)s
%(listen_directive)s
%(wsgi_socket_directive)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
DocumentRoot %(webdir)s
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
ErrorLog %(logdir)s/apache.err
LogLevel warn
LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing
CustomLog %(logdir)s/apache.log combined_with_timing
DirectoryIndex index.en.html index.html
Alias /img/ %(webdir)s/img/
Alias /css/ %(webdir)s/css/
Alias /js/ %(webdir)s/js/
Alias /flash/ %(webdir)s/flash/
Alias /css/ %(webdir)s/css/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /ckeditor/ %(webdir)s/ckeditor/
Alias /mediaelement/ %(webdir)s/mediaelement/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
WSGIDaemonProcess invenio processes=5 threads=1 display-name=%%{GROUP} inactivity-timeout=3600 maximum-requests=10000
WSGIImportScript %(wsgidir)s/invenio.wsgi process-group=invenio application-group=%%{GLOBAL}
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
%(xsendfile_directive)s
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
%(deflate_directive)s
""" % {'vhost_site_url_port': vhost_site_url_port,
'servername': vhost_site_url,
'serveralias': vhost_site_url.split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'),
'wsgidir': os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address() or '*',
'listen_directive': listen_directive_needed and 'Listen ' + vhost_site_url_port or \
'#Listen ' + vhost_site_url_port,
'wsgi_socket_directive': (wsgi_socket_directive_needed and \
'WSGISocketPrefix ' or '#WSGISocketPrefix ') + \
conf.get('Invenio', 'CFG_PREFIX') + os.sep + 'var' + os.sep + 'run',
'xsendfile_directive' : xsendfile_directive,
'deflate_directive': deflate_directive,
}
apache_vhost_ssl_body = """\
ServerSignature Off
ServerTokens Prod
%(listen_directive)s
NameVirtualHost %(vhost_ip_address)s:%(vhost_site_secure_url_port)s
%(ssl_pem_directive)s
%(ssl_crt_directive)s
%(ssl_key_directive)s
WSGIRestrictStdout Off
deny from all
deny from all
ServerName %(servername)s
ServerAlias %(serveralias)s
ServerAdmin %(serveradmin)s
SSLEngine on
DocumentRoot %(webdir)s
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
ErrorLog %(logdir)s/apache-ssl.err
LogLevel warn
LogFormat "%%h %%l %%u %%t \\"%%r\\" %%>s %%b \\"%%{Referer}i\\" \\"%%{User-agent}i\\" %%D" combined_with_timing
CustomLog %(logdir)s/apache-ssl.log combined_with_timing
DirectoryIndex index.en.html index.html
Alias /img/ %(webdir)s/img/
Alias /css/ %(webdir)s/css/
Alias /js/ %(webdir)s/js/
Alias /flash/ %(webdir)s/flash/
Alias /css/ %(webdir)s/css/
Alias /export/ %(webdir)s/export/
Alias /MathJax/ %(webdir)s/MathJax/
Alias /jsCalendar/ %(webdir)s/jsCalendar/
Alias /ckeditor/ %(webdir)s/ckeditor/
Alias /mediaelement/ %(webdir)s/mediaelement/
AliasMatch /sitemap-(.*) %(webdir)s/sitemap-$1
Alias /robots.txt %(webdir)s/robots.txt
Alias /favicon.ico %(webdir)s/favicon.ico
RedirectMatch /sslredirect/(.*) http://$1
WSGIScriptAlias / %(wsgidir)s/invenio.wsgi
WSGIPassAuthorization On
%(xsendfile_directive)s
WSGIProcessGroup invenio
WSGIApplicationGroup %%{GLOBAL}
Options FollowSymLinks MultiViews
AllowOverride None
Order allow,deny
Allow from all
%(deflate_directive)s
%(shibboleth_directive)s
""" % {'vhost_site_secure_url_port': vhost_site_secure_url_port,
'servername': vhost_site_secure_url,
'serveralias': vhost_site_secure_url.split('.')[0],
'serveradmin': conf.get('Invenio', 'CFG_SITE_ADMIN_EMAIL'),
'webdir': conf.get('Invenio', 'CFG_WEBDIR'),
'logdir': conf.get('Invenio', 'CFG_LOGDIR'),
'libdir' : conf.get('Invenio', 'CFG_PYLIBDIR'),
'wsgidir' : os.path.join(conf.get('Invenio', 'CFG_PREFIX'), 'var', 'www-wsgi'),
'vhost_ip_address': vhost_ip_address_needed and _detect_ip_address() or '*',
'listen_directive' : listen_directive_needed and 'Listen ' + vhost_site_secure_url_port or \
'#Listen ' + vhost_site_secure_url_port,
'ssl_pem_directive': ssl_pem_directive_needed and \
'SSLCertificateFile %s' % ssl_pem_path or \
'#SSLCertificateFile %s' % ssl_pem_path,
'ssl_crt_directive': ssl_pem_directive_needed and \
'#SSLCertificateFile %s' % ssl_crt_path or \
'SSLCertificateFile %s' % ssl_crt_path,
'ssl_key_directive': ssl_pem_directive_needed and \
'#SSLCertificateKeyFile %s' % ssl_key_path or \
'SSLCertificateKeyFile %s' % ssl_key_path,
'xsendfile_directive' : xsendfile_directive,
'deflate_directive': deflate_directive,
'shibboleth_directive': shibboleth_directive,
}
# write HTTP vhost snippet:
if os.path.exists(apache_vhost_file):
shutil.copy(apache_vhost_file,
apache_vhost_file + '.OLD')
fdesc = open(apache_vhost_file, 'w')
fdesc.write(apache_vhost_body)
fdesc.close()
print
print "Created file", apache_vhost_file
# write HTTPS vhost snippet:
vhost_ssl_created = False
if conf.get('Invenio', 'CFG_SITE_SECURE_URL').startswith("https://"):
if os.path.exists(apache_vhost_ssl_file):
shutil.copy(apache_vhost_ssl_file,
apache_vhost_ssl_file + '.OLD')
fdesc = open(apache_vhost_ssl_file, 'w')
fdesc.write(apache_vhost_ssl_body)
fdesc.close()
vhost_ssl_created = True
print "Created file", apache_vhost_ssl_file
print wrap_text_in_a_box("""\
Apache virtual host configuration file(s) for your Invenio site
was(were) created. Please check created file(s) and activate virtual
host(s). For example, you can put the following include statements in
your httpd.conf:\n
Include %s
%s
Please see the INSTALL file for more details.
""" % (apache_vhost_file, (vhost_ssl_created and 'Include ' or '#Include ') + apache_vhost_ssl_file))
print ">>> Apache conf files created."
def cli_cmd_get(conf, varname):
"""
Return value of VARNAME read from CONF files. Useful for
third-party programs to access values of conf options such as
CFG_PREFIX. Return None if VARNAME is not found.
"""
# do not pay attention to upper/lower case:
varname = varname.lower()
# do not pay attention to section names yet:
all_options = {}
for section in conf.sections():
for option in conf.options(section):
all_options[option] = conf.get(section, option)
return all_options.get(varname, None)
def cli_cmd_list(conf):
"""
Print a list of all conf options and values from CONF.
"""
sections = conf.sections()
sections.sort()
for section in sections:
options = conf.options(section)
options.sort()
for option in options:
print option.upper(), '=', conf.get(section, option)
def _grep_version_from_executable(path_to_exec, version_regexp):
"""
Try to detect a program version by digging into its binary
PATH_TO_EXEC and looking for VERSION_REGEXP. Return program
version as a string. Return empty string if not succeeded.
"""
from invenio.shellutils import run_shell_command
exec_version = ""
if os.path.exists(path_to_exec):
dummy1, cmd2_out, dummy2 = run_shell_command("strings %s | grep %s",
(path_to_exec, version_regexp))
if cmd2_out:
for cmd2_out_line in cmd2_out.split("\n"):
if len(cmd2_out_line) > len(exec_version):
# the longest the better
exec_version = cmd2_out_line
return exec_version
def detect_apache_version():
"""
Try to detect Apache version by localizing httpd or apache
executables and grepping inside binaries. Return list of all
found Apache versions and paths. (For a given executable, the
returned format is 'apache_version [apache_path]'.) Return empty
list if no success.
"""
from invenio.shellutils import run_shell_command
out = []
dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache")
for apache in cmd_out.split("\n"):
apache_version = _grep_version_from_executable(apache, '^Apache\/')
if apache_version:
out.append("%s [%s]" % (apache_version, apache))
return out
def cli_cmd_detect_system_details(conf):
"""
Detect and print system details such as Apache/Python/MySQL
versions etc. Useful for debugging problems on various OS.
"""
import MySQLdb
print ">>> Going to detect system details..."
print "* Hostname: " + socket.gethostname()
print "* Invenio version: " + conf.get("Invenio", "CFG_VERSION")
print "* Python version: " + sys.version.replace("\n", " ")
print "* Apache version: " + ";\n ".join(detect_apache_version())
print "* MySQLdb version: " + MySQLdb.__version__
try:
from invenio.dbquery import run_sql
print "* MySQL version:"
for key, val in run_sql("SHOW VARIABLES LIKE 'version%'") + \
run_sql("SHOW VARIABLES LIKE 'charact%'") + \
run_sql("SHOW VARIABLES LIKE 'collat%'"):
if False:
print " - %s: %s" % (key, val)
elif key in ['version',
'character_set_client',
'character_set_connection',
'character_set_database',
'character_set_results',
'character_set_server',
'character_set_system',
'collation_connection',
'collation_database',
'collation_server']:
print " - %s: %s" % (key, val)
except ImportError:
print "* ERROR: cannot import dbquery"
print ">>> System details detected successfully."
def main():
"""Main entry point."""
conf = ConfigParser()
if '--help' in sys.argv or \
'-h' in sys.argv:
print_usage()
elif '--version' in sys.argv or \
'-V' in sys.argv:
print_version()
else:
confdir = None
if '--conf-dir' in sys.argv:
try:
confdir = sys.argv[sys.argv.index('--conf-dir') + 1]
except IndexError:
pass # missing --conf-dir argument value
if not os.path.exists(confdir):
print "ERROR: bad or missing --conf-dir option value."
sys.exit(1)
else:
## try to detect path to conf dir (relative to this bin dir):
confdir = re.sub(r'/bin$', '/etc', sys.path[0])
## read conf files:
for conffile in [confdir + os.sep + 'invenio.conf',
confdir + os.sep + 'invenio-autotools.conf',
confdir + os.sep + 'invenio-local.conf',]:
if os.path.exists(conffile):
conf.read(conffile)
else:
if not conffile.endswith("invenio-local.conf"):
# invenio-local.conf is optional, otherwise stop
print "ERROR: Badly guessed conf file location", conffile
print "(Please use --conf-dir option.)"
sys.exit(1)
## decide what to do:
done = False
for opt_idx in range(0, len(sys.argv)):
opt = sys.argv[opt_idx]
if opt == '--conf-dir':
# already treated before, so skip silently:
pass
elif opt == '--get':
try:
varname = sys.argv[opt_idx + 1]
except IndexError:
print "ERROR: bad or missing --get option value."
sys.exit(1)
if varname.startswith('-'):
print "ERROR: bad or missing --get option value."
sys.exit(1)
varvalue = cli_cmd_get(conf, varname)
if varvalue is not None:
print varvalue
else:
sys.exit(1)
done = True
elif opt == '--list':
cli_cmd_list(conf)
done = True
elif opt == '--detect-system-details':
cli_cmd_detect_system_details(conf)
done = True
elif opt == '--create-tables':
cli_cmd_create_tables(conf)
done = True
elif opt == '--load-webstat-conf':
cli_cmd_load_webstat_conf(conf)
done = True
elif opt == '--drop-tables':
cli_cmd_drop_tables(conf)
done = True
elif opt == '--check-openoffice':
cli_check_openoffice(conf)
done = True
elif opt == '--create-demo-site':
cli_cmd_create_demo_site(conf)
done = True
elif opt == '--load-demo-records':
cli_cmd_load_demo_records(conf)
done = True
elif opt == '--remove-demo-records':
cli_cmd_remove_demo_records(conf)
done = True
elif opt == '--drop-demo-site':
cli_cmd_drop_demo_site(conf)
done = True
elif opt == '--run-unit-tests':
cli_cmd_run_unit_tests(conf)
done = True
elif opt == '--run-regression-tests':
cli_cmd_run_regression_tests(conf)
done = True
elif opt == '--run-web-tests':
cli_cmd_run_web_tests(conf)
done = True
elif opt == '--update-all':
cli_cmd_update_config_py(conf)
cli_cmd_update_dbquery_py(conf)
cli_cmd_update_dbexec(conf)
cli_cmd_update_bibconvert_tpl(conf)
cli_cmd_update_web_tests(conf)
done = True
elif opt == '--update-config-py':
cli_cmd_update_config_py(conf)
done = True
elif opt == '--update-dbquery-py':
cli_cmd_update_dbquery_py(conf)
done = True
elif opt == '--update-dbexec':
cli_cmd_update_dbexec(conf)
done = True
elif opt == '--update-bibconvert-tpl':
cli_cmd_update_bibconvert_tpl(conf)
done = True
elif opt == '--update-web-tests':
cli_cmd_update_web_tests(conf)
done = True
elif opt == '--reset-all':
cli_cmd_reset_sitename(conf)
cli_cmd_reset_siteadminemail(conf)
cli_cmd_reset_fieldnames(conf)
cli_cmd_reset_recstruct_cache(conf)
done = True
elif opt == '--reset-sitename':
cli_cmd_reset_sitename(conf)
done = True
elif opt == '--reset-siteadminemail':
cli_cmd_reset_siteadminemail(conf)
done = True
elif opt == '--reset-fieldnames':
cli_cmd_reset_fieldnames(conf)
done = True
elif opt == '--reset-recstruct-cache':
cli_cmd_reset_recstruct_cache(conf)
done = True
elif opt == '--create-apache-conf':
cli_cmd_create_apache_conf(conf)
done = True
elif opt.startswith("-") and opt != '--yes-i-know':
print "ERROR: unknown option", opt
sys.exit(1)
if not done:
print """ERROR: Please specify a command. Please see '--help'."""
sys.exit(1)
if __name__ == '__main__':
main()
diff --git a/modules/miscutil/lib/plotextractor_getter.py b/modules/miscutil/lib/plotextractor_getter.py
index f86f0cac1..9221bf5d4 100644
--- a/modules/miscutil/lib/plotextractor_getter.py
+++ b/modules/miscutil/lib/plotextractor_getter.py
@@ -1,630 +1,633 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
import urllib2, time, os, sys, re
from invenio.config import CFG_TMPDIR, \
CFG_PLOTEXTRACTOR_SOURCE_BASE_URL, \
CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER, \
CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER, \
CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT
from invenio.plotextractor_config import CFG_PLOTEXTRACTOR_DESY_BASE, \
CFG_PLOTEXTRACTOR_DESY_PIECE
from invenio.search_engine import get_record
from invenio.bibrecord import record_get_field_instances, \
field_get_subfield_values
from invenio.shellutils import run_shell_command
from invenio.plotextractor_output_utils import write_message
+from invenio.urlutils import make_invenio_opener
+
+PLOTEXTRACTOR_OPENER = make_invenio_opener('plotextractor')
PDF_EXTENSION = '.pdf'
ARXIV_HEADER = 'arXiv:'
HEP_EX = ['hep-ex/', 9405, ARXIV_HEADER + 'hep-ex_'] # experimental
# a note about hep-ex: the hep-ex papers from 9403 nad 9404 are stored
# in arXiv's servers as hep-ph
HEP_LAT = ['hep-lat/', 9107, ARXIV_HEADER + 'hep-lat_'] # lattice
HEP_PH = ['hep-ph/', 9203, ARXIV_HEADER + 'hep-ph_'] # phenomenology
HEP_TH = ['hep-th/', 9108, ARXIV_HEADER + 'hep-th_'] # theory
HEP_AREAS = [HEP_EX, HEP_LAT, HEP_PH, HEP_TH]
URL = 0
BEGIN_YEAR_MONTH_INDEX = 1
AREA_STRING_INDEX = 2
URL_MOVE = int('0704')
CENTURY_END = int('9912')
CENTURY_BEGIN = int('0001')
ARBITRARY_FROM_DATE = int('9101')
FIX_FOR_YEAR_END = 88
current_yearmonth = int(('%02d%02d' % (time.localtime().tm_year, \
time.localtime().tm_mon))[2:])
"""
each of the areas of hep began in a different year and month.
beginning in 0704, i.e. April 2007, arXiv moved its URLS from
ARXIV_BASE + E_PRINT + HEP_AREA + <>
to
ARXIV_BASE + E_PRINT + <>
the papers for a given month are numbered between yymm.0001 and yymm.9999
after the URL move, and before that they are between yymm001 and yymm999
"""
help_param = 'help'
dir_param = 'dir'
from_param = 'from'
from_index_param = 'fromindex'
ref_file_param = 'reffile'
single_param = 'single'
param_abbrs = 'hd:f:i:r:s:'
params = [help_param, dir_param + '=', from_param + '=', from_index_param + '=',
ref_file_param + '=', single_param + '=']
def harvest(to_dir, from_date, from_index):
"""
Calls upon arXiv using URLS as described above in order to grab
all the tarballs from HEP areas.
@param: dir (string): the directory where everything that gets
downloaded will sit
@param: from_date (int): the date from which we would like to harvest,
in YYMM format
@param: from_index (int): the index where we want to begin our harvest
in YYMM. i.e. we want to start with the 345th record in 1002.
@output: TONS OF .tar.gz FILES FROM ARXIV
@return: (none)
"""
global current_yearmonth
if from_date > current_yearmonth and from_date < ARBITRARY_FROM_DATE:
write_message('Please choose a from date that is not in the future!')
sys.exit(1)
if from_date % 100 > 12:
write_message('Please choose a from date in the form YYMM')
sys.exit(1)
if from_date >= ARBITRARY_FROM_DATE or from_date < URL_MOVE:
for area in HEP_AREAS:
yearmonthindex = area[BEGIN_YEAR_MONTH_INDEX]
# nasty casing!
# I find this particularly horrid because we have to wrap dates..
# i.e. although 9901 is more than 0001, we might want things in
# 0001 and not from 9901
if from_date < current_yearmonth:
# we want to start in the new century; skip the while below
yearmonthindex = CENTURY_END
elif from_date < CENTURY_END:
yearmonthindex = from_date
# grab stuff from between 92 and 99
old_URL_harvest(yearmonthindex, CENTURY_END, to_dir, area)
yearmonthindex = CENTURY_BEGIN
# more nasty casing
if from_date < URL_MOVE:
# that means we want to start sometime before the weird
# url change
yearmonthindex = from_date
elif from_date > URL_MOVE and from_date < ARBITRARY_FROM_DATE:
# we don't want to start yet
yearmonthindex = URL_MOVE
# grab stuff from between 00 and 07
old_URL_harvest(yearmonthindex, URL_MOVE, to_dir, area)
# also after the URL move, there was no distinction between
# papers from different areas. hence, outside the for loop
# even more nasty casing!
if from_date < current_yearmonth and from_date > URL_MOVE:
# we want to start someplace after the URL move and before now
yearmonthindex = from_date
else:
yearmonthindex = URL_MOVE
# grab stuff from between 07 and today
new_URL_harvest(yearmonthindex, from_index, to_dir)
def make_single_directory(to_dir, dirname):
"""
Makes a subdirectory for the arXiv record we are working with and
returns its exact location.
@param: to_dir (string): the name of the directory we want to make it
in
@param: dirname (string): the name of the directory we want to create
@output: a new directory called dirname located in to_dir
@return: the absolute path to the new directory
"""
new_dir = os.path.join(to_dir, dirname)
if not os.path.isdir(new_dir):
try:
os.mkdir(new_dir)
except OSError:
write_message('Failed to make new dir...')
return to_dir
return new_dir
def make_useful_directories(yearmonthindex, to_dir):
"""
Builds up the hierarchical filestructure for saving these things
in a useful way.
@param: yearmonthindex (int): YYMM
@param: to_dir (string): where we want to build the directories from
@return month_dir (string): the new directory we are going to put
stuff in
"""
year = yearmonthindex / 100
if year >= (ARBITRARY_FROM_DATE / 100):
year = '19%02d' % year
else:
year = '20%02d' % year
month = '%02d' % (yearmonthindex % 100)
year_dir = os.path.join(to_dir, year)
if not os.path.isdir(year_dir):
os.mkdir(year_dir)
month_dir = os.path.join(year_dir, month)
if not os.path.isdir(month_dir):
os.mkdir(month_dir)
return month_dir
def get_list_of_all_matching_files(basedir, filetypes):
"""
This function uses the os module in order tocrawl
through the directory tree rooted at basedir and find all the files
therein that include filetype in their 'file' output. Returns a list
of absolute paths to all files.
@param: basedir (string): the directory where we want to start crawling
@param: filetypes ([string, string]): something that will be contained in
the output of running 'file' on the types of files we're looking for
@return: file_paths ([string, string, ...]): a list of full paths to
the files that we discovered
"""
file_paths = []
for dirpath, dummy0, filenames in os.walk(basedir):
for filename in filenames:
full_path = os.path.join(dirpath, filename)
dummy1, cmd_out, dummy2 = run_shell_command('file %s', (full_path,))
for filetype in filetypes:
if cmd_out.find(filetype) > -1:
file_paths.append(full_path)
return file_paths
def tarballs_by_recids(recids, sdir):
"""
Take a string representing one recid or several and get the associated
tarballs for those ids.
@param: recids (string): the record id or ids
@param: sdir (string): where the tarballs should live
@return: tarballs ([string, string, ...]): locations of tarballs
"""
list_of_ids = []
if ',' in recids:
recids = recids.split(',')
for recid in recids:
if '-' in recid:
low, high = recid.split('-')
recid = range(int(low), int(high))
list_of_ids.extend(recid)
else:
recid = int(recid)
list_of_ids.append(recid)
else:
if '-' in recids:
low, high = recid.split('-')
list_of_ids = range(int(low), int(high))
else:
list_of_ids = int(recid)
arXiv_ids = []
for recid in list_of_ids:
rec = get_record(recid)
for afieldinstance in record_get_field_instances(rec, tag='037'):
if 'arXiv' == field_get_subfield_values(afieldinstance, '9')[0]:
arXiv_id = field_get_subfield_values(afieldinstance, 'a')[0]
arXiv_ids.append(arXiv_id)
return tarballs_by_arXiv_id(arXiv_ids, sdir)
def tarballs_by_arXiv_id(arXiv_ids, sdir):
"""
Takes an list of arXiv ids and downloads their tarballs
and returns a list of the tarballs' locations.
@param: arXiv_ids ([string, string, ...]): the arXiv ids you
would like to have tarballs for
@param: sdir (string): the place to download these tarballs to
@return: tarballs ([string, ...]): a list of the tarballs downloaded
"""
tarballs = []
for arXiv_id in arXiv_ids:
if 'arXiv' not in arXiv_id:
arXiv_id = 'arXiv:' + arXiv_id
tarball, dummy_pdf = harvest_single(arXiv_id, sdir, ("tarball",))
if tarball != None:
tarballs.append(tarball)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
return tarballs
def parse_and_download(infile, sdir):
"""
Read the write_messageation in the input file and download the corresponding
tarballs from arxiv.
@param: infile (string): the name of the file to parse
@param: sdir (string): where to put the downloaded tarballs
"""
tarfiles = []
tardir = os.path.join(sdir, 'tarballs')
if not os.path.isdir(tardir):
try:
os.makedirs(tardir)
except:
write_message(sys.exc_info()[0])
write_message('files will be loose, not in ' + tardir)
tardir = sdir
infile = open(infile)
for line in infile.readlines():
line = line.strip()
if line.startswith('http://'):
# hurray!
url = line
filename = url.split('/')[-1]
if not download(url, tardir, filename):
write_message(filename + ' may already exist')
write_message(sys.exc_info()[0])
filename = os.path.join(tardir, filename)
tarfiles.append(filename)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT) # be nice!
elif line.startswith('arXiv'):
tarfiles.extend(tarballs_by_arXiv_id([line.strip()], sdir))
return tarfiles
def harvest_single(single, to_dir, selection=("tarball", "pdf")):
"""
if we only want to harvest one id (arXiv or DESY), we can use this.
@param: single (string): an id from arXiv or DESY
@param: to_dir (string): where the output should be saved
@output: the PDF and source tarball (if applicable) of this single record
@return: (tarball, pdf): the location of the source tarball and PDF, None
if not found
"""
if single.find('arXiv') > -1 and 'arxiv.org' in CFG_PLOTEXTRACTOR_SOURCE_BASE_URL.lower():
id_str = re.findall('[a-zA-Z\\-]+/\\d+|\\d+\\.\\d+', single)[0]
idno = id_str.split('/')
if len(idno) > 0:
idno = idno[-1]
yymm = int(idno[:4])
yymm_dir = make_useful_directories(yymm, to_dir)
url_for_file = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + \
CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER + \
id_str
url_for_pdf = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + \
CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER + \
id_str + '.pdf' # adds '.pdf' to avoid arXiv internal redirect from arXivID to arXivID.pdf
individual_file = 'arXiv:' + id_str.replace('/', '_')
individual_dir = make_single_directory(yymm_dir, individual_file)
abs_path = os.path.join(individual_dir, individual_file)
tarball = abs_path
pdf = abs_path + '.pdf'
write_message('download ' + url_for_file + ' to ' + abs_path)
if "tarball" in selection and not download(url_for_file, individual_file, individual_dir):
write_message('download of tarball failed/skipped')
tarball = None
if "pdf" in selection and not download(url_for_pdf, individual_file + '.pdf', individual_dir):
write_message('download of pdf failed/skipped')
pdf = None
return (tarball, pdf)
elif single.find('arXiv') > -1 and CFG_PLOTEXTRACTOR_SOURCE_BASE_URL != '':
# hmm... is it a filesystem?
if CFG_PLOTEXTRACTOR_SOURCE_BASE_URL.startswith('/'):
if not os.path.exists(CFG_PLOTEXTRACTOR_SOURCE_BASE_URL):
write_message('PROBLEM WITH CFG_PLOTEXTRACTOR_SOURCE_BASE_URL: we cannot ' + \
'find this folder!')
return (None, None)
for root, files, dummy in os.walk(CFG_PLOTEXTRACTOR_SOURCE_BASE_URL):
for file_name in files:
id_no = single.replace('arXiv', '')
if file_name.find(id_no) > -1 or\
file_name.find(id_no.replace('/', '_')) > -1 or\
file_name.find(id_no.replace('_', '/')) > -1 or\
file_name.find(id_no.replace(':', '')) > -1:
# that's our file! probably.
return (os.path.join(root, file_name), None)
# well, no luck there
return (None, None)
# okay... is it... a website?
elif CFG_PLOTEXTRACTOR_SOURCE_BASE_URL.startswith('http') and "tarball" in selection:
url_for_file = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + single
individual_file = os.path.join(to_dir, single)
download(url_for_file, individual_file, to_dir)
return (individual_file, None)
# well, I don't know what to do with it
else:
write_message('unsure how to handle CFG_PLOTEXTRACTOR_SOURCE_BASE_URL. ' + \
'please fix the harvest_single function in ' + \
'miscutil/lib/plotextractor_getter.py')
return (None, None)
elif single.find('DESY') > -1 and "pdf" in selection:
# also okay!
idno = re.findall('\\d{2,4}-\\d{3}', single)[0]
year, number = idno.split('-')
if len(year) < 4:
if int(year) > 92:
year = '19' + year
else:
year = '20' + year
year_dir = make_single_directory(to_dir, year)
desy_dir = make_single_directory(year_dir, 'DESY')
individual_dir = make_single_directory(desy_dir, number)
id_no = year[2:] + '-' + number + '.pdf'
url_for_file = CFG_PLOTEXTRACTOR_DESY_BASE + year + \
CFG_PLOTEXTRACTOR_DESY_PIECE + id_no
individual_file = id_no
write_message('download ' + url_for_file + ' to ' + \
os.path.join(individual_dir, individual_file))
download(url_for_file, individual_file, individual_dir)
return (None, individual_file)
write_message('END')
return (None, None)
def src_pdf_from_marc(marc_file):
"""
Given a marc file, this function attempts to determine where to find
a pdf for that record
@param: marc_file (string): the location of a marc file we can look at
@return: pdfloc (string): the location of the downloaded PDF source file,
None if no pdf was downloaded
"""
if not os.path.exists(marc_file):
return None
marc_file = open(marc_file)
marc_text = marc_file.read()
marc_file.close()
arXiv_match = '(([a-zA-Z\\-]+/\\d{7})|(\\d{4}\\.\\d{4}))'
DESY_match = 'DESY-\\d{2,4}-\\d{3}'
pdf_loc = None
to_dir = os.path.join(CFG_TMPDIR, 'plotdata')
possible_match = re.search(arXiv_match, marc_text)
if possible_match != None:
# it's listed on arXiv, hooray!
arXiv_id = possible_match.group(0)
dummy1, pdf_loc = harvest_single(arXiv_id, to_dir, ("pdf",))
possible_match = re.search(DESY_match, marc_text)
if possible_match != None:
# it's listed on DESY, hooray!
desy_id = possible_match.group(0)
dummy1, pdf_loc = harvest_single(desy_id, to_dir, ("pdf",))
return pdf_loc
def harvest_from_file(filename, to_dir):
"""
Harvest from the file Tibor made.
Format of a single entry:
oai:arXiv.org:area/YYMMIII
or
oai:arXiv.org:YYMM.IIII
"""
ok_format = '^oai:arXiv.org:(([a-zA-Z\\-]+/\\d+)|(\\d+\\.\\d+))$'
try:
names_file = open(filename)
for arXiv_name in names_file.readlines():
if re.match(ok_format, arXiv_name) == None:
write_message('error on ' + arXiv_name + '. continuing.')
continue
harvest_single(arXiv_name, to_dir)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
except IOError:
write_message('Something is wrong with the file!')
def old_URL_harvest(from_date, to_date, to_dir, area):
"""
Grab all the PDFs and tarballs off arXiv between from_date and to_date,
where from_date and to_date are in YYMM form, and put them in their own
separate folders inside of to_dir. Folder hierarchy will be
to_dir/YYYY/MM/arXiv_id/stuff_downloaded_from_arXiv
this obeys the old URL format
@param: from_date (int): YYMM form of the date where we want to start
harvesting
@param: to_date (int): YYMM form of the date where we want to stop
harvesting
@param: to_dir (string): the base directory to put all these subdirs in
@param: area (int): the index in the HEP_AREAS array of the area we are
currently working on downloading
@output: PDFs and tarballs from arXiv in a hierarchy rooted at to_dir
@return: None
"""
yearmonthindex = from_date
while yearmonthindex < to_date:
sub_dir = make_useful_directories(yearmonthindex, to_dir)
for paperindex in range(1, 1000):
# for whatever reason, we can't count on these things to
# start at 1 (in HEP_PH from 9403 to CENTURY_END only).
# they start at frickin 202.
#if area == HEP_PH and yearmonthindex < ARBITRARY_FROM_INDEX:
# paperindex = paperindex + 201
# of note: before the URL change happened in 0704, it was
# also the case that the paper numbers only had 3 digits
next_to_harvest = '%04d%03d' % (yearmonthindex, paperindex)
arXiv_id = area[AREA_STRING_INDEX] + next_to_harvest
individual_dir = make_single_directory(sub_dir, arXiv_id)
full_url = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER + \
area[URL] + next_to_harvest
if not download(full_url, \
area[AREA_STRING_INDEX] + next_to_harvest, individual_dir):
break
full_pdf_url = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER + \
area[URL] + next_to_harvest
download(full_pdf_url, \
area[AREA_STRING_INDEX] + next_to_harvest + PDF_EXTENSION, \
individual_dir)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
if yearmonthindex % 100 == 12:
# we reached the end of the year!
yearmonthindex = yearmonthindex + FIX_FOR_YEAR_END
yearmonthindex = yearmonthindex + 1
def new_URL_harvest(from_date, from_index, to_dir):
"""
Grab all the PDFs and tarballs off arXiv between from_date and to_date,
where from_date and to_date are in YYMM form, and put them in their own
separate folders inside of to_dir. Folder hierarchy will be
to_dir/YYYY/MM/arXiv_id/stuff_downloaded_from_arXiv
this obeys the new URL format
@param: from_date (int): YYMM form of the date where we want to start
harvesting
@param: to_date (int): YYMM form of the date where we want to stop
harvesting
@param: to_dir (string): the base directory to put all these subdirs in
@output: PDFs and tarballs from arXiv in a hierarchy rooted at to_dir
@return: None
"""
global current_yearmonth
yearmonthindex = from_date
while yearmonthindex < current_yearmonth:
if yearmonthindex == from_date:
fro = from_index
else:
fro = 1
sub_dir = make_useful_directories(yearmonthindex, to_dir)
for paperindex in range(fro, 10000):
# of note: after the URL change happened in 0704, it was
# the case that paper numbers had 4 digits
next_to_harvest = '%04d.%04d' % (yearmonthindex, paperindex)
arXiv_id = ARXIV_HEADER + next_to_harvest
individual_dir = make_single_directory(sub_dir, arXiv_id)
full_url = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + CFG_PLOTEXTRACTOR_SOURCE_TARBALL_FOLDER + \
next_to_harvest
if not download(full_url, ARXIV_HEADER + next_to_harvest, \
individual_dir):
break
full_pdf_url = CFG_PLOTEXTRACTOR_SOURCE_BASE_URL + CFG_PLOTEXTRACTOR_SOURCE_PDF_FOLDER + \
next_to_harvest
download(full_pdf_url, \
ARXIV_HEADER + next_to_harvest + PDF_EXTENSION, \
individual_dir)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT) # be nice to remote server
if yearmonthindex % 100 == 12:
# we reached the end of the year!
yearmonthindex = yearmonthindex + FIX_FOR_YEAR_END
yearmonthindex = yearmonthindex + 1
def download(url, filename, to_dir):
"""
Actually does the call and download given a URL and desired output
filename.
@param: url (string): where the file lives on the interwebs
@param: filename (string): where the file should live after download
@param: to_dir (string): the dir where our new files will live
@output: a file in to_dir
@return: True on success, False on failure
"""
new_file = os.path.join(to_dir, filename)
try:
- conn = urllib2.urlopen(url)
+ conn = PLOTEXTRACTOR_OPENER.open(url)
response = conn.read()
conn.close()
new_file_fd = open(new_file, 'w')
new_file_fd.write(response)
new_file_fd.close()
write_message('Downloaded to ' + new_file)
return True
except (IOError, urllib2.URLError), e:
# this could be a permissions error, but it probably means that
# there's nothing left in that section YYMM
write_message('Error downloading from %s: \n%s\n' % (url, str(e)))
return False
diff --git a/modules/miscutil/lib/solrutils.py b/modules/miscutil/lib/solrutils.py
index e2c5ea20a..94485de00 100644
--- a/modules/miscutil/lib/solrutils.py
+++ b/modules/miscutil/lib/solrutils.py
@@ -1,60 +1,63 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Solr utilities.
"""
import urllib2
import urllib
import mimetools
from invenio import intbitset
+from invenio.urlutils import make_invenio_opener
from invenio.jsonutils import json
+SOLRUTILS_OPENER = make_invenio_opener('solrutils')
+
def solr_get_facets(bitset, solr_url):
facet_query_url = "%s/invenio_facets" % solr_url
# now use the bitset to fetch the facet data
r = urllib2.Request(facet_query_url)
data = bitset.fastdump()
boundary = mimetools.choose_boundary()
# fool solr into thinking we're uploading a file so it will read our data as a stream
contents = '--%s\r\n' % boundary
contents += 'Content-Disposition: form-data; name="bitset"; filename="bitset"\r\n'
contents += 'Content-Type: application/octet-stream\r\n'
contents += '\r\n' + data + '\r\n'
contents += '--%s--\r\n\r\n' % boundary
r.add_data(contents)
contenttype = 'multipart/form-data; boundary=%s' % boundary
r.add_unredirected_header('Content-Type', contenttype)
# post the request and get back the facets as json
- u = urllib2.urlopen(r)
+ u = SOLRUTILS_OPENER.open(r)
return json.load(u)
def solr_get_bitset(query, solr_url):
invenio_query_url = "%s/select?qt=invenio_query&q=fulltext:%s" % (solr_url, urllib.quote(query))
# query to get a bitset
bitset = intbitset.intbitset()
- u = urllib2.urlopen(invenio_query_url)
+ u = SOLRUTILS_OPENER.open(invenio_query_url)
data = u.read()
bitset.fastload(data)
return bitset
diff --git a/modules/miscutil/lib/urlutils.py b/modules/miscutil/lib/urlutils.py
index 7b87d50f5..339818b37 100644
--- a/modules/miscutil/lib/urlutils.py
+++ b/modules/miscutil/lib/urlutils.py
@@ -1,732 +1,792 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
urlutils.py -- helper functions for URL related problems such as
argument washing, redirection, etc.
"""
__revision__ = "$Id$"
import time
import base64
import hmac
import re
import sys
import os
import inspect
import urllib
-from urllib import urlencode, quote_plus, quote
+import urllib2
+from urllib import urlencode, quote_plus, quote, FancyURLopener
from urlparse import urlparse
from cgi import parse_qs, escape
from md5 import md5
try:
import BeautifulSoup
BEAUTIFUL_SOUP_IMPORTED = True
except ImportError:
BEAUTIFUL_SOUP_IMPORTED = False
try:
from hashlib import sha256, sha1
HASHLIB_IMPORTED = True
except ImportError:
HASHLIB_IMPORTED = False
from invenio import webinterface_handler_config as apache
from invenio.config import \
CFG_SITE_URL, \
CFG_WEBSTYLE_EMAIL_ADDRESSES_OBFUSCATION_MODE, \
- CFG_WEBDIR
+ CFG_WEBDIR, CFG_SITE_NAME, CFG_VERSION
def wash_url_argument(var, new_type):
"""
Wash argument into 'new_type', that can be 'list', 'str',
'int', 'tuple' or 'dict'.
If needed, the check 'type(var) is not None' should be done before
calling this function.
@param var: variable value
@param new_type: variable type, 'list', 'str', 'int', 'tuple' or 'dict'
@return: as much as possible, value var as type new_type
If var is a list, will change first element into new_type.
If int check unsuccessful, returns 0
"""
out = []
if new_type == 'list': # return lst
if isinstance(var, list):
out = var
else:
out = [var]
elif new_type == 'str': # return str
if isinstance(var, list):
try:
out = "%s" % var[0]
except:
out = ""
elif isinstance(var, str):
out = var
else:
out = "%s" % var
elif new_type == 'int': # return int
if isinstance(var, list):
try:
out = int(var[0])
except:
out = 0
elif isinstance(var, (int, long)):
out = var
elif isinstance(var, str):
try:
out = int(var)
except:
out = 0
else:
out = 0
elif new_type == 'tuple': # return tuple
if isinstance(var, tuple):
out = var
else:
out = (var, )
elif new_type == 'dict': # return dictionary
if isinstance(var, dict):
out = var
else:
out = {0: var}
return out
def redirect_to_url(req, url, redirection_type=None, norobot=False):
"""
Redirect current page to url.
@param req: request as received from apache
@param url: url to redirect to
@param redirection_type: what kind of redirection is required:
e.g.: apache.HTTP_MULTIPLE_CHOICES = 300
apache.HTTP_MOVED_PERMANENTLY = 301
apache.HTTP_MOVED_TEMPORARILY = 302
apache.HTTP_SEE_OTHER = 303
apache.HTTP_NOT_MODIFIED = 304
apache.HTTP_USE_PROXY = 305
apache.HTTP_TEMPORARY_REDIRECT = 307
The default is apache.HTTP_MOVED_TEMPORARILY
@param norobot: wether to instruct crawlers and robots such as GoogleBot
not to index past this point.
@see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3
"""
if redirection_type is None:
redirection_type = apache.HTTP_MOVED_TEMPORARILY
- req.headers_out["Location"] = url
del req.headers_out["Cache-Control"]
req.headers_out["Cache-Control"] = "no-cache, private, no-store, " \
"must-revalidate, post-check=0, pre-check=0, max-age=0"
req.headers_out["Pragma"] = "no-cache"
if norobot:
req.headers_out["X-Robots-Tag"] = "noarchive, nosnippet, noindex, nocache"
+ user_agent = req.headers_in.get('User-Agent')
+ if 'Microsoft Office Existence Discovery' in user_agent or 'ms-office' in user_agent:
+ ## HACK: this is to workaround Microsoft Office trying to be smart
+ ## when users click on URLs in Office documents that require
+ ## authentication. Office will check the validity of the URL
+ ## but will pass the browser the redirected URL rather than
+ ## the original one. This is incompatible with e.g. Shibboleth
+ ## based SSO since the referer would be lost.
+ ## See: http://support.microsoft.com/kb/899927
+ req.status = 200
+ req.content_type = 'text/html'
+ if req.method != 'HEAD':
+ req.write("""
+
+
+ Intermediate page for URLs clicked on MS Office Documents
+
+
+
+ You are going to be redirected to the desired content within 5 seconds. If the redirection does not happen automatically please click on %(url_ok)s.
+
+""" % {
+ 'url': escape(req.unparsed_uri, True),
+ 'url_ok': escape(req.unparsed_uri)
+ })
+ raise apache.SERVER_RETURN(apache.DONE)
+
+ req.headers_out["Location"] = url
+
if req.response_sent_p:
raise IOError("Cannot redirect after headers have already been sent.")
req.status = redirection_type
req.write('Please go to here
\n' % url)
raise apache.SERVER_RETURN, apache.DONE
def get_referer(req, replace_ampersands=False):
""" Return the referring page of a request.
Referer (wikipedia): Referer is a common misspelling of the word
"referrer"; so common, in fact, that it made it into the official
specification of HTTP. When visiting a webpage, the referer or
referring page is the URL of the previous webpage from which a link was
followed.
@param req: request
@param replace_ampersands: if 1, replace & by & in url
(correct HTML cannot contain & characters alone)
"""
try:
referer = req.headers_in['Referer']
if replace_ampersands == 1:
return referer.replace('&', '&')
return referer
except KeyError:
return ''
def drop_default_urlargd(urlargd, default_urlargd):
lndefault = {}
lndefault.update(default_urlargd)
## Commented out. An Invenio URL now should always specify the desired
## language, in order not to raise the automatic language discovery
## (client browser language can be used now in place of CFG_SITE_LANG)
# lndefault['ln'] = (str, CFG_SITE_LANG)
canonical = {}
canonical.update(urlargd)
for k, v in urlargd.items():
try:
d = lndefault[k]
if d[1] == v:
del canonical[k]
except KeyError:
pass
return canonical
def make_canonical_urlargd(urlargd, default_urlargd):
""" Build up the query part of an URL from the arguments passed in
the 'urlargd' dictionary. 'default_urlargd' is a secondary dictionary which
contains tuples of the form (type, default value) for the query
arguments (this is the same dictionary as the one you can pass to
webinterface_handler.wash_urlargd).
When a query element has its default value, it is discarded, so
that the simplest (canonical) url query is returned.
The result contains the initial '?' if there are actual query
items remaining.
"""
canonical = drop_default_urlargd(urlargd, default_urlargd)
if canonical:
return '?' + urlencode(canonical, doseq=True).replace('&', '&')
return ''
def create_html_link(urlbase, urlargd, link_label, linkattrd=None,
escape_urlargd=True, escape_linkattrd=True):
"""Creates a W3C compliant link.
@param urlbase: base url (e.g. invenio.config.CFG_SITE_URL/search)
@param urlargd: dictionary of parameters. (e.g. p={'recid':3, 'of'='hb'})
@param link_label: text displayed in a browser (has to be already escaped)
@param linkattrd: dictionary of attributes (e.g. a={'class': 'img'})
@param escape_urlargd: boolean indicating if the function should escape
arguments (e.g. < becomes < or " becomes ")
@param escape_linkattrd: boolean indicating if the function should escape
attributes (e.g. < becomes < or " becomes ")
"""
attributes_separator = ' '
output = ''
return output
def create_html_mailto(email, subject=None, body=None, cc=None, bcc=None,
link_label="%(email)s", linkattrd=None,
escape_urlargd=True, escape_linkattrd=True,
email_obfuscation_mode=CFG_WEBSTYLE_EMAIL_ADDRESSES_OBFUSCATION_MODE):
"""Creates a W3C compliant 'mailto' link.
Encode/encrypt given email to reduce undesired automated email
harvesting when embedded in a web page.
NOTE: there is no ultimate solution to protect against email
harvesting. All have drawbacks and can more or less be
circumvented. There are other techniques to protect email
adresses. We implement the less annoying one for users.
@param email: the recipient of the email
@param subject: a default subject for the email (must not contain
line feeds)
@param body: a default body for the email
@param cc: the co-recipient(s) of the email
@param bcc: the hidden co-recpient(s) of the email
@param link_label: the label of this mailto link. String
replacement is performed on key %(email)s with
the email address if needed.
@param linkattrd: dictionary of attributes (e.g. a={'class': 'img'})
@param escape_urlargd: boolean indicating if the function should escape
arguments (e.g. < becomes < or " becomes ")
@param escape_linkattrd: boolean indicating if the function should escape
attributes (e.g. < becomes < or " becomes ")
@param email_obfuscation_mode: the protection mode. See below:
You can choose among several modes to protect emails. It is
advised to keep the default
CFG_MISCUTIL_EMAIL_HARVESTING_PROTECTION value, so that it is
possible for an admin to change the policy globally.
Available modes ([t] means "transparent" for the user):
-1: hide all emails, excepted CFG_SITE_ADMIN_EMAIL and
CFG_SITE_SUPPORT_EMAIL.
[t] 0 : no protection, email returned as is.
foo@example.com => foo@example.com
1 : basic email munging: replaces @ by [at] and . by [dot]
foo@example.com => foo [at] example [dot] com
[t] 2 : transparent name mangling: characters are replaced by
equivalent HTML entities.
foo@example.com => foo@example.com
[t] 3 : javascript insertion. Requires Javascript enabled on client side.
4 : replaces @ and . characters by gif equivalents.
foo@example.com => foo![[at]](at.gif)
example![[dot]](dot.gif)
com
"""
# TODO: implement other protection modes to encode/encript email:
#
## [t] 5 : form submission. User is redirected to a form that he can
## fills in to send the email (??Use webmessage??).
## Depending on WebAccess, ask to answer a question.
##
## [t] 6 : if user can see (controlled by WebAccess), display. Else
## ask to login to see email. If user cannot see, display
## form submission.
if linkattrd is None:
linkattrd = {}
parameters = {}
if subject:
parameters["subject"] = subject
if body:
parameters["body"] = body.replace('\r\n', '\n').replace('\n', '\r\n')
if cc:
parameters["cc"] = cc
if bcc:
parameters["bcc"] = bcc
# Preprocessing values for some modes
if email_obfuscation_mode == 1:
# Basic Munging
email = email.replace("@", " [at] ").replace(".", " [dot] ")
elif email_obfuscation_mode == 2:
# Transparent name mangling
email = string_to_numeric_char_reference(email)
if '%(email)s' in link_label:
link_label = link_label % {'email': email}
mailto_link = create_html_link('mailto:' + email, parameters,
link_label, linkattrd,
escape_urlargd, escape_linkattrd)
if email_obfuscation_mode == 0:
# Return "as is"
return mailto_link
elif email_obfuscation_mode == 1:
# Basic Munging
return mailto_link
elif email_obfuscation_mode == 2:
# Transparent name mangling
return mailto_link
elif email_obfuscation_mode == 3:
# Javascript-based
return '''''' % \
mailto_link[::-1].replace("'", "\\'")
elif email_obfuscation_mode == 4:
# GIFs-based
email = email.replace('.',
'![[dot]](%s/img/dot.gif)
' % CFG_SITE_URL)
email = email.replace('@',
'![[at]](%s/img/at.gif)
' % CFG_SITE_URL)
return email
# All other cases, including mode -1:
return ""
def string_to_numeric_char_reference(string):
"""
Encode a string to HTML-compatible numeric character reference.
Eg: encode_html_entities("abc") == 'abc'
"""
out = ""
for char in string:
out += "&#" + str(ord(char)) + ";"
return out
def create_url(urlbase, urlargd, escape_urlargd=True):
"""Creates a W3C compliant URL. Output will look like this:
'urlbase?param1=value1¶m2=value2'
@param urlbase: base url (e.g. invenio.config.CFG_SITE_URL/search)
@param urlargd: dictionary of parameters. (e.g. p={'recid':3, 'of'='hb'}
@param escape_urlargd: boolean indicating if the function should escape
arguments (e.g. < becomes < or " becomes ")
"""
separator = '&'
output = urlbase
if urlargd:
output += '?'
if escape_urlargd:
arguments = [escape(quote(str(key)), quote=True) + '=' + \
escape(quote(str(urlargd[key])), quote=True)
for key in urlargd.keys()]
else:
arguments = [str(key) + '=' + str(urlargd[key])
for key in urlargd.keys()]
output += separator.join(arguments)
return output
def same_urls_p(a, b):
""" Compare two URLs, ignoring reorganizing of query arguments """
ua = list(urlparse(a))
ub = list(urlparse(b))
ua[4] = parse_qs(ua[4])
ub[4] = parse_qs(ub[4])
return ua == ub
def urlargs_replace_text_in_arg(urlargs, regexp_argname, text_old, text_new):
"""Analyze `urlargs' (URL CGI GET query arguments in string form)
and for each occurrence of argument matching `regexp_argname'
replace every substring `text_old' by `text_new'. Return the
resulting new URL.
Used to be used for search engine's create_nearest_terms_box,
now it is not used there anymore. It is left here in case it
will become possibly useful later.
"""
out = ""
# parse URL arguments into a dictionary:
urlargsdict = parse_qs(urlargs)
## construct new URL arguments:
urlargsdictnew = {}
for key in urlargsdict.keys():
if re.match(regexp_argname, key): # replace `arg' by new values
urlargsdictnew[key] = []
for parg in urlargsdict[key]:
urlargsdictnew[key].append(parg.replace(text_old, text_new))
else: # keep old values
urlargsdictnew[key] = urlargsdict[key]
# build new URL for this word:
for key in urlargsdictnew.keys():
for val in urlargsdictnew[key]:
out += "&" + key + "=" + quote_plus(val, '')
if out.startswith("&"):
out = out[5:]
return out
def get_title_of_page(url):
"""
@param url: page to get the title from
@return: the page title in utf-8 or None in case
that any kind of exception occured e.g. connection error,
URL not known
"""
if BEAUTIFUL_SOUP_IMPORTED:
try:
- soup = BeautifulSoup.BeautifulSoup(urllib.urlopen(url))
+ opener = make_invenio_opener('UrlUtils')
+ soup = BeautifulSoup.BeautifulSoup(opener.open(url))
return soup.title.string.encode("utf-8")
except:
return None
else:
return "Title not available"
+def make_user_agent_string(component=None):
+ """
+ Return a nice and uniform user-agent string to be used when Invenio
+ act as a client in HTTP requests.
+ """
+ ret = "Invenio-%s (+%s; \"%s\")" % (CFG_VERSION, CFG_SITE_URL, CFG_SITE_NAME)
+ if component:
+ ret += " %s" % component
+ return ret
+
+class InvenioFancyURLopener(FancyURLopener):
+ ## Provide default user agent string
+ version = make_user_agent_string()
+ def prompt_user_passwd(self, host, realm):
+ """Don't prompt"""
+ return None, None
+
+## Let's override default useragent string
+## See: http://docs.python.org/release/2.4.4/lib/module-urllib.html
+urllib._urlopener = InvenioFancyURLopener()
+
+def make_invenio_opener(component=None):
+ """
+ Return an urllib2 opener with the useragent already set in the appropriate
+ way.
+ """
+ opener = urllib2.build_opener()
+ opener.addheaders = [('User-agent', make_user_agent_string(component))]
+ return opener
+
def create_AWS_request_url(base_url, argd, _amazon_secret_access_key,
_timestamp=None):
"""
Create a signed AWS (Amazon Web Service) request URL corresponding
to the given parameters.
Example:
>> create_AWS_request_url("http://ecs.amazon.com/onca/xml",
{'AWSAccessKeyID': '0000000000',
'Service': 'AWSECommerceService',
'Operation': 'ItemLookup',
'ItemID': '0679722769',
'ResponseGroup': 'ItemAttributes,Offers,Images,Review'},
"1234567890")
@param base_url: Service URL of the Amazon store to query
@param argd: dictionary of arguments defining the query
@param _amazon_secret_access_key: your Amazon secret key
@param _timestamp: for testing purpose only (default: current timestamp)
@type base_url: string
@type argd: dict
@type _amazon_secret_access_key: string
@type _timestamp: string
@return signed URL of the request (string)
"""
## First define a few util functions
def get_AWS_signature(argd, _amazon_secret_access_key,
method="GET", request_host="webservices.amazon.com",
request_uri="/onca/xml",
_timestamp=None):
"""
Returns the signature of an Amazon request, based on the
arguments of the request.
@param argd: dictionary of arguments defining the query
@param _amazon_secret_access_key: your Amazon secret key
@param method: method of the request POST or GET
@param request_host: host contacted for the query. To embed in the signature.
@param request_uri: uri contacted at 'request_host'. To embed in the signature.
@param _timestamp: for testing purpose only (default: current timestamp)
@type argd: dict
@type _amazon_secret_access_key: string
@type method: string
@type host_header: string
@type http_request_uri: string
@type _timestamp: string
@return signature of the request (string)
"""
# Add timestamp
if not _timestamp:
argd["Timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%SZ",
time.gmtime())
else:
argd["Timestamp"] = _timestamp
# Order parameter keys by byte value
parameter_keys = argd.keys()
parameter_keys.sort()
# Encode arguments, according to RFC 3986. Make sure we
# generate a list which is ordered by byte value of the keys
arguments = [quote(str(key), safe="~/") + "=" + \
quote(str(argd[key]), safe="~/") \
for key in parameter_keys]
# Join
parameters_string = "&".join(arguments)
# Prefix
parameters_string = method.upper() + "\n" + \
request_host.lower() + "\n" + \
(request_uri or "/") + "\n" + \
parameters_string
# Sign and return
return calculate_RFC2104_HMAC(parameters_string,
_amazon_secret_access_key)
def calculate_RFC2104_HMAC(data, _amazon_secret_access_key):
"""
Computes a RFC 2104 compliant HMAC Signature and then Base64
encodes it.
Module hashlib must be installed if Python < 2.5
@param data: data to sign
@param _amazon_secret_access_key: your Amazon secret key
@type data: string
@type _amazon_secret_access_key: string. Empty if hashlib module not installed
"""
if not HASHLIB_IMPORTED:
try:
raise Exception("Module hashlib not installed. Please install it.")
except:
from invenio.errorlib import register_exception
register_exception(stream='warning', alert_admin=True, subject='Cannot create AWS signature')
return ""
else:
if sys.version_info < (2, 5):
# compatibility mode for Python < 2.5 and hashlib
my_digest_algo = _MySHA256(sha256())
else:
my_digest_algo = sha256
return base64.encodestring(hmac.new(_amazon_secret_access_key,
data, my_digest_algo).digest()).strip()
## End util functions
parsed_url = urlparse(base_url)
signature = get_AWS_signature(argd, _amazon_secret_access_key,
request_host=parsed_url[1],
request_uri=parsed_url[2],
_timestamp=_timestamp)
if signature:
argd["Signature"] = signature
return base_url + "?" + urlencode(argd)
def create_Indico_request_url(base_url, indico_what, indico_loc, indico_id, indico_type, indico_params, indico_key, indico_sig, _timestamp=None):
"""
Create a signed Indico request URL to access Indico HTTP Export APIs.
See U{http://indico.cern.ch/ihelp/html/ExportAPI/index.html} for more
information.
Example:
>> create_Indico_request_url("https://indico.cern.ch",
"categ",
"",
[1, 7],
"xml",
{'onlypublic': 'yes',
'order': 'title',
'from': 'today',
'to': 'tomorrow'},
'00000000-0000-0000-0000-000000000000',
'00000000-0000-0000-0000-000000000000')
@param base_url: Service base URL of the Indico instance to query
@param indico_what: element to export
@type indico_what: one of the strings: C{categ}, C{event}, C{room}, C{reservation}
@param indico_loc: location of the element(s) specified by ID (only used for some elements)
@param indico_id: ID of the element to be exported
@type indico_id: a string or a list/tuple of strings
@param indico_type: output format
@type indico_type: one of the strings: C{json}, C{jsonp}, C{xml}, C{html}, C{ics}, C{atom}
@param indico_params: parameters of the query. See U{http://indico.cern.ch/ihelp/html/ExportAPI/common.html}
@param indico_key: API key provided for the given Indico instance
@param indico_sig: API secret key (signature) provided for the given Indico instance
@param _timestamp: for testing purpose only (default: current timestamp)
@return signed URL of the request (string)
"""
url = '/export/' + indico_what + '/'
if indico_loc:
url += indico_loc + '/'
if type(indico_id) in (list, tuple):
# dash separated list of values
indico_id = '-'.join([str(x) for x in indico_id])
url += indico_id + '.' + str(indico_type)
if hasattr(indico_params, 'items'):
items = indico_params.items()
else:
items = list(indico_params)
if indico_key:
items.append(('apikey', indico_key))
if indico_sig and HASHLIB_IMPORTED:
if _timestamp:
items.append(('timestamp', str(_timestamp)))
else:
items.append(('timestamp', str(int(time.time()))))
items = sorted(items, key=lambda x: x[0].lower())
url_to_sign = '%s?%s' % (url, urlencode(items))
if sys.version_info < (2, 5):
# compatibility mode for Python < 2.5 and hashlib
my_digest_algo = _MySHA1(sha1())
else:
my_digest_algo = sha1
signature = hmac.new(indico_sig, url_to_sign, my_digest_algo).hexdigest()
items.append(('signature', signature))
elif not HASHLIB_IMPORTED:
try:
raise Exception("Module hashlib not installed. Please install it.")
except:
from invenio.errorlib import register_exception
register_exception(stream='warning', alert_admin=True, subject='Cannot create AWS signature')
if not items:
return url
url = '%s%s?%s' % (base_url.strip('/'), url, urlencode(items))
return url
class _MyHashlibAlgo(object):
'''
Define a subclass of any hashlib algorithm class, with an additional "new()"
function, to work with the Python < 2.5 version of the hmac module.
(This class is more complex than it should, but it is not
possible to subclass haslib algorithm)
'''
def __init__(self, obj):
"""Set the wrapped object."""
super(_MyHashlibAlgo, self).__setattr__('_obj', obj)
methods = []
for name_value in inspect.getmembers(obj, inspect.ismethod):
methods.append(name_value[0])
super(_MyHashlibAlgo, self).__setattr__('__methods__', methods)
def isnotmethod(object_):
"Opposite of ismethod(..)"
return not inspect.ismethod(object_)
members = []
for name_value in inspect.getmembers(obj, isnotmethod):
members.append(name_value[0])
super(_MyHashlibAlgo, self).__setattr__('__members__', members)
def __getattr__(self, name):
"""Redirect unhandled get attribute to self._obj."""
if not hasattr(self._obj, name):
raise AttributeError, ("'%s' has no attribute %s" %
(self.__class__.__name__, name))
else:
return getattr(self._obj, name)
def __setattr__(self, name, value):
"""Redirect set attribute to self._obj if necessary."""
self_has_attr = True
try:
super(_MyHashlibAlgo, self).__getattribute__(name)
except AttributeError:
self_has_attr = False
if (name == "_obj" or not hasattr(self, "_obj") or
not hasattr(self._obj, name) or self_has_attr):
return super(_MyHashlibAlgo, self).__setattr__(name, value)
else:
return setattr(self._obj, name, value)
class _MySHA256(_MyHashlibAlgo):
"A _MyHashlibAlgo subsclass for sha256"
new = lambda d = '': sha256()
class _MySHA1(_MyHashlibAlgo):
"A _MyHashlibAlgo subsclass for sha1"
new = lambda d = '': sha1()
def auto_version_url(file_path):
""" Appends modification time of the file to the request URL in order for the
browser to refresh the cache when file changes
@param file_path: path to the file, e.g js/foo.js
@return: file_path with modification time appended to URL
"""
return file_path + "?%s" % md5(open(CFG_WEBDIR + os.sep + file_path).read()).hexdigest()
diff --git a/modules/miscutil/lib/web_api_key.py b/modules/miscutil/lib/web_api_key.py
new file mode 100644
index 000000000..0ba4dd10b
--- /dev/null
+++ b/modules/miscutil/lib/web_api_key.py
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2006, 2007, 2008, 2010, 2011 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ Invenio utilities to perform a REST like authentication
+"""
+import sys
+if sys.version_info < (2, 5):
+ import sha as sha1
+else:
+ from hashlib import sha1
+import hmac
+import time
+from cgi import parse_qsl
+from urllib import urlencode
+import re
+
+try:
+ from uuid import uuid4
+except ImportError:
+ import random
+ def uuid4():
+ return "%x" % random.getrandbits(16*8)
+
+from invenio.dbquery import run_sql, IntegrityError
+from invenio.config import CFG_WEB_API_KEY_ALLOWED_URL
+from invenio.access_control_config import CFG_WEB_API_KEY_STATUS
+
+
+_CFG_WEB_API_KEY_ALLOWED_URL = [(re.compile(_url), _authorized_time, _need_timestamp)
+ for _url, _authorized_time, _need_timestamp in CFG_WEB_API_KEY_ALLOWED_URL]
+
+
+def create_new_web_api_key(uid, key_description=None):
+ """
+ Creates a new pair REST API key / secret key for the user. To do that it
+ uses the uuid4 function.
+
+ @param uid: User's id for the new REST API key
+ @type uid: int
+ @param key_description: User's description for the REST API key
+ @type key_description: string
+ """
+ key_id = str(uuid4())
+ key_secrect = str(uuid4())
+ while True:
+ try:
+ run_sql("INSERT INTO webapikey (id,secret,id_user,description) VALUES(%s,%s,%s,%s)",
+ (key_id, key_secrect, uid, key_description))
+ break
+ except IntegrityError:
+ key_id = str(uuid4())
+
+def show_web_api_keys(uid, diff_status=CFG_WEB_API_KEY_STATUS['REMOVED']):
+ """
+ Makes a query to the DB to obtain all the user's REST API keys
+
+ @param uid: User's id
+ @type uid: int
+ @param diff_status: This string indicates if the query will show
+ all the REST API keys or only the ones that still active (usefull in the
+ admin part)
+ @type diff_statusparam: string
+
+ @return: Tuples with the id, description and status of the user's REST API
+ keys
+ """
+ keys_info = run_sql("SELECT id, description, status FROM webapikey WHERE id_user = %s AND status <> %s",
+ (uid, diff_status))
+ return keys_info
+
+def mark_web_api_key_as_removed(key_id):
+ """
+ When the user wants to remove one of his key, this functions puts the status
+ value of that key to remove, this way the user doesn't see the key anymore
+ but the admin user stills see it, make statistics whit it, etc.
+
+ @param key_id: The id of the REST key that will be "removed"
+ @type key_id: string
+ """
+ run_sql("UPDATE webapikey SET status=%s WHERE id=%s", (CFG_WEB_API_KEY_STATUS['REMOVED'], key_id, ))
+
+def get_available_web_api_keys(uid):
+ """
+ Search for all the available REST keys, it means all the user's keys that are
+ not marked as REMOVED or REVOKED
+
+ @param uid: The user id
+ @type uid: int
+
+ @return: Tuples of REST API public keys
+ """
+ keys = run_sql("SELECT id FROM webapikey WHERE id_user=%s AND status <> %s AND status <> %s",
+ (uid, CFG_WEB_API_KEY_STATUS['REMOVED'], CFG_WEB_API_KEY_STATUS['REVOKED']))
+ return keys
+
+def acc_get_uid_from_request(path, args):
+ """
+ Looks in the data base for the secret that matches with the API key in the
+ request. If the REST API key is found and if the signature is correct
+ returns the user's id.
+
+ @param path: uri of the request until the "?" (i.e.: req.uri)
+ @type path: string
+ @param args: All the params of the request (i.e.: req.args)
+ @type args: string
+
+ @return: If everything goes well it returns the user's uid, it not -1
+ """
+ from invenio.webstat import register_customevent
+
+ params = parse_qsl(args)
+ api_key = signature = timestamp = None
+
+ for param in params:
+ if param[0] == 'apikey':
+ api_key = param[1]
+ elif param[0] == 'signature':
+ signature = param[1]
+ params.remove(param) #Get rid of the signature
+ elif param [0] == 'timestamp':
+ timestamp = param[1]
+ #Check if the url is well built
+ if api_key == None or signature == None:
+ return -1
+
+ url_req = "%s?%s" % (path, urlencode(params))
+
+ authorized_time = None
+ need_timestamp = False
+ for url, authorized_time, need_timestamp in _CFG_WEB_API_KEY_ALLOWED_URL:
+ if url.match(url_req) is not None:
+ break
+
+ if need_timestamp and timestamp == None:
+ return -1
+
+ if authorized_time is None:
+ return -1
+
+ if authorized_time != 0 and need_timestamp:
+ time_lapse = time.time() - float(timestamp)
+ if time_lapse > authorized_time or time_lapse < 0:
+ return -1
+
+ key = run_sql("SELECT id_user, secret FROM webapikey WHERE id=%s AND status <> %s AND status <> %s",
+ (api_key, CFG_WEB_API_KEY_STATUS['REMOVED'], CFG_WEB_API_KEY_STATUS['REVOKED']))
+ if len(key) == 0 or not key:
+ return -1
+ else:
+ uid = key[0][0]
+ secret_key = key[0][1]
+ server_signature = hmac.new(secret_key, url_req, sha1).hexdigest()
+ if signature == server_signature:
+ #If the signature is fine, log the key activity and return the UID
+ register_customevent("apikeyusage", [uid, api_key, path, url_req])
+ return uid
+ else:
+ return -1
+
+def build_web_request(path, params, uid=-1, api_key=None, timestamp=True):
+ """
+ Build a new request that uses REST authentication.
+ 1. Add your REST API key to the params
+ 2. Add the current timestamp to the params, if needed
+ 3. Sort the query string params
+ 4. Merge path and the sorted query string to a single string
+ 5. Create a HMAC-SHA1 signature of this string using your secret key as the key
+ 6. Append the hex-encoded signature to your query string
+
+ @note: If the api_key parameter is None, then this method performs a search
+ in the data base using the uid parameter to get on of the user's REST
+ API key. If the user has one or more usable REST API key this method
+ uses the first to appear.
+
+ @param path: uri of the request until the "?" (i.e.: /search)
+ @type path: string
+ @param params: All the params of the request (i.e.: req.args or a dictionary
+ with the param name as key)
+ @type params: string or dict
+ @param api_key: User REST API key
+ @type api_key: string
+ @param uid: User's id to do the search for the REST API key
+ @type uid: int
+ @param timestamp: Indicates if timestamp is needed in the request
+ @type timestamp: boolean
+
+ @return: Signed request string or, in case of error, ''
+ """
+ if not isinstance(params, dict):
+ if len(params) != 0 and params[0] == '?':
+ params = params.replace('?','')
+ params = parse_qsl(params)
+
+ items = (hasattr(params, 'items') and [params.items()] or [list(params)])[0]
+
+ if api_key:
+ items.append(('apikey', api_key))
+ elif uid > 0:
+ keys = run_sql("SELECT id FROM webapikey WHERE id_user=%s AND status <> %s AND status <> %s",
+ (uid, CFG_WEB_API_KEY_STATUS['REMOVED'], CFG_WEB_API_KEY_STATUS['REVOKED']))
+ if keys is not None and len(keys) != 0:
+ api_key = keys[0][0]
+ items.append(('apikey', api_key))
+ else:
+ return ''
+ else:
+ return ''
+
+ if timestamp:
+ items.append(('timestamp', str(int(time.time()))))
+
+ items = sorted(items, key=lambda x: x[0].lower())
+ url = '%s?%s' % (path, urlencode(items))
+
+ secret_key = run_sql("SELECT secret FROM webapikey WHERE id=%s", (api_key,))
+ if len(secret_key) == 0 or not secret_key:
+ return ''
+ signature = hmac.new(secret_key[0][0], url, sha1).hexdigest()
+ items.append(('signature', signature))
+ if not items:
+ return path
+ return '%s?%s' % (path, urlencode(items))
diff --git a/modules/miscutil/lib/web_api_key_tests.py b/modules/miscutil/lib/web_api_key_tests.py
new file mode 100644
index 000000000..5b1944ddb
--- /dev/null
+++ b/modules/miscutil/lib/web_api_key_tests.py
@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2006, 2007, 2008, 2010, 2011 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+from invenio import web_api_key
+
+"""Unit tests for REST like authentication API."""
+
+try:
+ import hashlib
+except:
+ pass
+import unittest
+import re
+import hmac
+import urllib
+import time
+import string
+
+from invenio.testutils import make_test_suite, run_test_suite
+from invenio.dbquery import run_sql
+
+web_api_key.CFG_WEB_API_KEY_ALLOWED_URL = [('/search\?*', 0, True),
+ ('/bad\?*', -1, True)] #Just for testing
+
+web_api_key._CFG_WEB_API_KEY_ALLOWED_URL = [(re.compile(_url), _authorized_time, _need_timestamp)
+ for _url, _authorized_time, _need_timestamp in web_api_key.CFG_WEB_API_KEY_ALLOWED_URL]
+
+def build_web_request(path, params, api_key=None, secret_key=None):
+ items = (hasattr(params, 'items') and [params.items()] or [list(params)])[0]
+ if api_key:
+ items.append(('apikey', api_key))
+ if secret_key:
+ items.append(('timestamp', str(int(time.time()))))
+ items = sorted(items, key=lambda x: x[0].lower())
+ url = '%s?%s' % (path, urllib.urlencode(items))
+ signature = hmac.new(secret_key, url, hashlib.sha1).hexdigest()
+ items.append(('signature', signature))
+ if not items:
+ return path
+ return '%s?%s' % (path, urllib.urlencode(items))
+
+class APIKeyTest(unittest.TestCase):
+ """ Test functions related to the REST authentication API """
+ def setUp(self):
+ self.id_admin = run_sql('SELECT id FROM user WHERE nickname="admin"')[0][0]
+
+ def test_create_remove_show_key(self):
+ """apikey - create/list/delete REST key"""
+ self.assertEqual(0, len(web_api_key.show_web_api_keys(uid=self.id_admin)))
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key I")
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key II")
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key III")
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key IV")
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key V")
+ self.assertEqual(5, len(web_api_key.show_web_api_keys(uid=self.id_admin)))
+ self.assertEqual(5, len(web_api_key.show_web_api_keys(uid=self.id_admin, diff_status='')))
+ keys_info = web_api_key.show_web_api_keys(uid=self.id_admin)
+ web_api_key.mark_web_api_key_as_removed(keys_info[0][0])
+ self.assertEqual(4, len(web_api_key.show_web_api_keys(uid=self.id_admin)))
+ self.assertEqual(5, len(web_api_key.show_web_api_keys(uid=self.id_admin,diff_status='')))
+
+ run_sql("UPDATE webapikey SET status='WARNING' WHERE id=%s", (keys_info[1][0],))
+ run_sql("UPDATE webapikey SET status='REVOKED' WHERE id=%s", (keys_info[2][0],))
+
+ self.assertEqual(4, len(web_api_key.show_web_api_keys(uid=self.id_admin)))
+ self.assertEqual(5, len(web_api_key.show_web_api_keys(uid=self.id_admin, diff_status='')))
+
+ run_sql("DELETE FROM webapikey")
+
+ def test_acc_get_uid_from_request(self):
+ """webapikey - Login user from request using REST key"""
+ path = '/search'
+ params = 'ln=es&sc=1&c=Articles & Preprints&action_search=Buscar&p=ellis'
+
+ self.assertEqual(0, len(web_api_key.show_web_api_keys(uid=self.id_admin)))
+ web_api_key.create_new_web_api_key(self.id_admin, "Test key I")
+
+ key_info = run_sql("SELECT id FROM webapikey WHERE id_user=%s", (self.id_admin,))
+ url = web_api_key.build_web_request(path, params, api_key=key_info[0][0])
+ url = string.split(url, '?')
+ uid = web_api_key.acc_get_uid_from_request(url[0], url[1])
+ self.assertEqual(uid, self.id_admin)
+
+ url = web_api_key.build_web_request(path, params, api_key=key_info[0][0])
+ url += "123" # corrupt the key
+ url = string.split(url, '?')
+ uid = web_api_key.acc_get_uid_from_request(url[0], url[1])
+ self.assertEqual(uid, -1)
+
+ path = '/bad'
+ uid = web_api_key.acc_get_uid_from_request(path, "")
+ self.assertEqual(uid, -1)
+ params = { 'nocache': 'yes', 'limit': 123 }
+ url = web_api_key.build_web_request(path, params, api_key=key_info[0][0])
+ url = string.split(url, '?')
+ uid = web_api_key.acc_get_uid_from_request(url[0], url[1])
+ self.assertEqual(uid, -1)
+
+ run_sql("DELETE FROM webapikey")
+
+TEST_SUITE = make_test_suite(APIKeyTest)
+
+if __name__ == "__main__":
+ run_test_suite(TEST_SUITE)
+ run_sql("DELETE FROM webapikey")
\ No newline at end of file
diff --git a/modules/miscutil/sql/tabbibclean.sql b/modules/miscutil/sql/tabbibclean.sql
index e3321df9a..5ed7a3e5e 100644
--- a/modules/miscutil/sql/tabbibclean.sql
+++ b/modules/miscutil/sql/tabbibclean.sql
@@ -1,341 +1,340 @@
TRUNCATE bibrec;
TRUNCATE bib00x;
TRUNCATE bib01x;
TRUNCATE bib02x;
TRUNCATE bib03x;
TRUNCATE bib04x;
TRUNCATE bib05x;
TRUNCATE bib06x;
TRUNCATE bib07x;
TRUNCATE bib08x;
TRUNCATE bib09x;
TRUNCATE bib10x;
TRUNCATE bib11x;
TRUNCATE bib12x;
TRUNCATE bib13x;
TRUNCATE bib14x;
TRUNCATE bib15x;
TRUNCATE bib16x;
TRUNCATE bib17x;
TRUNCATE bib18x;
TRUNCATE bib19x;
TRUNCATE bib20x;
TRUNCATE bib21x;
TRUNCATE bib22x;
TRUNCATE bib23x;
TRUNCATE bib24x;
TRUNCATE bib25x;
TRUNCATE bib26x;
TRUNCATE bib27x;
TRUNCATE bib28x;
TRUNCATE bib29x;
TRUNCATE bib30x;
TRUNCATE bib31x;
TRUNCATE bib32x;
TRUNCATE bib33x;
TRUNCATE bib34x;
TRUNCATE bib35x;
TRUNCATE bib36x;
TRUNCATE bib37x;
TRUNCATE bib38x;
TRUNCATE bib39x;
TRUNCATE bib40x;
TRUNCATE bib41x;
TRUNCATE bib42x;
TRUNCATE bib43x;
TRUNCATE bib44x;
TRUNCATE bib45x;
TRUNCATE bib46x;
TRUNCATE bib47x;
TRUNCATE bib48x;
TRUNCATE bib49x;
TRUNCATE bib50x;
TRUNCATE bib51x;
TRUNCATE bib52x;
TRUNCATE bib53x;
TRUNCATE bib54x;
TRUNCATE bib55x;
TRUNCATE bib56x;
TRUNCATE bib57x;
TRUNCATE bib58x;
TRUNCATE bib59x;
TRUNCATE bib60x;
TRUNCATE bib61x;
TRUNCATE bib62x;
TRUNCATE bib63x;
TRUNCATE bib64x;
TRUNCATE bib65x;
TRUNCATE bib66x;
TRUNCATE bib67x;
TRUNCATE bib68x;
TRUNCATE bib69x;
TRUNCATE bib70x;
TRUNCATE bib71x;
TRUNCATE bib72x;
TRUNCATE bib73x;
TRUNCATE bib74x;
TRUNCATE bib75x;
TRUNCATE bib76x;
TRUNCATE bib77x;
TRUNCATE bib78x;
TRUNCATE bib79x;
TRUNCATE bib80x;
TRUNCATE bib81x;
TRUNCATE bib82x;
TRUNCATE bib83x;
TRUNCATE bib84x;
TRUNCATE bib85x;
TRUNCATE bib86x;
TRUNCATE bib87x;
TRUNCATE bib88x;
TRUNCATE bib89x;
TRUNCATE bib90x;
TRUNCATE bib91x;
TRUNCATE bib92x;
TRUNCATE bib93x;
TRUNCATE bib94x;
TRUNCATE bib95x;
TRUNCATE bib96x;
TRUNCATE bib97x;
TRUNCATE bib98x;
TRUNCATE bib99x;
TRUNCATE bibrec_bib00x;
TRUNCATE bibrec_bib01x;
TRUNCATE bibrec_bib02x;
TRUNCATE bibrec_bib03x;
TRUNCATE bibrec_bib04x;
TRUNCATE bibrec_bib05x;
TRUNCATE bibrec_bib06x;
TRUNCATE bibrec_bib07x;
TRUNCATE bibrec_bib08x;
TRUNCATE bibrec_bib09x;
TRUNCATE bibrec_bib10x;
TRUNCATE bibrec_bib11x;
TRUNCATE bibrec_bib12x;
TRUNCATE bibrec_bib13x;
TRUNCATE bibrec_bib14x;
TRUNCATE bibrec_bib15x;
TRUNCATE bibrec_bib16x;
TRUNCATE bibrec_bib17x;
TRUNCATE bibrec_bib18x;
TRUNCATE bibrec_bib19x;
TRUNCATE bibrec_bib20x;
TRUNCATE bibrec_bib21x;
TRUNCATE bibrec_bib22x;
TRUNCATE bibrec_bib23x;
TRUNCATE bibrec_bib24x;
TRUNCATE bibrec_bib25x;
TRUNCATE bibrec_bib26x;
TRUNCATE bibrec_bib27x;
TRUNCATE bibrec_bib28x;
TRUNCATE bibrec_bib29x;
TRUNCATE bibrec_bib30x;
TRUNCATE bibrec_bib31x;
TRUNCATE bibrec_bib32x;
TRUNCATE bibrec_bib33x;
TRUNCATE bibrec_bib34x;
TRUNCATE bibrec_bib35x;
TRUNCATE bibrec_bib36x;
TRUNCATE bibrec_bib37x;
TRUNCATE bibrec_bib38x;
TRUNCATE bibrec_bib39x;
TRUNCATE bibrec_bib40x;
TRUNCATE bibrec_bib41x;
TRUNCATE bibrec_bib42x;
TRUNCATE bibrec_bib43x;
TRUNCATE bibrec_bib44x;
TRUNCATE bibrec_bib45x;
TRUNCATE bibrec_bib46x;
TRUNCATE bibrec_bib47x;
TRUNCATE bibrec_bib48x;
TRUNCATE bibrec_bib49x;
TRUNCATE bibrec_bib50x;
TRUNCATE bibrec_bib51x;
TRUNCATE bibrec_bib52x;
TRUNCATE bibrec_bib53x;
TRUNCATE bibrec_bib54x;
TRUNCATE bibrec_bib55x;
TRUNCATE bibrec_bib56x;
TRUNCATE bibrec_bib57x;
TRUNCATE bibrec_bib58x;
TRUNCATE bibrec_bib59x;
TRUNCATE bibrec_bib60x;
TRUNCATE bibrec_bib61x;
TRUNCATE bibrec_bib62x;
TRUNCATE bibrec_bib63x;
TRUNCATE bibrec_bib64x;
TRUNCATE bibrec_bib65x;
TRUNCATE bibrec_bib66x;
TRUNCATE bibrec_bib67x;
TRUNCATE bibrec_bib68x;
TRUNCATE bibrec_bib69x;
TRUNCATE bibrec_bib70x;
TRUNCATE bibrec_bib71x;
TRUNCATE bibrec_bib72x;
TRUNCATE bibrec_bib73x;
TRUNCATE bibrec_bib74x;
TRUNCATE bibrec_bib75x;
TRUNCATE bibrec_bib76x;
TRUNCATE bibrec_bib77x;
TRUNCATE bibrec_bib78x;
TRUNCATE bibrec_bib79x;
TRUNCATE bibrec_bib80x;
TRUNCATE bibrec_bib81x;
TRUNCATE bibrec_bib82x;
TRUNCATE bibrec_bib83x;
TRUNCATE bibrec_bib84x;
TRUNCATE bibrec_bib85x;
TRUNCATE bibrec_bib86x;
TRUNCATE bibrec_bib87x;
TRUNCATE bibrec_bib88x;
TRUNCATE bibrec_bib89x;
TRUNCATE bibrec_bib90x;
TRUNCATE bibrec_bib91x;
TRUNCATE bibrec_bib92x;
TRUNCATE bibrec_bib93x;
TRUNCATE bibrec_bib94x;
TRUNCATE bibrec_bib95x;
TRUNCATE bibrec_bib96x;
TRUNCATE bibrec_bib97x;
TRUNCATE bibrec_bib98x;
TRUNCATE bibrec_bib99x;
TRUNCATE bibfmt;
TRUNCATE idxWORD01F;
TRUNCATE idxWORD02F;
TRUNCATE idxWORD03F;
TRUNCATE idxWORD04F;
TRUNCATE idxWORD05F;
TRUNCATE idxWORD06F;
TRUNCATE idxWORD07F;
TRUNCATE idxWORD08F;
TRUNCATE idxWORD09F;
TRUNCATE idxWORD10F;
TRUNCATE idxWORD11F;
TRUNCATE idxWORD12F;
TRUNCATE idxWORD13F;
TRUNCATE idxWORD14F;
TRUNCATE idxWORD15F;
TRUNCATE idxWORD16F;
TRUNCATE idxWORD17F;
TRUNCATE idxWORD18F;
TRUNCATE idxWORD01R;
TRUNCATE idxWORD02R;
TRUNCATE idxWORD03R;
TRUNCATE idxWORD04R;
TRUNCATE idxWORD05R;
TRUNCATE idxWORD06R;
TRUNCATE idxWORD07R;
TRUNCATE idxWORD08R;
TRUNCATE idxWORD09R;
TRUNCATE idxWORD10R;
TRUNCATE idxWORD11R;
TRUNCATE idxWORD12R;
TRUNCATE idxWORD13R;
TRUNCATE idxWORD14R;
TRUNCATE idxWORD15R;
TRUNCATE idxWORD16R;
TRUNCATE idxWORD17R;
TRUNCATE idxWORD18R;
TRUNCATE idxPAIR01F;
TRUNCATE idxPAIR02F;
TRUNCATE idxPAIR03F;
TRUNCATE idxPAIR04F;
TRUNCATE idxPAIR05F;
TRUNCATE idxPAIR06F;
TRUNCATE idxPAIR07F;
TRUNCATE idxPAIR08F;
TRUNCATE idxPAIR09F;
TRUNCATE idxPAIR10F;
TRUNCATE idxPAIR11F;
TRUNCATE idxPAIR12F;
TRUNCATE idxPAIR13F;
TRUNCATE idxPAIR14F;
TRUNCATE idxPAIR15F;
TRUNCATE idxPAIR16F;
TRUNCATE idxPAIR17F;
TRUNCATE idxPAIR18F;
TRUNCATE idxPAIR01R;
TRUNCATE idxPAIR02R;
TRUNCATE idxPAIR03R;
TRUNCATE idxPAIR04R;
TRUNCATE idxPAIR05R;
TRUNCATE idxPAIR06R;
TRUNCATE idxPAIR07R;
TRUNCATE idxPAIR08R;
TRUNCATE idxPAIR09R;
TRUNCATE idxPAIR10R;
TRUNCATE idxPAIR11R;
TRUNCATE idxPAIR12R;
TRUNCATE idxPAIR13R;
TRUNCATE idxPAIR14R;
TRUNCATE idxPAIR15R;
TRUNCATE idxPAIR16R;
TRUNCATE idxPAIR17R;
TRUNCATE idxPAIR18R;
TRUNCATE idxPHRASE01F;
TRUNCATE idxPHRASE02F;
TRUNCATE idxPHRASE03F;
TRUNCATE idxPHRASE04F;
TRUNCATE idxPHRASE05F;
TRUNCATE idxPHRASE06F;
TRUNCATE idxPHRASE07F;
TRUNCATE idxPHRASE08F;
TRUNCATE idxPHRASE09F;
TRUNCATE idxPHRASE10F;
TRUNCATE idxPHRASE11F;
TRUNCATE idxPHRASE12F;
TRUNCATE idxPHRASE13F;
TRUNCATE idxPHRASE14F;
TRUNCATE idxPHRASE15F;
TRUNCATE idxPHRASE16F;
TRUNCATE idxPHRASE17F;
TRUNCATE idxPHRASE18F;
TRUNCATE idxPHRASE01R;
TRUNCATE idxPHRASE02R;
TRUNCATE idxPHRASE03R;
TRUNCATE idxPHRASE04R;
TRUNCATE idxPHRASE05R;
TRUNCATE idxPHRASE06R;
TRUNCATE idxPHRASE07R;
TRUNCATE idxPHRASE08R;
TRUNCATE idxPHRASE09R;
TRUNCATE idxPHRASE10R;
TRUNCATE idxPHRASE11R;
TRUNCATE idxPHRASE12R;
TRUNCATE idxPHRASE13R;
TRUNCATE idxPHRASE14R;
TRUNCATE idxPHRASE15R;
TRUNCATE idxPHRASE16R;
TRUNCATE idxPHRASE17R;
TRUNCATE idxPHRASE18R;
TRUNCATE rnkMETHODDATA;
TRUNCATE rnkCITATIONDATA;
TRUNCATE rnkDOWNLOADS;
TRUNCATE rnkPAGEVIEWS;
TRUNCATE rnkWORD01F;
TRUNCATE rnkWORD01R;
TRUNCATE bibdoc;
TRUNCATE bibrec_bibdoc;
TRUNCATE bibdoc_bibdoc;
TRUNCATE bibdocfsinfo;
TRUNCATE sbmAPPROVAL;
TRUNCATE sbmSUBMISSIONS;
TRUNCATE sbmPUBLICATION;
TRUNCATE sbmPUBLICATIONCOMM;
TRUNCATE sbmPUBLICATIONDATA;
TRUNCATE hstRECORD;
TRUNCATE hstDOCUMENT;
TRUNCATE bibHOLDINGPEN;
TRUNCATE hstEXCEPTION;
TRUNCATE aidPERSONIDDATA;
TRUNCATE aidRESULTS;
-TRUNCATE aidPROBCACHE;
TRUNCATE aidCACHE;
TRUNCATE aidPERSONIDPAPERS;
TRUNCATE aidUSERINPUTLOG;
TRUNCATE lnkENTRY;
TRUNCATE lnkENTRYURLTITLE;
TRUNCATE lnkENTRYLOG;
TRUNCATE lnkLOG;
TRUNCATE lnkADMINURL;
TRUNCATE lnkADMINURLLOG;
diff --git a/modules/miscutil/sql/tabcreate.sql b/modules/miscutil/sql/tabcreate.sql
index ed89f1abd..9b2333402 100644
--- a/modules/miscutil/sql/tabcreate.sql
+++ b/modules/miscutil/sql/tabcreate.sql
@@ -1,4163 +1,4180 @@
-- This file is part of Invenio.
-- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
--
-- Invenio is free software; you can redistribute it and/or
-- modify it under the terms of the GNU General Public License as
-- published by the Free Software Foundation; either version 2 of the
-- License, or (at your option) any later version.
--
-- Invenio is distributed in the hope that it will be useful, but
-- WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-- General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with Invenio; if not, write to the Free Software Foundation, Inc.,
-- 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-- tables for bibliographic records:
CREATE TABLE IF NOT EXISTS bibrec (
id mediumint(8) unsigned NOT NULL auto_increment,
creation_date datetime NOT NULL default '0000-00-00',
modification_date datetime NOT NULL default '0000-00-00',
PRIMARY KEY (id),
KEY creation_date (creation_date),
KEY modification_date (modification_date)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib00x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib01x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib02x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib03x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib04x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib05x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib06x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib07x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib08x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib09x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib10x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib11x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib12x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib13x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib14x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib15x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib16x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib17x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib18x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib19x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib20x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib21x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib22x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib23x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib24x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib25x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib26x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib27x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib28x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib29x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib30x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib31x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib32x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib33x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib34x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib35x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib36x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib37x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib38x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib39x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib40x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib41x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib42x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib43x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib44x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib45x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib46x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib47x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib48x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib49x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib50x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib51x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib52x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib53x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib54x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib55x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib56x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib57x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib58x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib59x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib60x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib61x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib62x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib63x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib64x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib65x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib66x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib67x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib68x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib69x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib70x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib71x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib72x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib73x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib74x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib75x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib76x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib77x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib78x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib79x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib80x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib81x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib82x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib83x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib84x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib85x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(100)) -- URLs need usually a larger index for speedy lookups
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib86x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib87x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib88x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib89x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib90x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib91x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib92x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib93x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib94x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib95x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib96x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib97x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib98x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bib99x (
id mediumint(8) unsigned NOT NULL auto_increment,
tag varchar(6) NOT NULL default '',
value text NOT NULL,
PRIMARY KEY (id),
KEY kt (tag),
KEY kv (value(35))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib00x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib01x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib02x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib03x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib04x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib05x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib06x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib07x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib08x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib09x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib10x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib11x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib12x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib13x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib14x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib15x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib16x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib17x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib18x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib19x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib20x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib21x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib22x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib23x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib24x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib25x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib26x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib27x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib28x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib29x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib30x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib31x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib32x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib33x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib34x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib35x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib36x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib37x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib38x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib39x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib40x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib41x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib42x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib43x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib44x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib45x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib46x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib47x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib48x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib49x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib50x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib51x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib52x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib53x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib54x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib55x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib56x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib57x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib58x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib59x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib60x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib61x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib62x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib63x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib64x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib65x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib66x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib67x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib68x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib69x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib70x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib71x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib72x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib73x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib74x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib75x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib76x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib77x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib78x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib79x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib80x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib81x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib82x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib83x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib84x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib85x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib86x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib87x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib88x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib89x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib90x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib91x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib92x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib93x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib94x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib95x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib96x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib97x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib98x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bib99x (
id_bibrec mediumint(8) unsigned NOT NULL default '0',
id_bibxxx mediumint(8) unsigned NOT NULL default '0',
field_number smallint(5) unsigned default NULL,
KEY id_bibxxx (id_bibxxx),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
-- tables for bibliographic records formatted:
CREATE TABLE IF NOT EXISTS bibfmt (
id mediumint(8) unsigned NOT NULL auto_increment,
id_bibrec int(8) unsigned NOT NULL default '0',
format varchar(10) NOT NULL default '',
last_updated datetime NOT NULL default '0000-00-00',
value longblob,
PRIMARY KEY (id),
KEY id_bibrec (id_bibrec),
KEY format (format)
) ENGINE=MyISAM;
-- tables for index files:
CREATE TABLE IF NOT EXISTS idxINDEX (
id mediumint(9) unsigned NOT NULL,
name varchar(50) NOT NULL default '',
description varchar(255) NOT NULL default '',
last_updated datetime NOT NULL default '0000-00-00 00:00:00',
stemming_language varchar(10) NOT NULL default '',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxINDEXNAME (
id_idxINDEX mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_idxINDEX,ln,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxINDEX_field (
id_idxINDEX mediumint(9) unsigned NOT NULL,
id_field mediumint(9) unsigned NOT NULL,
regexp_punctuation varchar(255) NOT NULL default "[\.\,\:\;\?\!\"]",
regexp_alphanumeric_separators varchar(255) NOT NULL default "[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~]",
PRIMARY KEY (id_idxINDEX,id_field)
) ENGINE=MyISAM;
-- this comment line here is just to fix the SQL display mode in Emacs '
CREATE TABLE IF NOT EXISTS idxWORD01F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD01R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD02F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD02R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD03F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD03R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD04F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD04R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD05F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD05R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD06F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD06R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD07F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD07R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD08F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD08R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD09F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD09R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD10F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD10R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD11F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD11R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD12F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD12R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD13F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD13R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD14F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD14R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD15F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD15R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD16F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD16R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD17F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD17R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD18F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxWORD18R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR01F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR01R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR02F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR02R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR03F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR03R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR04F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR04R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR05F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR05R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR06F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR06R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR07F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR07R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR08F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR08R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR09F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR09R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR10F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR10R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR11F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR11R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR12F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR12R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR13F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR13R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR14F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR14R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR15F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR15R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR16F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR16R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR17F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR17R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR18F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(100) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPAIR18R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE01F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE01R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE02F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE02R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE03F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE03R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE04F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE04R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE05F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE05R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE06F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE06R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE07F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE07R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE08F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE08R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE09F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE09R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE10F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE10R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE11F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE11R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE12F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE12R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE13F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE13R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE14F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE14R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE15F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE15R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE16F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE16R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE17F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE17R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE18F (
id mediumint(9) unsigned NOT NULL auto_increment,
term text default NULL,
hitlist longblob,
PRIMARY KEY (id),
KEY term (term(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS idxPHRASE18R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
-- tables for ranking:
CREATE TABLE IF NOT EXISTS rnkMETHOD (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(20) NOT NULL default '',
last_updated datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkMETHODNAME (
id_rnkMETHOD mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_rnkMETHOD,ln,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkMETHODDATA (
id_rnkMETHOD mediumint(9) unsigned NOT NULL,
relevance_data longblob,
PRIMARY KEY (id_rnkMETHOD)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS collection_rnkMETHOD (
id_collection mediumint(9) unsigned NOT NULL,
id_rnkMETHOD mediumint(9) unsigned NOT NULL,
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection,id_rnkMETHOD)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkWORD01F (
id mediumint(9) unsigned NOT NULL auto_increment,
term varchar(50) default NULL,
hitlist longblob,
PRIMARY KEY (id),
UNIQUE KEY term (term)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkWORD01R (
id_bibrec mediumint(9) unsigned NOT NULL,
termlist longblob,
type enum('CURRENT','FUTURE','TEMPORARY') NOT NULL default 'CURRENT',
PRIMARY KEY (id_bibrec,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkAUTHORDATA (
aterm varchar(50) default NULL,
hitlist longblob,
UNIQUE KEY aterm (aterm)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkPAGEVIEWS (
id_bibrec mediumint(8) unsigned default NULL,
id_user int(15) unsigned default '0',
client_host int(10) unsigned default NULL,
view_time datetime default '0000-00-00 00:00:00',
KEY view_time (view_time),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS rnkDOWNLOADS (
id_bibrec mediumint(8) unsigned default NULL,
download_time datetime default '0000-00-00 00:00:00',
client_host int(10) unsigned default NULL,
id_user int(15) unsigned default NULL,
id_bibdoc mediumint(9) unsigned default NULL,
file_version smallint(2) unsigned default NULL,
file_format varchar(10) NULL default NULL,
KEY download_time (download_time),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
-- a table for citations. record-cites-record
CREATE TABLE IF NOT EXISTS rnkCITATIONDATA (
id mediumint(8) unsigned NOT NULL auto_increment,
object_name varchar(255) NOT NULL,
object_value longblob,
last_updated datetime NOT NULL default '0000-00-00',
PRIMARY KEY id (id),
UNIQUE KEY object_name (object_name)
) ENGINE=MyISAM;
-- a table for missing citations. This should be scanned by a program
-- occasionally to check if some publication has been cited more than
-- 50 times (or such), and alert cataloguers to create record for that
-- external citation
--
-- id_bibrec is the id of the record. extcitepubinfo is publication info
-- that looks in general like hep-th/0112088
CREATE TABLE IF NOT EXISTS rnkCITATIONDATAEXT (
id_bibrec int(8) unsigned,
extcitepubinfo varchar(255) NOT NULL,
PRIMARY KEY (id_bibrec, extcitepubinfo),
KEY extcitepubinfo (extcitepubinfo)
) ENGINE=MyISAM;
-- tables for collections and collection tree:
CREATE TABLE IF NOT EXISTS collection (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
dbquery text,
nbrecs int(10) unsigned default '0',
reclist longblob,
PRIMARY KEY (id),
UNIQUE KEY name (name),
KEY dbquery (dbquery(50))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS collectionname (
id_collection mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_collection,ln,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS collection_collection (
id_dad mediumint(9) unsigned NOT NULL,
id_son mediumint(9) unsigned NOT NULL,
type char(1) NOT NULL default 'r',
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_dad,id_son)
) ENGINE=MyISAM;
-- tables for OAI sets:
CREATE TABLE IF NOT EXISTS oaiREPOSITORY (
id mediumint(9) unsigned NOT NULL auto_increment,
setName varchar(255) NOT NULL default '',
setSpec varchar(255) NOT NULL default 'GLOBAL_SET',
setCollection varchar(255) NOT NULL default '',
setDescription text NOT NULL default '',
setDefinition text NOT NULL default '',
setRecList longblob,
p1 text NOT NULL default '',
f1 text NOT NULL default '',
m1 text NOT NULL default '',
p2 text NOT NULL default '',
f2 text NOT NULL default '',
m2 text NOT NULL default '',
p3 text NOT NULL default '',
f3 text NOT NULL default '',
m3 text NOT NULL default '',
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS oaiHARVEST (
id mediumint(9) unsigned NOT NULL auto_increment,
baseurl varchar(255) NOT NULL default '',
metadataprefix varchar(255) NOT NULL default 'oai_dc',
arguments text,
comment text,
bibconvertcfgfile varchar(255),
name varchar(255) NOT NULL,
lastrun datetime,
frequency mediumint(12) NOT NULL default '0',
postprocess varchar(20) NOT NULL default 'h',
bibfilterprogram varchar(255) NOT NULL default '',
setspecs text NOT NULL default '',
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS oaiHARVESTLOG (
id_oaiHARVEST mediumint(9) unsigned NOT NULL REFERENCES oaiHARVEST, -- source we harvest from
id_bibrec mediumint(8) unsigned NOT NULL default '0', -- internal record id ( filled by bibupload )
bibupload_task_id int NOT NULL default 0, -- bib upload task number
oai_id varchar(40) NOT NULL default "", -- OAI record identifier we harvested
date_harvested datetime NOT NULL default '0000-00-00', -- when we harvested
date_inserted datetime NOT NULL default '0000-00-00', -- when it was inserted
inserted_to_db char(1) NOT NULL default 'P', -- where it was inserted (P=prod, H=holding-pen, etc)
PRIMARY KEY (bibupload_task_id, oai_id, date_harvested)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibHOLDINGPEN (
changeset_id INT NOT NULL AUTO_INCREMENT, -- the identifier of the changeset stored in the holding pen
changeset_date datetime NOT NULL DEFAULT '0000:00:00 00:00:00', -- when was the changeset inserted
changeset_xml TEXT NOT NULL DEFAULT '',
oai_id varchar(40) NOT NULL DEFAULT '', -- OAI identifier of concerned record
id_bibrec mediumint(8) unsigned NOT NULL default '0', -- record ID of concerned record (filled by bibupload)
PRIMARY KEY (changeset_id),
KEY changeset_date (changeset_date),
KEY id_bibrec (id_bibrec)
) ENGINE=MyISAM;
-- tables for portal elements:
CREATE TABLE IF NOT EXISTS collection_portalbox (
id_collection mediumint(9) unsigned NOT NULL,
id_portalbox mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
position char(3) NOT NULL default 'top',
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection,id_portalbox,ln)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS portalbox (
id mediumint(9) unsigned NOT NULL auto_increment,
title text NOT NULL,
body text NOT NULL,
UNIQUE KEY id (id)
) ENGINE=MyISAM;
-- tables for search examples:
CREATE TABLE IF NOT EXISTS collection_example (
id_collection mediumint(9) unsigned NOT NULL,
id_example mediumint(9) unsigned NOT NULL,
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection,id_example)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS example (
id mediumint(9) unsigned NOT NULL auto_increment,
type text NOT NULL default '',
body text NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- tables for collection formats:
CREATE TABLE IF NOT EXISTS collection_format (
id_collection mediumint(9) unsigned NOT NULL,
id_format mediumint(9) unsigned NOT NULL,
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection,id_format)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS format (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
code varchar(6) NOT NULL,
description varchar(255) default '',
content_type varchar(255) default '',
visibility tinyint NOT NULL default '1',
PRIMARY KEY (id),
UNIQUE KEY code (code)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS formatname (
id_format mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_format,ln,type)
) ENGINE=MyISAM;
-- tables for collection detailed page options
CREATE TABLE IF NOT EXISTS collectiondetailedrecordpagetabs (
id_collection mediumint(9) unsigned NOT NULL,
tabs varchar(255) NOT NULL default '',
PRIMARY KEY (id_collection)
) ENGINE=MyISAM;
-- tables for search options and MARC tags:
CREATE TABLE IF NOT EXISTS collection_field_fieldvalue (
id_collection mediumint(9) unsigned NOT NULL,
id_field mediumint(9) unsigned NOT NULL,
id_fieldvalue mediumint(9) unsigned,
type char(3) NOT NULL default 'src',
score tinyint(4) unsigned NOT NULL default '0',
score_fieldvalue tinyint(4) unsigned NOT NULL default '0',
KEY id_collection (id_collection),
KEY id_field (id_field),
KEY id_fieldvalue (id_fieldvalue)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS field (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
code varchar(255) NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY code (code)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS fieldname (
id_field mediumint(9) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_field,ln,type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS fieldvalue (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
value text NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS field_tag (
id_field mediumint(9) unsigned NOT NULL,
id_tag mediumint(9) unsigned NOT NULL,
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_field,id_tag)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS tag (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
value char(6) NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- tables for file management
CREATE TABLE IF NOT EXISTS bibdoc (
id mediumint(9) unsigned NOT NULL auto_increment,
status text NOT NULL default '',
docname varchar(250) COLLATE utf8_bin NOT NULL default 'file',
creation_date datetime NOT NULL default '0000-00-00',
modification_date datetime NOT NULL default '0000-00-00',
text_extraction_date datetime NOT NULL default '0000-00-00',
more_info mediumblob NULL default NULL,
PRIMARY KEY (id),
KEY docname (docname),
KEY creation_date (creation_date),
KEY modification_date (modification_date)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibrec_bibdoc (
id_bibrec mediumint(9) unsigned NOT NULL default '0',
id_bibdoc mediumint(9) unsigned NOT NULL default '0',
type varchar(255),
KEY (id_bibrec),
KEY (id_bibdoc)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibdoc_bibdoc (
id_bibdoc1 mediumint(9) unsigned NOT NULL,
id_bibdoc2 mediumint(9) unsigned NOT NULL,
type varchar(255),
KEY (id_bibdoc1),
KEY (id_bibdoc2)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bibdocfsinfo (
id_bibdoc mediumint(9) unsigned NOT NULL,
version tinyint(4) unsigned NOT NULL,
format varchar(50) NOT NULL,
last_version boolean NOT NULL,
cd datetime NOT NULL,
md datetime NOT NULL,
checksum char(32) NOT NULL,
filesize bigint(15) unsigned NOT NULL,
mime varchar(100) NOT NULL,
master_format varchar(50) NULL default NULL,
PRIMARY KEY (id_bibdoc, version, format),
KEY (last_version),
KEY (format),
KEY (cd),
KEY (md),
KEY (filesize),
KEY (mime)
) ENGINE=MyISAM;
-- tables for publication requests:
CREATE TABLE IF NOT EXISTS publreq (
id int(11) NOT NULL auto_increment,
host varchar(255) NOT NULL default '',
date varchar(255) NOT NULL default '',
name varchar(255) NOT NULL default '',
email varchar(255) NOT NULL default '',
address text NOT NULL,
publication text NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- table for sessions and users:
CREATE TABLE IF NOT EXISTS session (
session_key varchar(32) NOT NULL default '',
session_expiry datetime NOT NULL default '0000-00-00 00:00:00',
session_object longblob,
uid int(15) unsigned NOT NULL,
UNIQUE KEY session_key (session_key),
KEY uid (uid),
KEY session_expiry (session_expiry)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user (
id int(15) unsigned NOT NULL auto_increment,
email varchar(255) NOT NULL default '',
password blob NOT NULL,
note varchar(255) default NULL,
settings blob default NULL,
nickname varchar(255) NOT NULL default '',
last_login datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY id (id),
KEY email (email),
KEY nickname (nickname)
) ENGINE=MyISAM;
+CREATE TABLE IF NOT EXISTS userEXT (
+ id varbinary(255) NOT NULL,
+ method varchar(50) NOT NULL,
+ id_user int(15) unsigned NOT NULL,
+ PRIMARY KEY (id, method),
+ UNIQUE KEY (id_user, method)
+) ENGINE=MyISAM;
+
+
-- tables for usergroups
CREATE TABLE IF NOT EXISTS usergroup (
id int(15) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL default '',
description text default '',
join_policy char(2) NOT NULL default '',
login_method varchar(255) NOT NULL default 'INTERNAL',
PRIMARY KEY (id),
UNIQUE KEY login_method_name (login_method(70), name),
KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_usergroup (
id_user int(15) unsigned NOT NULL default '0',
id_usergroup int(15) unsigned NOT NULL default '0',
user_status char(1) NOT NULL default '',
user_status_date datetime NOT NULL default '0000-00-00 00:00:00',
KEY id_user (id_user),
KEY id_usergroup (id_usergroup)
) ENGINE=MyISAM;
-- tables for access control engine
CREATE TABLE IF NOT EXISTS accROLE (
id int(15) unsigned NOT NULL auto_increment,
name varchar(32),
description varchar(255),
firerole_def_ser blob NULL,
firerole_def_src text NULL,
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_accROLE (
id_user int(15) unsigned NOT NULL,
id_accROLE int(15) unsigned NOT NULL,
expiration datetime NOT NULL default '9999-12-31 23:59:59',
PRIMARY KEY (id_user, id_accROLE)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS accMAILCOOKIE (
id int(15) unsigned NOT NULL auto_increment,
data blob NOT NULL,
expiration datetime NOT NULL default '9999-12-31 23:59:59',
kind varchar(32) NOT NULL,
onetime boolean NOT NULL default 0,
status char(1) NOT NULL default 'W',
PRIMARY KEY (id),
KEY expiration (expiration)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS accACTION (
id int(15) unsigned NOT NULL auto_increment,
name varchar(32),
description varchar(255),
allowedkeywords varchar(255),
optional ENUM ('yes', 'no') NOT NULL default 'no',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS accARGUMENT (
id int(15) unsigned NOT NULL auto_increment,
keyword varchar (32),
value varchar(255),
PRIMARY KEY (id),
KEY KEYVAL (keyword, value)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS accROLE_accACTION_accARGUMENT (
id_accROLE int(15),
id_accACTION int(15),
id_accARGUMENT int(15),
argumentlistid mediumint(8),
KEY id_accROLE (id_accROLE),
KEY id_accACTION (id_accACTION),
KEY id_accARGUMENT (id_accARGUMENT)
) ENGINE=MyISAM;
-- tables for personal/collaborative features (baskets, alerts, searches, messages, usergroups):
CREATE TABLE IF NOT EXISTS user_query (
id_user int(15) unsigned NOT NULL default '0',
id_query int(15) unsigned NOT NULL default '0',
hostname varchar(50) default 'unknown host',
date datetime default NULL,
KEY id_user (id_user,id_query)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS query (
id int(15) unsigned NOT NULL auto_increment,
type char(1) NOT NULL default 'r',
urlargs text NOT NULL,
PRIMARY KEY (id),
KEY urlargs (urlargs(100))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_query_basket (
id_user int(15) unsigned NOT NULL default '0',
id_query int(15) unsigned NOT NULL default '0',
id_basket int(15) unsigned NOT NULL default '0',
frequency varchar(5) NOT NULL default '',
date_creation date default NULL,
date_lastrun date default '0000-00-00',
alert_name varchar(30) NOT NULL default '',
alert_desc text default NULL,
notification char(1) NOT NULL default 'y',
PRIMARY KEY (id_user,id_query,frequency,id_basket),
KEY alert_name (alert_name)
) ENGINE=MyISAM;
-- baskets
CREATE TABLE IF NOT EXISTS bskBASKET (
id int(15) unsigned NOT NULL auto_increment,
id_owner int(15) unsigned NOT NULL default '0',
name varchar(50) NOT NULL default '',
date_modification datetime NOT NULL default '0000-00-00 00:00:00',
nb_views int(15) NOT NULL default '0',
PRIMARY KEY (id),
KEY id_owner (id_owner),
KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bskREC (
id_bibrec_or_bskEXTREC int(16) NOT NULL default '0',
id_bskBASKET int(15) unsigned NOT NULL default '0',
id_user_who_added_item int(15) NOT NULL default '0',
score int(15) NOT NULL default '0',
date_added datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id_bibrec_or_bskEXTREC,id_bskBASKET),
KEY id_bibrec_or_bskEXTREC (id_bibrec_or_bskEXTREC),
KEY id_bskBASKET (id_bskBASKET),
KEY score (score),
KEY date_added (date_added)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bskEXTREC (
id int(15) unsigned NOT NULL auto_increment,
external_id int(15) NOT NULL default '0',
collection_id int(15) unsigned NOT NULL default '0',
original_url text,
creation_date datetime NOT NULL default '0000-00-00 00:00:00',
modification_date datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bskEXTFMT (
id int(15) unsigned NOT NULL auto_increment,
id_bskEXTREC int(15) unsigned NOT NULL default '0',
format varchar(10) NOT NULL default '',
last_updated datetime NOT NULL default '0000-00-00 00:00:00',
value longblob,
PRIMARY KEY (id),
KEY id_bskEXTREC (id_bskEXTREC),
KEY format (format)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_bskBASKET (
id_user int(15) unsigned NOT NULL default '0',
id_bskBASKET int(15) unsigned NOT NULL default '0',
topic varchar(50) NOT NULL default '',
PRIMARY KEY (id_user,id_bskBASKET),
KEY id_user (id_user),
KEY id_bskBASKET (id_bskBASKET)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS usergroup_bskBASKET (
id_usergroup int(15) unsigned NOT NULL default '0',
id_bskBASKET int(15) unsigned NOT NULL default '0',
topic varchar(50) NOT NULL default '',
date_shared datetime NOT NULL default '0000-00-00 00:00:00',
share_level char(2) NOT NULL default '',
PRIMARY KEY (id_usergroup,id_bskBASKET),
KEY id_usergroup (id_usergroup),
KEY id_bskBASKET (id_bskBASKET)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bskRECORDCOMMENT (
id int(15) unsigned NOT NULL auto_increment,
id_bibrec_or_bskEXTREC int(16) NOT NULL default '0',
id_bskBASKET int(15) unsigned NOT NULL default '0',
id_user int(15) unsigned NOT NULL default '0',
title varchar(255) NOT NULL default '',
body text NOT NULL,
date_creation datetime NOT NULL default '0000-00-00 00:00:00',
priority int(15) NOT NULL default '0',
in_reply_to_id_bskRECORDCOMMENT int(15) unsigned NOT NULL default '0',
reply_order_cached_data blob NULL default NULL,
PRIMARY KEY (id),
KEY id_bskBASKET (id_bskBASKET),
KEY id_bibrec_or_bskEXTREC (id_bibrec_or_bskEXTREC),
KEY date_creation (date_creation),
KEY in_reply_to_id_bskRECORDCOMMENT (in_reply_to_id_bskRECORDCOMMENT),
INDEX (reply_order_cached_data(40))
) ENGINE=MyISAM;
-- tables for messaging system
CREATE TABLE IF NOT EXISTS msgMESSAGE (
id int(15) unsigned NOT NULL auto_increment,
id_user_from int(15) unsigned NOT NULL default '0',
sent_to_user_nicks text NOT NULL default '',
sent_to_group_names text NOT NULL default '',
subject text NOT NULL default '',
body text default NULL,
sent_date datetime NOT NULL default '0000-00-00 00:00:00',
received_date datetime NULL default '0000-00-00 00:00:00',
PRIMARY KEY id (id),
KEY id_user_from (id_user_from)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_msgMESSAGE (
id_user_to int(15) unsigned NOT NULL default '0',
id_msgMESSAGE int(15) unsigned NOT NULL default '0',
status char(1) NOT NULL default 'N',
PRIMARY KEY id (id_user_to, id_msgMESSAGE),
KEY id_user_to (id_user_to),
KEY id_msgMESSAGE (id_msgMESSAGE)
) ENGINE=MyISAM;
-- tables for WebComment
CREATE TABLE IF NOT EXISTS cmtRECORDCOMMENT (
id int(15) unsigned NOT NULL auto_increment,
id_bibrec int(15) unsigned NOT NULL default '0',
id_user int(15) unsigned NOT NULL default '0',
title varchar(255) NOT NULL default '',
body text NOT NULL default '',
date_creation datetime NOT NULL default '0000-00-00 00:00:00',
star_score tinyint(5) unsigned NOT NULL default '0',
nb_votes_yes int(10) NOT NULL default '0',
nb_votes_total int(10) unsigned NOT NULL default '0',
nb_abuse_reports int(10) NOT NULL default '0',
status char(2) NOT NULL default 'ok',
round_name varchar(255) NOT NULL default '',
restriction varchar(50) NOT NULL default '',
in_reply_to_id_cmtRECORDCOMMENT int(15) unsigned NOT NULL default '0',
reply_order_cached_data blob NULL default NULL,
PRIMARY KEY (id),
KEY id_bibrec (id_bibrec),
KEY id_user (id_user),
KEY status (status),
KEY in_reply_to_id_cmtRECORDCOMMENT (in_reply_to_id_cmtRECORDCOMMENT),
INDEX (reply_order_cached_data(40))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS cmtACTIONHISTORY (
id_cmtRECORDCOMMENT int(15) unsigned NULL,
id_bibrec int(15) unsigned NULL,
id_user int(15) unsigned NULL default NULL,
client_host int(10) unsigned default NULL,
action_time datetime NOT NULL default '0000-00-00 00:00:00',
action_code char(1) NOT NULL,
KEY id_cmtRECORDCOMMENT (id_cmtRECORDCOMMENT),
KEY client_host (client_host),
KEY id_user (id_user),
KEY action_code (action_code)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS cmtSUBSCRIPTION (
id_bibrec mediumint(8) unsigned NOT NULL,
id_user int(15) unsigned NOT NULL,
creation_time datetime NOT NULL default '0000-00-00 00:00:00',
KEY id_user (id_bibrec, id_user)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS cmtCOLLAPSED (
id_bibrec int(15) unsigned NOT NULL default '0',
id_cmtRECORDCOMMENT int(15) unsigned NULL,
id_user int(15) unsigned NOT NULL,
PRIMARY KEY (id_user, id_bibrec, id_cmtRECORDCOMMENT)
) ENGINE=MyISAM;
-- tables for BibKnowledge:
CREATE TABLE IF NOT EXISTS knwKB (
id mediumint(8) unsigned NOT NULL auto_increment,
name varchar(255) default '',
description text default '',
kbtype char default NULL,
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS knwKBRVAL (
id mediumint(8) unsigned NOT NULL auto_increment,
m_key varchar(255) NOT NULL default '',
m_value text NOT NULL default '',
id_knwKB mediumint(8) NOT NULL default '0',
PRIMARY KEY (id),
KEY id_knwKB (id_knwKB),
KEY m_key (m_key(30)),
KEY m_value (m_value(30))
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS knwKBDDEF (
id_knwKB mediumint(8) unsigned NOT NULL,
id_collection mediumint(9),
output_tag text default '',
search_expression text default '',
PRIMARY KEY (id_knwKB)
) ENGINE=MyISAM;
-- tables for WebSubmit:
CREATE TABLE IF NOT EXISTS sbmACTION (
lactname text,
sactname char(3) NOT NULL default '',
dir text,
cd date default NULL,
md date default NULL,
actionbutton text,
statustext text,
PRIMARY KEY (sactname)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmALLFUNCDESCR (
function varchar(40) NOT NULL default '',
description tinytext,
PRIMARY KEY (function)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmAPPROVAL (
doctype varchar(10) NOT NULL default '',
categ varchar(50) NOT NULL default '',
rn varchar(50) NOT NULL default '',
status varchar(10) NOT NULL default '',
dFirstReq datetime NOT NULL default '0000-00-00 00:00:00',
dLastReq datetime NOT NULL default '0000-00-00 00:00:00',
dAction datetime NOT NULL default '0000-00-00 00:00:00',
access varchar(20) NOT NULL default '0',
note text NOT NULL default '',
PRIMARY KEY (rn)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCPLXAPPROVAL (
doctype varchar(10) NOT NULL default '',
categ varchar(50) NOT NULL default '',
rn varchar(50) NOT NULL default '',
type varchar(10) NOT NULL,
status varchar(10) NOT NULL,
id_group int(15) unsigned NOT NULL default '0',
id_bskBASKET int(15) unsigned NOT NULL default '0',
id_EdBoardGroup int(15) unsigned NOT NULL default '0',
dFirstReq datetime NOT NULL default '0000-00-00 00:00:00',
dLastReq datetime NOT NULL default '0000-00-00 00:00:00',
dEdBoardSel datetime NOT NULL default '0000-00-00 00:00:00',
dRefereeSel datetime NOT NULL default '0000-00-00 00:00:00',
dRefereeRecom datetime NOT NULL default '0000-00-00 00:00:00',
dEdBoardRecom datetime NOT NULL default '0000-00-00 00:00:00',
dPubComRecom datetime NOT NULL default '0000-00-00 00:00:00',
dProjectLeaderAction datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (rn, type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCOLLECTION (
id int(11) NOT NULL auto_increment,
name varchar(100) NOT NULL default '',
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCOLLECTION_sbmCOLLECTION (
id_father int(11) NOT NULL default '0',
id_son int(11) NOT NULL default '0',
catalogue_order int(11) NOT NULL default '0'
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCOLLECTION_sbmDOCTYPE (
id_father int(11) NOT NULL default '0',
id_son char(10) NOT NULL default '0',
catalogue_order int(11) NOT NULL default '0'
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCATEGORIES (
doctype varchar(10) NOT NULL default '',
sname varchar(75) NOT NULL default '',
lname varchar(75) NOT NULL default '',
score tinyint unsigned NOT NULL default 0,
PRIMARY KEY (doctype, sname),
KEY doctype (doctype),
KEY sname (sname)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCHECKS (
chname varchar(15) NOT NULL default '',
chdesc text,
cd date default NULL,
md date default NULL,
chefi1 text,
chefi2 text,
PRIMARY KEY (chname)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmDOCTYPE (
ldocname text,
sdocname varchar(10) default NULL,
cd date default NULL,
md date default NULL,
description text
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmFIELD (
subname varchar(13) default NULL,
pagenb int(11) default NULL,
fieldnb int(11) default NULL,
fidesc varchar(15) default NULL,
fitext text,
level char(1) default NULL,
sdesc text,
checkn text,
cd date default NULL,
md date default NULL,
fiefi1 text,
fiefi2 text
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmFIELDDESC (
name varchar(15) NOT NULL default '',
alephcode varchar(50) default NULL,
marccode varchar(50) NOT NULL default '',
type char(1) default NULL,
size int(11) default NULL,
rows int(11) default NULL,
cols int(11) default NULL,
maxlength int(11) default NULL,
val text,
fidesc text,
cd date default NULL,
md date default NULL,
modifytext text,
fddfi2 text,
cookie int(11) default '0',
PRIMARY KEY (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmFORMATEXTENSION (
FILE_FORMAT text NOT NULL,
FILE_EXTENSION text NOT NULL
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmFUNCTIONS (
action varchar(10) NOT NULL default '',
doctype varchar(10) NOT NULL default '',
function varchar(40) NOT NULL default '',
score int(11) NOT NULL default '0',
step tinyint(4) NOT NULL default '1'
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmFUNDESC (
function varchar(40) NOT NULL default '',
param varchar(40) default NULL
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmGFILERESULT (
FORMAT text NOT NULL,
RESULT text NOT NULL
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmIMPLEMENT (
docname varchar(10) default NULL,
actname char(3) default NULL,
displayed char(1) default NULL,
subname varchar(13) default NULL,
nbpg int(11) default NULL,
cd date default NULL,
md date default NULL,
buttonorder int(11) default NULL,
statustext text,
level char(1) NOT NULL default '',
score int(11) NOT NULL default '0',
stpage int(11) NOT NULL default '0',
endtxt varchar(100) NOT NULL default ''
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmPARAMETERS (
doctype varchar(10) NOT NULL default '',
name varchar(40) NOT NULL default '',
value text NOT NULL default '',
PRIMARY KEY (doctype,name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmPUBLICATION (
doctype varchar(10) NOT NULL default '',
categ varchar(50) NOT NULL default '',
rn varchar(50) NOT NULL default '',
status varchar(10) NOT NULL default '',
dFirstReq datetime NOT NULL default '0000-00-00 00:00:00',
dLastReq datetime NOT NULL default '0000-00-00 00:00:00',
dAction datetime NOT NULL default '0000-00-00 00:00:00',
accessref varchar(20) NOT NULL default '',
accessedi varchar(20) NOT NULL default '',
access varchar(20) NOT NULL default '',
referees varchar(50) NOT NULL default '',
authoremail varchar(50) NOT NULL default '',
dRefSelection datetime NOT NULL default '0000-00-00 00:00:00',
dRefRec datetime NOT NULL default '0000-00-00 00:00:00',
dEdiRec datetime NOT NULL default '0000-00-00 00:00:00',
accessspo varchar(20) NOT NULL default '',
journal varchar(100) default NULL,
PRIMARY KEY (doctype,categ,rn)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmPUBLICATIONCOMM (
id int(11) NOT NULL auto_increment,
id_parent int(11) default '0',
rn varchar(100) NOT NULL default '',
firstname varchar(100) default NULL,
secondname varchar(100) default NULL,
email varchar(100) default NULL,
date varchar(40) NOT NULL default '',
synopsis varchar(255) NOT NULL default '',
commentfulltext text,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmPUBLICATIONDATA (
doctype varchar(10) NOT NULL default '',
editoboard varchar(250) NOT NULL default '',
base varchar(10) NOT NULL default '',
logicalbase varchar(10) NOT NULL default '',
spokesperson varchar(50) NOT NULL default '',
PRIMARY KEY (doctype)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmREFEREES (
doctype varchar(10) NOT NULL default '',
categ varchar(10) NOT NULL default '',
name varchar(50) NOT NULL default '',
address varchar(50) NOT NULL default '',
rid int(11) NOT NULL auto_increment,
PRIMARY KEY (rid)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmSUBMISSIONS (
email varchar(50) NOT NULL default '',
doctype varchar(10) NOT NULL default '',
action varchar(10) NOT NULL default '',
status varchar(10) NOT NULL default '',
id varchar(30) NOT NULL default '',
reference varchar(40) NOT NULL default '',
cd datetime NOT NULL default '0000-00-00 00:00:00',
md datetime NOT NULL default '0000-00-00 00:00:00'
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS sbmCOOKIES (
id int(15) unsigned NOT NULL auto_increment,
name varchar(100) NOT NULL,
value text,
uid int(15) NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- Scheduler tables
CREATE TABLE IF NOT EXISTS schTASK (
id int(15) unsigned NOT NULL auto_increment,
proc varchar(255) NOT NULL,
host varchar(255) NOT NULL default '',
user varchar(50) NOT NULL,
runtime datetime NOT NULL,
sleeptime varchar(20),
arguments mediumblob,
status varchar(50),
progress varchar(255),
priority tinyint(4) NOT NULL default 0,
sequenceid int(8) NULL default NULL,
PRIMARY KEY (id),
KEY status (status),
KEY runtime (runtime),
KEY priority (priority),
KEY sequenceid (sequenceid)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS hstTASK (
id int(15) unsigned NOT NULL,
proc varchar(255) NOT NULL,
host varchar(255) NOT NULL default '',
user varchar(50) NOT NULL,
runtime datetime NOT NULL,
sleeptime varchar(20),
arguments mediumblob,
status varchar(50),
progress varchar(255),
priority tinyint(4) NOT NULL default 0,
sequenceid int(8) NULL default NULL,
PRIMARY KEY (id),
KEY status (status),
KEY runtime (runtime),
KEY priority (priority),
KEY sequenceid (sequenceid)
) ENGINE=MyISAM;
-- Batch Upload History
CREATE TABLE IF NOT EXISTS hstBATCHUPLOAD (
id int(15) unsigned NOT NULL auto_increment,
user varchar(50) NOT NULL,
submitdate datetime NOT NULL,
filename varchar(255) NOT NULL,
execdate datetime NOT NULL,
id_schTASK int(15) unsigned NOT NULL,
batch_mode varchar(15) NOT NULL,
PRIMARY KEY (id),
KEY user (user)
) ENGINE=MyISAM;
-- External collections
CREATE TABLE IF NOT EXISTS collection_externalcollection (
id_collection mediumint(9) unsigned NOT NULL default '0',
id_externalcollection mediumint(9) unsigned NOT NULL default '0',
type tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection, id_externalcollection)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS externalcollection (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL default '',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
-- WebStat tables:
CREATE TABLE IF NOT EXISTS staEVENT (
id varchar(255) NOT NULL,
number smallint(2) unsigned ZEROFILL NOT NULL auto_increment,
name varchar(255),
creation_time TIMESTAMP DEFAULT NOW(),
cols varchar(255),
PRIMARY KEY (id),
UNIQUE KEY number (number)
) ENGINE=MyISAM;
-- BibClassify tables:
CREATE TABLE IF NOT EXISTS clsMETHOD (
id mediumint(9) unsigned NOT NULL,
name varchar(50) NOT NULL default '',
location varchar(255) NOT NULL default '',
description varchar(255) NOT NULL default '',
last_updated datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS collection_clsMETHOD (
id_collection mediumint(9) unsigned NOT NULL,
id_clsMETHOD mediumint(9) unsigned NOT NULL,
PRIMARY KEY (id_collection, id_clsMETHOD)
) ENGINE=MyISAM;
-- WebJournal tables:
CREATE TABLE IF NOT EXISTS jrnJOURNAL (
id mediumint(9) unsigned NOT NULL auto_increment,
name varchar(50) NOT NULL default '',
PRIMARY KEY (id),
UNIQUE KEY name (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS jrnISSUE (
id_jrnJOURNAL mediumint(9) unsigned NOT NULL,
issue_number varchar(50) NOT NULL default '',
issue_display varchar(50) NOT NULL default '',
date_released datetime NOT NULL default '0000-00-00 00:00:00',
date_announced datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id_jrnJOURNAL,issue_number)
) ENGINE=MyISAM;
-- tables recording history of record's metadata and fulltext documents:
CREATE TABLE IF NOT EXISTS hstRECORD (
id_bibrec mediumint(8) unsigned NOT NULL,
marcxml blob NOT NULL,
job_id mediumint(15) unsigned NOT NULL,
job_name varchar(255) NOT NULL,
job_person varchar(255) NOT NULL,
job_date datetime NOT NULL,
job_details blob NOT NULL,
KEY (id_bibrec),
KEY (job_id),
KEY (job_name),
KEY (job_person),
KEY (job_date)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS hstDOCUMENT (
id_bibdoc mediumint(9) unsigned NOT NULL,
docname varchar(250) NOT NULL,
docformat varchar(50) NOT NULL,
docversion tinyint(4) unsigned NOT NULL,
docsize bigint(15) unsigned NOT NULL,
docchecksum char(32) NOT NULL,
doctimestamp datetime NOT NULL,
action varchar(50) NOT NULL,
job_id mediumint(15) unsigned NULL default NULL,
job_name varchar(255) NULL default NULL,
job_person varchar(255) NULL default NULL,
job_date datetime NULL default NULL,
job_details blob NULL default NULL,
KEY (action),
KEY (id_bibdoc),
KEY (docname),
KEY (docformat),
KEY (doctimestamp),
KEY (job_id),
KEY (job_name),
KEY (job_person),
KEY (job_date)
) ENGINE=MyISAM;
-- BibCirculation tables:
CREATE TABLE IF NOT EXISTS crcBORROWER (
id int(15) unsigned NOT NULL auto_increment,
ccid int(15) unsigned NULL default NULL,
name varchar(255) NOT NULL default '',
email varchar(255) NOT NULL default '',
phone varchar(60) default NULL,
address varchar(60) default NULL,
mailbox varchar(30) default NULL,
borrower_since datetime NOT NULL default '0000-00-00 00:00:00',
borrower_until datetime NOT NULL default '0000-00-00 00:00:00',
notes text,
PRIMARY KEY (id),
UNIQUE KEY (ccid),
KEY (name),
KEY (email)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcILLREQUEST (
id int(15) unsigned NOT NULL auto_increment,
id_crcBORROWER int(15) unsigned NOT NULL default '0',
barcode varchar(30) NOT NULL default '',
period_of_interest_from datetime NOT NULL default '0000-00-00 00:00:00',
period_of_interest_to datetime NOT NULL default '0000-00-00 00:00:00',
id_crcLIBRARY int(15) unsigned NOT NULL default '0',
request_date datetime NOT NULL default '0000-00-00 00:00:00',
expected_date datetime NOT NULL default '0000-00-00 00:00:00',
arrival_date datetime NOT NULL default '0000-00-00 00:00:00',
due_date datetime NOT NULL default '0000-00-00 00:00:00',
return_date datetime NOT NULL default '0000-00-00 00:00:00',
status varchar(20) NOT NULL default '',
cost varchar(30) NOT NULL default '',
budget_code varchar(60) NOT NULL default '',
item_info text,
request_type text,
borrower_comments text,
only_this_edition varchar(10) NOT NULL default '',
library_notes text,
PRIMARY KEY (id),
KEY id_crcborrower (id_crcBORROWER),
KEY id_crclibrary (id_crcLIBRARY)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcITEM (
barcode varchar(30) NOT NULL default '',
id_bibrec int(15) unsigned NOT NULL default '0',
id_crcLIBRARY int(15) unsigned NOT NULL default '0',
collection varchar(60) default NULL,
location varchar(60) default NULL,
description varchar(60) default NULL,
loan_period varchar(30) NOT NULL default '',
status varchar(20) NOT NULL default '',
expected_arrival_date varchar(60) NOT NULL default '',
creation_date datetime NOT NULL default '0000-00-00 00:00:00',
modification_date datetime NOT NULL default '0000-00-00 00:00:00',
number_of_requests int(3) unsigned NOT NULL default '0',
PRIMARY KEY (barcode),
KEY id_bibrec (id_bibrec),
KEY id_crclibrary (id_crcLIBRARY)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcLIBRARY (
id int(15) unsigned NOT NULL auto_increment,
name varchar(80) NOT NULL default '',
address varchar(255) NOT NULL default '',
email varchar(255) NOT NULL default '',
phone varchar(30) NOT NULL default '',
type varchar(30) default NULL,
notes text,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcLOAN (
id int(15) unsigned NOT NULL auto_increment,
id_crcBORROWER int(15) unsigned NOT NULL default '0',
id_bibrec int(15) unsigned NOT NULL default '0',
barcode varchar(30) NOT NULL default '',
loaned_on datetime NOT NULL default '0000-00-00 00:00:00',
returned_on date NOT NULL default '0000-00-00',
due_date datetime NOT NULL default '0000-00-00 00:00:00',
number_of_renewals int(3) unsigned NOT NULL default '0',
overdue_letter_number int(3) unsigned NOT NULL default '0',
overdue_letter_date datetime NOT NULL default '0000-00-00 00:00:00',
status varchar(20) NOT NULL default '',
type varchar(20) NOT NULL default '',
notes text,
PRIMARY KEY (id),
KEY id_crcborrower (id_crcBORROWER),
KEY id_bibrec (id_bibrec),
KEY barcode (barcode)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcLOANREQUEST (
id int(15) unsigned NOT NULL auto_increment,
id_crcBORROWER int(15) unsigned NOT NULL default '0',
id_bibrec int(15) unsigned NOT NULL default '0',
barcode varchar(30) NOT NULL default '',
period_of_interest_from datetime NOT NULL default '0000-00-00 00:00:00',
period_of_interest_to datetime NOT NULL default '0000-00-00 00:00:00',
status varchar(20) NOT NULL default '',
notes text,
request_date datetime NOT NULL default '0000-00-00 00:00:00',
PRIMARY KEY (id),
KEY id_crcborrower (id_crcBORROWER),
KEY id_bibrec (id_bibrec),
KEY barcode (barcode)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcPURCHASE (
id int(15) unsigned NOT NULL auto_increment,
id_bibrec int(15) unsigned NOT NULL default '0',
id_crcVENDOR int(15) unsigned NOT NULL default '0',
ordered_date datetime NOT NULL default '0000-00-00 00:00:00',
expected_date datetime NOT NULL default '0000-00-00 00:00:00',
price varchar(20) NOT NULL default '0',
status varchar(20) NOT NULL default '',
notes text,
PRIMARY KEY (id),
KEY id_bibrec (id_bibrec),
KEY id_crcVENDOR (id_crcVENDOR)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS crcVENDOR (
id int(15) unsigned NOT NULL auto_increment,
name varchar(80) NOT NULL default '',
address varchar(255) NOT NULL default '',
email varchar(255) NOT NULL default '',
phone varchar(30) NOT NULL default '',
notes text,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- BibExport tables:
CREATE TABLE IF NOT EXISTS expJOB (
id int(15) unsigned NOT NULL auto_increment,
jobname varchar(50) NOT NULL default '',
jobfreq mediumint(12) NOT NULL default '0',
output_format mediumint(12) NOT NULL default '0',
deleted mediumint(12) NOT NULL default '0',
lastrun datetime NOT NULL default '0000-00-00 00:00:00',
output_directory text,
PRIMARY KEY (id),
UNIQUE KEY jobname (jobname)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS expQUERY (
id int(15) unsigned NOT NULL auto_increment,
name varchar(255) NOT NULL,
search_criteria text NOT NULL,
output_fields text NOT NULL,
notes text,
deleted mediumint(12) NOT NULL default '0',
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS expJOB_expQUERY (
id_expJOB int(15) NOT NULL,
id_expQUERY int(15) NOT NULL,
PRIMARY KEY (id_expJOB,id_expQUERY),
KEY id_expJOB (id_expJOB),
KEY id_expQUERY (id_expQUERY)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS expQUERYRESULT (
id int(15) unsigned NOT NULL auto_increment,
id_expQUERY int(15) NOT NULL,
result text NOT NULL,
status mediumint(12) NOT NULL default '0',
status_message text NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS expJOBRESULT (
id int(15) unsigned NOT NULL auto_increment,
id_expJOB int(15) NOT NULL,
execution_time datetime NOT NULL default '0000-00-00 00:00:00',
status mediumint(12) NOT NULL default '0',
status_message text NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS expJOBRESULT_expQUERYRESULT (
id_expJOBRESULT int(15) NOT NULL,
id_expQUERYRESULT int(15) NOT NULL,
PRIMARY KEY (id_expJOBRESULT, id_expQUERYRESULT),
KEY id_expJOBRESULT (id_expJOBRESULT),
KEY id_expQUERYRESULT (id_expQUERYRESULT)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS user_expJOB (
id_user int(15) NOT NULL,
id_expJOB int(15) NOT NULL,
PRIMARY KEY (id_user, id_expJOB),
KEY id_user (id_user),
KEY id_expJOB (id_expJOB)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS swrREMOTESERVER (
id int(15) unsigned NOT NULL auto_increment,
name varchar(50) unique NOT NULL,
host varchar(50) NOT NULL,
username varchar(50) NOT NULL,
password varchar(50) NOT NULL,
email varchar(50) NOT NULL,
realm varchar(50) NOT NULL,
url_base_record varchar(50) NOT NULL,
url_servicedocument varchar(80) NOT NULL,
xml_servicedocument longblob,
last_update int(15) unsigned NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS swrCLIENTDATA (
id int(15) unsigned NOT NULL auto_increment,
id_swrREMOTESERVER int(15) NOT NULL,
id_record int(15) NOT NULL,
report_no varchar(50) NOT NULL,
id_remote varchar(50) NOT NULL,
id_user int(15) NOT NULL,
user_name varchar(100) NOT NULL,
user_email varchar(100) NOT NULL,
xml_media_deposit longblob NOT NULL,
xml_metadata_submit longblob NOT NULL,
submission_date datetime NOT NULL default '0000-00-00 00:00:00',
publication_date datetime NOT NULL default '0000-00-00 00:00:00',
removal_date datetime NOT NULL default '0000-00-00 00:00:00',
link_medias varchar(150) NOT NULL,
link_metadata varchar(150) NOT NULL,
link_status varchar(150) NOT NULL,
status varchar(150) NOT NULL default 'submitted',
last_update datetime NOT NULL,
PRIMARY KEY (id)
) ENGINE=MyISAM;
-- tables for exception management
-- This table is used to log exceptions
-- to discover the full details of an exception either check the email
-- that are sent to CFG_SITE_ADMIN_EMAIL or look into invenio.err
CREATE TABLE IF NOT EXISTS hstEXCEPTION (
id int(15) unsigned NOT NULL auto_increment,
name varchar(50) NOT NULL, -- name of the exception
filename varchar(255) NULL, -- file where the exception was raised
line int(9) NULL, -- line at which the exception was raised
last_seen datetime NOT NULL default '0000-00-00 00:00:00', -- last time this exception has been seen
last_notified datetime NOT NULL default '0000-00-00 00:00:00', -- last time this exception has been notified
counter int(15) NOT NULL default 0, -- internal counter to decide when to notify this exception
total int(15) NOT NULL default 0, -- total number of times this exception has been seen
PRIMARY KEY (id),
KEY (last_seen),
KEY (last_notified),
KEY (total),
UNIQUE KEY (name(50), filename(255), line)
) ENGINE=MyISAM;
-- tables for BibAuthorID module:
CREATE TABLE IF NOT EXISTS `aidPERSONIDPAPERS` (
`personid` BIGINT( 16 ) UNSIGNED NOT NULL ,
`bibref_table` ENUM( '100', '700' ) NOT NULL ,
`bibref_value` MEDIUMINT( 8 ) UNSIGNED NOT NULL ,
`bibrec` MEDIUMINT( 8 ) UNSIGNED NOT NULL ,
`name` VARCHAR( 256 ) NOT NULL ,
`flag` SMALLINT( 2 ) NOT NULL DEFAULT '0' ,
`lcul` SMALLINT( 2 ) NOT NULL DEFAULT '0' ,
`last_updated` TIMESTAMP ON UPDATE CURRENT_TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ,
INDEX `personid-b` (`personid`) ,
INDEX `reftable-b` (`bibref_table`) ,
INDEX `refvalue-b` (`bibref_value`) ,
INDEX `rec-b` (`bibrec`) ,
INDEX `name-b` (`name`) ,
INDEX `pn-b` (`personid`, `name`) ,
INDEX `timestamp-b` (`last_updated`) ,
INDEX `flag-b` (`flag`) ,
INDEX `ptvrf-b` (`personid`, `bibref_table`, `bibref_value`, `bibrec`, `flag`)
) ENGINE=MYISAM;
CREATE TABLE IF NOT EXISTS `aidRESULTS` (
`personid` VARCHAR( 256 ) NOT NULL ,
`bibref_table` ENUM( '100', '700' ) NOT NULL ,
`bibref_value` MEDIUMINT( 8 ) UNSIGNED NOT NULL ,
`bibrec` MEDIUMINT( 8 ) UNSIGNED NOT NULL ,
INDEX `personid-b` (`personid`) ,
INDEX `reftable-b` (`bibref_table`) ,
INDEX `refvalue-b` (`bibref_value`) ,
INDEX `rec-b` (`bibrec`)
) ENGINE=MYISAM;
CREATE TABLE IF NOT EXISTS `aidPERSONIDDATA` (
`personid` BIGINT( 16 ) UNSIGNED NOT NULL ,
`tag` VARCHAR( 64 ) NOT NULL ,
`data` VARCHAR( 256 ) NOT NULL ,
`opt1` MEDIUMINT( 8 ) NULL DEFAULT NULL ,
`opt2` MEDIUMINT( 8 ) NULL DEFAULT NULL ,
`opt3` VARCHAR( 256 ) NULL DEFAULT NULL ,
INDEX `personid-b` (`personid`) ,
INDEX `tag-b` (`tag`) ,
INDEX `data-b` (`data`) ,
INDEX `opt1` (`opt1`)
) ENGINE=MYISAM;
CREATE TABLE IF NOT EXISTS `aidUSERINPUTLOG` (
`id` bigint(15) NOT NULL AUTO_INCREMENT,
`transactionid` bigint(15) NOT NULL,
`timestamp` datetime NOT NULL,
+ `userid` int,
`userinfo` varchar(255) NOT NULL,
`personid` bigint(15) NOT NULL,
`action` varchar(50) NOT NULL,
`tag` varchar(50) NOT NULL,
`value` varchar(200) NOT NULL,
`comment` text,
PRIMARY KEY (`id`),
INDEX `transactionid-b` (`transactionid`),
INDEX `timestamp-b` (`timestamp`),
INDEX `userinfo-b` (`userinfo`),
+ INDEX `userid-b` (`userid`),
INDEX `personid-b` (`personid`),
INDEX `action-b` (`action`),
INDEX `tag-b` (`tag`),
INDEX `value-b` (`value`)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS `aidCACHE` (
`id` int(15) NOT NULL auto_increment,
`object_name` varchar(120) NOT NULL,
`object_key` varchar(120) NOT NULL,
`object_value` text,
`last_updated` datetime NOT NULL,
PRIMARY KEY (`id`),
INDEX `name-b` (`object_name`),
INDEX `key-b` (`object_key`),
INDEX `last_updated-b` (`last_updated`)
) ENGINE=MyISAM;
-CREATE TABLE IF NOT EXISTS `aidPROBCACHE` (
- `cluster` VARCHAR( 256 ) NOT NULL ,
- `bibmap` MEDIUMBLOB NOT NULL ,
- `matrix` LONGBLOB NOT NULL ,
- PRIMARY KEY ( `cluster` )
-) ENGINE = MYISAM ;
-
-- refextract tables:
CREATE TABLE IF NOT EXISTS `xtrJOB` (
`id` tinyint(4) NOT NULL AUTO_INCREMENT,
`name` varchar(30) NOT NULL,
`last_updated` datetime NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM;
-- tables for bibsort module
CREATE TABLE IF NOT EXISTS bsrMETHOD (
id mediumint(8) unsigned NOT NULL auto_increment,
name varchar(20) NOT NULL,
definition varchar(255) NOT NULL,
washer varchar(255) NOT NULL,
PRIMARY KEY (id),
UNIQUE KEY (name)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bsrMETHODNAME (
id_bsrMETHOD mediumint(8) unsigned NOT NULL,
ln char(5) NOT NULL default '',
type char(3) NOT NULL default 'sn',
value varchar(255) NOT NULL,
PRIMARY KEY (id_bsrMETHOD, ln, type)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bsrMETHODDATA (
id_bsrMETHOD mediumint(8) unsigned NOT NULL,
data_dict longblob,
data_dict_ordered longblob,
data_list_sorted longblob,
last_updated datetime,
PRIMARY KEY (id_bsrMETHOD)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS bsrMETHODDATABUCKET (
id_bsrMETHOD mediumint(8) unsigned NOT NULL,
bucket_no tinyint(2) NOT NULL,
bucket_data longblob,
bucket_last_value varchar(255),
last_updated datetime,
PRIMARY KEY (id_bsrMETHOD, bucket_no)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS collection_bsrMETHOD (
id_collection mediumint(9) unsigned NOT NULL,
id_bsrMETHOD mediumint(9) unsigned NOT NULL,
score tinyint(4) unsigned NOT NULL default '0',
PRIMARY KEY (id_collection, id_bsrMETHOD)
) ENGINE=MyISAM;
-- tables for sequence storage
CREATE TABLE IF NOT EXISTS seqSTORE (
id int(15) NOT NULL auto_increment,
seq_name varchar(15),
seq_value varchar(20),
PRIMARY KEY (id),
UNIQUE KEY seq_name_value (seq_name, seq_value)
) ENGINE=MyISAM;
-- tables for linkbacks:
CREATE TABLE IF NOT EXISTS lnkENTRY (
id int(15) NOT NULL auto_increment,
origin_url varchar(100) NOT NULL, -- url of the originating resource
id_bibrec mediumint(8) unsigned NOT NULL, -- bibrecord
additional_properties longblob,
type varchar(30) NOT NULL,
status varchar(30) NOT NULL default 'PENDING',
insert_time datetime default '0000-00-00 00:00:00',
PRIMARY KEY (id),
INDEX (id_bibrec),
INDEX (type),
INDEX (status),
INDEX (insert_time)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS lnkENTRYURLTITLE (
id int(15) unsigned NOT NULL auto_increment,
url varchar(100) NOT NULL,
title varchar(100) NOT NULL,
manual_set boolean NOT NULL default 0,
broken_count int(5) default 0,
broken boolean NOT NULL default 0,
PRIMARY KEY (id),
UNIQUE (url),
INDEX (title)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS lnkENTRYLOG (
id_lnkENTRY int(15) unsigned NOT NULL,
id_lnkLOG int(15) unsigned NOT NULL,
FOREIGN KEY (id_lnkENTRY) REFERENCES lnkENTRY(id),
FOREIGN KEY (id_lnkLOG) REFERENCES lnkLOG(id)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS lnkLOG (
id int(15) unsigned NOT NULL auto_increment,
id_user int(15) unsigned,
action varchar(30) NOT NULL,
log_time datetime default '0000-00-00 00:00:00',
PRIMARY KEY (id),
INDEX (id_user),
INDEX (action),
INDEX (log_time)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS lnkADMINURL (
id int(15) unsigned NOT NULL auto_increment,
url varchar(100) NOT NULL,
list varchar(30) NOT NULL,
PRIMARY KEY (id),
UNIQUE (url),
INDEX (list)
) ENGINE=MyISAM;
CREATE TABLE IF NOT EXISTS lnkADMINURLLOG (
id_lnkADMINURL int(15) unsigned NOT NULL,
id_lnkLOG int(15) unsigned NOT NULL,
FOREIGN KEY (id_lnkADMINURL) REFERENCES lnkADMINURL(id),
FOREIGN KEY (id_lnkLOG) REFERENCES lnkLOG(id)
) ENGINE=MyISAM;
+-- table for API key
+
+CREATE TABLE IF NOT EXISTS webapikey (
+ id varchar(150) NOT NULL,
+ secret varchar(150) NOT NULL,
+ id_user int(15) NOT NULL,
+ status varchar(25) NOT NULL default 'OK',
+ description varchar(255) default NULL,
+ PRIMARY KEY (id),
+ KEY (id_user),
+ KEY (status)
+) ENGINE=MyISAM;
+
-- end of file
diff --git a/modules/miscutil/sql/tabdrop.sql b/modules/miscutil/sql/tabdrop.sql
index 7122ee01e..80295c749 100644
--- a/modules/miscutil/sql/tabdrop.sql
+++ b/modules/miscutil/sql/tabdrop.sql
@@ -1,475 +1,475 @@
-- $Id$
-- This file is part of Invenio.
-- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
--
-- Invenio is free software; you can redistribute it and/or
-- modify it under the terms of the GNU General Public License as
-- published by the Free Software Foundation; either version 2 of the
-- License, or (at your option) any later version.
--
-- Invenio is distributed in the hope that it will be useful, but
-- WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-- General Public License for more details.
--
-- You should have received a copy of the GNU General Public License
-- along with Invenio; if not, write to the Free Software Foundation, Inc.,
-- 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
DROP TABLE IF EXISTS bibrec;
DROP TABLE IF EXISTS bib00x;
DROP TABLE IF EXISTS bib01x;
DROP TABLE IF EXISTS bib02x;
DROP TABLE IF EXISTS bib03x;
DROP TABLE IF EXISTS bib04x;
DROP TABLE IF EXISTS bib05x;
DROP TABLE IF EXISTS bib06x;
DROP TABLE IF EXISTS bib07x;
DROP TABLE IF EXISTS bib08x;
DROP TABLE IF EXISTS bib09x;
DROP TABLE IF EXISTS bib10x;
DROP TABLE IF EXISTS bib11x;
DROP TABLE IF EXISTS bib12x;
DROP TABLE IF EXISTS bib13x;
DROP TABLE IF EXISTS bib14x;
DROP TABLE IF EXISTS bib15x;
DROP TABLE IF EXISTS bib16x;
DROP TABLE IF EXISTS bib17x;
DROP TABLE IF EXISTS bib18x;
DROP TABLE IF EXISTS bib19x;
DROP TABLE IF EXISTS bib20x;
DROP TABLE IF EXISTS bib21x;
DROP TABLE IF EXISTS bib22x;
DROP TABLE IF EXISTS bib23x;
DROP TABLE IF EXISTS bib24x;
DROP TABLE IF EXISTS bib25x;
DROP TABLE IF EXISTS bib26x;
DROP TABLE IF EXISTS bib27x;
DROP TABLE IF EXISTS bib28x;
DROP TABLE IF EXISTS bib29x;
DROP TABLE IF EXISTS bib30x;
DROP TABLE IF EXISTS bib31x;
DROP TABLE IF EXISTS bib32x;
DROP TABLE IF EXISTS bib33x;
DROP TABLE IF EXISTS bib34x;
DROP TABLE IF EXISTS bib35x;
DROP TABLE IF EXISTS bib36x;
DROP TABLE IF EXISTS bib37x;
DROP TABLE IF EXISTS bib38x;
DROP TABLE IF EXISTS bib39x;
DROP TABLE IF EXISTS bib40x;
DROP TABLE IF EXISTS bib41x;
DROP TABLE IF EXISTS bib42x;
DROP TABLE IF EXISTS bib43x;
DROP TABLE IF EXISTS bib44x;
DROP TABLE IF EXISTS bib45x;
DROP TABLE IF EXISTS bib46x;
DROP TABLE IF EXISTS bib47x;
DROP TABLE IF EXISTS bib48x;
DROP TABLE IF EXISTS bib49x;
DROP TABLE IF EXISTS bib50x;
DROP TABLE IF EXISTS bib51x;
DROP TABLE IF EXISTS bib52x;
DROP TABLE IF EXISTS bib53x;
DROP TABLE IF EXISTS bib54x;
DROP TABLE IF EXISTS bib55x;
DROP TABLE IF EXISTS bib56x;
DROP TABLE IF EXISTS bib57x;
DROP TABLE IF EXISTS bib58x;
DROP TABLE IF EXISTS bib59x;
DROP TABLE IF EXISTS bib60x;
DROP TABLE IF EXISTS bib61x;
DROP TABLE IF EXISTS bib62x;
DROP TABLE IF EXISTS bib63x;
DROP TABLE IF EXISTS bib64x;
DROP TABLE IF EXISTS bib65x;
DROP TABLE IF EXISTS bib66x;
DROP TABLE IF EXISTS bib67x;
DROP TABLE IF EXISTS bib68x;
DROP TABLE IF EXISTS bib69x;
DROP TABLE IF EXISTS bib70x;
DROP TABLE IF EXISTS bib71x;
DROP TABLE IF EXISTS bib72x;
DROP TABLE IF EXISTS bib73x;
DROP TABLE IF EXISTS bib74x;
DROP TABLE IF EXISTS bib75x;
DROP TABLE IF EXISTS bib76x;
DROP TABLE IF EXISTS bib77x;
DROP TABLE IF EXISTS bib78x;
DROP TABLE IF EXISTS bib79x;
DROP TABLE IF EXISTS bib80x;
DROP TABLE IF EXISTS bib81x;
DROP TABLE IF EXISTS bib82x;
DROP TABLE IF EXISTS bib83x;
DROP TABLE IF EXISTS bib84x;
DROP TABLE IF EXISTS bib85x;
DROP TABLE IF EXISTS bib86x;
DROP TABLE IF EXISTS bib87x;
DROP TABLE IF EXISTS bib88x;
DROP TABLE IF EXISTS bib89x;
DROP TABLE IF EXISTS bib90x;
DROP TABLE IF EXISTS bib91x;
DROP TABLE IF EXISTS bib92x;
DROP TABLE IF EXISTS bib93x;
DROP TABLE IF EXISTS bib94x;
DROP TABLE IF EXISTS bib95x;
DROP TABLE IF EXISTS bib96x;
DROP TABLE IF EXISTS bib97x;
DROP TABLE IF EXISTS bib98x;
DROP TABLE IF EXISTS bib99x;
DROP TABLE IF EXISTS bibrec_bib00x;
DROP TABLE IF EXISTS bibrec_bib01x;
DROP TABLE IF EXISTS bibrec_bib02x;
DROP TABLE IF EXISTS bibrec_bib03x;
DROP TABLE IF EXISTS bibrec_bib04x;
DROP TABLE IF EXISTS bibrec_bib05x;
DROP TABLE IF EXISTS bibrec_bib06x;
DROP TABLE IF EXISTS bibrec_bib07x;
DROP TABLE IF EXISTS bibrec_bib08x;
DROP TABLE IF EXISTS bibrec_bib09x;
DROP TABLE IF EXISTS bibrec_bib10x;
DROP TABLE IF EXISTS bibrec_bib11x;
DROP TABLE IF EXISTS bibrec_bib12x;
DROP TABLE IF EXISTS bibrec_bib13x;
DROP TABLE IF EXISTS bibrec_bib14x;
DROP TABLE IF EXISTS bibrec_bib15x;
DROP TABLE IF EXISTS bibrec_bib16x;
DROP TABLE IF EXISTS bibrec_bib17x;
DROP TABLE IF EXISTS bibrec_bib18x;
DROP TABLE IF EXISTS bibrec_bib19x;
DROP TABLE IF EXISTS bibrec_bib20x;
DROP TABLE IF EXISTS bibrec_bib21x;
DROP TABLE IF EXISTS bibrec_bib22x;
DROP TABLE IF EXISTS bibrec_bib23x;
DROP TABLE IF EXISTS bibrec_bib24x;
DROP TABLE IF EXISTS bibrec_bib25x;
DROP TABLE IF EXISTS bibrec_bib26x;
DROP TABLE IF EXISTS bibrec_bib27x;
DROP TABLE IF EXISTS bibrec_bib28x;
DROP TABLE IF EXISTS bibrec_bib29x;
DROP TABLE IF EXISTS bibrec_bib30x;
DROP TABLE IF EXISTS bibrec_bib31x;
DROP TABLE IF EXISTS bibrec_bib32x;
DROP TABLE IF EXISTS bibrec_bib33x;
DROP TABLE IF EXISTS bibrec_bib34x;
DROP TABLE IF EXISTS bibrec_bib35x;
DROP TABLE IF EXISTS bibrec_bib36x;
DROP TABLE IF EXISTS bibrec_bib37x;
DROP TABLE IF EXISTS bibrec_bib38x;
DROP TABLE IF EXISTS bibrec_bib39x;
DROP TABLE IF EXISTS bibrec_bib40x;
DROP TABLE IF EXISTS bibrec_bib41x;
DROP TABLE IF EXISTS bibrec_bib42x;
DROP TABLE IF EXISTS bibrec_bib43x;
DROP TABLE IF EXISTS bibrec_bib44x;
DROP TABLE IF EXISTS bibrec_bib45x;
DROP TABLE IF EXISTS bibrec_bib46x;
DROP TABLE IF EXISTS bibrec_bib47x;
DROP TABLE IF EXISTS bibrec_bib48x;
DROP TABLE IF EXISTS bibrec_bib49x;
DROP TABLE IF EXISTS bibrec_bib50x;
DROP TABLE IF EXISTS bibrec_bib51x;
DROP TABLE IF EXISTS bibrec_bib52x;
DROP TABLE IF EXISTS bibrec_bib53x;
DROP TABLE IF EXISTS bibrec_bib54x;
DROP TABLE IF EXISTS bibrec_bib55x;
DROP TABLE IF EXISTS bibrec_bib56x;
DROP TABLE IF EXISTS bibrec_bib57x;
DROP TABLE IF EXISTS bibrec_bib58x;
DROP TABLE IF EXISTS bibrec_bib59x;
DROP TABLE IF EXISTS bibrec_bib60x;
DROP TABLE IF EXISTS bibrec_bib61x;
DROP TABLE IF EXISTS bibrec_bib62x;
DROP TABLE IF EXISTS bibrec_bib63x;
DROP TABLE IF EXISTS bibrec_bib64x;
DROP TABLE IF EXISTS bibrec_bib65x;
DROP TABLE IF EXISTS bibrec_bib66x;
DROP TABLE IF EXISTS bibrec_bib67x;
DROP TABLE IF EXISTS bibrec_bib68x;
DROP TABLE IF EXISTS bibrec_bib69x;
DROP TABLE IF EXISTS bibrec_bib70x;
DROP TABLE IF EXISTS bibrec_bib71x;
DROP TABLE IF EXISTS bibrec_bib72x;
DROP TABLE IF EXISTS bibrec_bib73x;
DROP TABLE IF EXISTS bibrec_bib74x;
DROP TABLE IF EXISTS bibrec_bib75x;
DROP TABLE IF EXISTS bibrec_bib76x;
DROP TABLE IF EXISTS bibrec_bib77x;
DROP TABLE IF EXISTS bibrec_bib78x;
DROP TABLE IF EXISTS bibrec_bib79x;
DROP TABLE IF EXISTS bibrec_bib80x;
DROP TABLE IF EXISTS bibrec_bib81x;
DROP TABLE IF EXISTS bibrec_bib82x;
DROP TABLE IF EXISTS bibrec_bib83x;
DROP TABLE IF EXISTS bibrec_bib84x;
DROP TABLE IF EXISTS bibrec_bib85x;
DROP TABLE IF EXISTS bibrec_bib86x;
DROP TABLE IF EXISTS bibrec_bib87x;
DROP TABLE IF EXISTS bibrec_bib88x;
DROP TABLE IF EXISTS bibrec_bib89x;
DROP TABLE IF EXISTS bibrec_bib90x;
DROP TABLE IF EXISTS bibrec_bib91x;
DROP TABLE IF EXISTS bibrec_bib92x;
DROP TABLE IF EXISTS bibrec_bib93x;
DROP TABLE IF EXISTS bibrec_bib94x;
DROP TABLE IF EXISTS bibrec_bib95x;
DROP TABLE IF EXISTS bibrec_bib96x;
DROP TABLE IF EXISTS bibrec_bib97x;
DROP TABLE IF EXISTS bibrec_bib98x;
DROP TABLE IF EXISTS bibrec_bib99x;
DROP TABLE IF EXISTS bibfmt;
DROP TABLE IF EXISTS idxINDEX;
DROP TABLE IF EXISTS idxINDEXNAME;
DROP TABLE IF EXISTS idxINDEX_field;
DROP TABLE IF EXISTS idxWORD01F;
DROP TABLE IF EXISTS idxWORD02F;
DROP TABLE IF EXISTS idxWORD03F;
DROP TABLE IF EXISTS idxWORD04F;
DROP TABLE IF EXISTS idxWORD05F;
DROP TABLE IF EXISTS idxWORD06F;
DROP TABLE IF EXISTS idxWORD07F;
DROP TABLE IF EXISTS idxWORD08F;
DROP TABLE IF EXISTS idxWORD09F;
DROP TABLE IF EXISTS idxWORD10F;
DROP TABLE IF EXISTS idxWORD11F;
DROP TABLE IF EXISTS idxWORD12F;
DROP TABLE IF EXISTS idxWORD13F;
DROP TABLE IF EXISTS idxWORD14F;
DROP TABLE IF EXISTS idxWORD15F;
DROP TABLE IF EXISTS idxWORD16F;
DROP TABLE IF EXISTS idxWORD17F;
DROP TABLE IF EXISTS idxWORD18F;
DROP TABLE IF EXISTS idxWORD01R;
DROP TABLE IF EXISTS idxWORD02R;
DROP TABLE IF EXISTS idxWORD03R;
DROP TABLE IF EXISTS idxWORD04R;
DROP TABLE IF EXISTS idxWORD05R;
DROP TABLE IF EXISTS idxWORD06R;
DROP TABLE IF EXISTS idxWORD07R;
DROP TABLE IF EXISTS idxWORD08R;
DROP TABLE IF EXISTS idxWORD09R;
DROP TABLE IF EXISTS idxWORD10R;
DROP TABLE IF EXISTS idxWORD11R;
DROP TABLE IF EXISTS idxWORD12R;
DROP TABLE IF EXISTS idxWORD13R;
DROP TABLE IF EXISTS idxWORD14R;
DROP TABLE IF EXISTS idxWORD15R;
DROP TABLE IF EXISTS idxWORD16R;
DROP TABLE IF EXISTS idxWORD17R;
DROP TABLE IF EXISTS idxWORD18R;
DROP TABLE IF EXISTS idxPAIR01F;
DROP TABLE IF EXISTS idxPAIR02F;
DROP TABLE IF EXISTS idxPAIR03F;
DROP TABLE IF EXISTS idxPAIR04F;
DROP TABLE IF EXISTS idxPAIR05F;
DROP TABLE IF EXISTS idxPAIR06F;
DROP TABLE IF EXISTS idxPAIR07F;
DROP TABLE IF EXISTS idxPAIR08F;
DROP TABLE IF EXISTS idxPAIR09F;
DROP TABLE IF EXISTS idxPAIR10F;
DROP TABLE IF EXISTS idxPAIR11F;
DROP TABLE IF EXISTS idxPAIR12F;
DROP TABLE IF EXISTS idxPAIR13F;
DROP TABLE IF EXISTS idxPAIR14F;
DROP TABLE IF EXISTS idxPAIR15F;
DROP TABLE IF EXISTS idxPAIR16F;
DROP TABLE IF EXISTS idxPAIR17F;
DROP TABLE IF EXISTS idxPAIR18F;
DROP TABLE IF EXISTS idxPAIR01R;
DROP TABLE IF EXISTS idxPAIR02R;
DROP TABLE IF EXISTS idxPAIR03R;
DROP TABLE IF EXISTS idxPAIR04R;
DROP TABLE IF EXISTS idxPAIR05R;
DROP TABLE IF EXISTS idxPAIR06R;
DROP TABLE IF EXISTS idxPAIR07R;
DROP TABLE IF EXISTS idxPAIR08R;
DROP TABLE IF EXISTS idxPAIR09R;
DROP TABLE IF EXISTS idxPAIR10R;
DROP TABLE IF EXISTS idxPAIR11R;
DROP TABLE IF EXISTS idxPAIR12R;
DROP TABLE IF EXISTS idxPAIR13R;
DROP TABLE IF EXISTS idxPAIR14R;
DROP TABLE IF EXISTS idxPAIR15R;
DROP TABLE IF EXISTS idxPAIR16R;
DROP TABLE IF EXISTS idxPAIR17R;
DROP TABLE IF EXISTS idxPAIR18R;
DROP TABLE IF EXISTS idxPHRASE01F;
DROP TABLE IF EXISTS idxPHRASE02F;
DROP TABLE IF EXISTS idxPHRASE03F;
DROP TABLE IF EXISTS idxPHRASE04F;
DROP TABLE IF EXISTS idxPHRASE05F;
DROP TABLE IF EXISTS idxPHRASE06F;
DROP TABLE IF EXISTS idxPHRASE07F;
DROP TABLE IF EXISTS idxPHRASE08F;
DROP TABLE IF EXISTS idxPHRASE09F;
DROP TABLE IF EXISTS idxPHRASE10F;
DROP TABLE IF EXISTS idxPHRASE11F;
DROP TABLE IF EXISTS idxPHRASE12F;
DROP TABLE IF EXISTS idxPHRASE13F;
DROP TABLE IF EXISTS idxPHRASE14F;
DROP TABLE IF EXISTS idxPHRASE15F;
DROP TABLE IF EXISTS idxPHRASE16F;
DROP TABLE IF EXISTS idxPHRASE17F;
DROP TABLE IF EXISTS idxPHRASE18F;
DROP TABLE IF EXISTS idxPHRASE01R;
DROP TABLE IF EXISTS idxPHRASE02R;
DROP TABLE IF EXISTS idxPHRASE03R;
DROP TABLE IF EXISTS idxPHRASE04R;
DROP TABLE IF EXISTS idxPHRASE05R;
DROP TABLE IF EXISTS idxPHRASE06R;
DROP TABLE IF EXISTS idxPHRASE07R;
DROP TABLE IF EXISTS idxPHRASE08R;
DROP TABLE IF EXISTS idxPHRASE09R;
DROP TABLE IF EXISTS idxPHRASE10R;
DROP TABLE IF EXISTS idxPHRASE11R;
DROP TABLE IF EXISTS idxPHRASE12R;
DROP TABLE IF EXISTS idxPHRASE13R;
DROP TABLE IF EXISTS idxPHRASE14R;
DROP TABLE IF EXISTS idxPHRASE15R;
DROP TABLE IF EXISTS idxPHRASE16R;
DROP TABLE IF EXISTS idxPHRASE17R;
DROP TABLE IF EXISTS idxPHRASE18R;
DROP TABLE IF EXISTS rnkMETHOD;
DROP TABLE IF EXISTS rnkMETHODNAME;
DROP TABLE IF EXISTS rnkMETHODDATA;
DROP TABLE IF EXISTS rnkWORD01F;
DROP TABLE IF EXISTS rnkWORD01R;
DROP TABLE IF EXISTS rnkPAGEVIEWS;
DROP TABLE IF EXISTS rnkDOWNLOADS;
DROP TABLE IF EXISTS rnkCITATIONDATA;
DROP TABLE IF EXISTS rnkCITATIONDATAEXT;
DROP TABLE IF EXISTS rnkAUTHORDATA;
DROP TABLE IF EXISTS collection_rnkMETHOD;
DROP TABLE IF EXISTS collection;
DROP TABLE IF EXISTS collectionname;
DROP TABLE IF EXISTS oaiREPOSITORY;
DROP TABLE IF EXISTS oaiHARVEST;
DROP TABLE IF EXISTS oaiHARVESTLOG;
DROP TABLE IF EXISTS bibHOLDINGPEN;
DROP TABLE IF EXISTS collection_collection;
DROP TABLE IF EXISTS collection_portalbox;
DROP TABLE IF EXISTS portalbox;
DROP TABLE IF EXISTS collection_example;
DROP TABLE IF EXISTS example;
DROP TABLE IF EXISTS collection_format;
DROP TABLE IF EXISTS format;
DROP TABLE IF EXISTS formatname;
DROP TABLE IF EXISTS collection_field_fieldvalue;
DROP TABLE IF EXISTS field;
DROP TABLE IF EXISTS fieldname;
DROP TABLE IF EXISTS fieldvalue;
DROP TABLE IF EXISTS field_tag;
DROP TABLE IF EXISTS tag;
DROP TABLE IF EXISTS publreq;
DROP TABLE IF EXISTS session;
DROP TABLE IF EXISTS user;
DROP TABLE IF EXISTS accROLE;
DROP TABLE IF EXISTS accMAILCOOKIE;
DROP TABLE IF EXISTS user_accROLE;
DROP TABLE IF EXISTS accACTION;
DROP TABLE IF EXISTS accARGUMENT;
DROP TABLE IF EXISTS accROLE_accACTION_accARGUMENT;
DROP TABLE IF EXISTS user_query;
DROP TABLE IF EXISTS query;
DROP TABLE IF EXISTS user_basket;
DROP TABLE IF EXISTS basket;
DROP TABLE IF EXISTS basket_record;
DROP TABLE IF EXISTS record;
DROP TABLE IF EXISTS user_query_basket;
DROP TABLE IF EXISTS cmtRECORDCOMMENT;
DROP TABLE IF EXISTS knwKB;
DROP TABLE IF EXISTS knwKBRVAL;
DROP TABLE IF EXISTS knwKBDDEF;
DROP TABLE IF EXISTS sbmACTION;
DROP TABLE IF EXISTS sbmALLFUNCDESCR;
DROP TABLE IF EXISTS sbmAPPROVAL;
DROP TABLE IF EXISTS sbmCPLXAPPROVAL;
DROP TABLE IF EXISTS sbmCOLLECTION;
DROP TABLE IF EXISTS sbmCOLLECTION_sbmCOLLECTION;
DROP TABLE IF EXISTS sbmCOLLECTION_sbmDOCTYPE;
DROP TABLE IF EXISTS sbmCATEGORIES;
DROP TABLE IF EXISTS sbmCHECKS;
DROP TABLE IF EXISTS sbmCOOKIES;
DROP TABLE IF EXISTS sbmDOCTYPE;
DROP TABLE IF EXISTS sbmFIELD;
DROP TABLE IF EXISTS sbmFIELDDESC;
DROP TABLE IF EXISTS sbmFORMATEXTENSION;
DROP TABLE IF EXISTS sbmFUNCTIONS;
DROP TABLE IF EXISTS sbmFUNDESC;
DROP TABLE IF EXISTS sbmGFILERESULT;
DROP TABLE IF EXISTS sbmIMPLEMENT;
DROP TABLE IF EXISTS sbmPARAMETERS;
DROP TABLE IF EXISTS sbmPUBLICATION;
DROP TABLE IF EXISTS sbmPUBLICATIONCOMM;
DROP TABLE IF EXISTS sbmPUBLICATIONDATA;
DROP TABLE IF EXISTS sbmREFEREES;
DROP TABLE IF EXISTS sbmSUBMISSIONS;
DROP TABLE IF EXISTS schTASK;
DROP TABLE IF EXISTS bibdoc;
DROP TABLE IF EXISTS bibdoc_bibdoc;
DROP TABLE IF EXISTS bibrec_bibdoc;
DROP TABLE IF EXISTS bibdocfsinfo;
DROP TABLE IF EXISTS usergroup;
DROP TABLE IF EXISTS user_usergroup;
DROP TABLE IF EXISTS user_basket;
DROP TABLE IF EXISTS msgMESSAGE;
DROP TABLE IF EXISTS user_msgMESSAGE;
DROP TABLE IF EXISTS bskBASKET;
DROP TABLE IF EXISTS bskEXTREC;
DROP TABLE IF EXISTS bskEXTFMT;
DROP TABLE IF EXISTS bskREC;
DROP TABLE IF EXISTS bskRECORDCOMMENT;
DROP TABLE IF EXISTS cmtACTIONHISTORY;
DROP TABLE IF EXISTS cmtSUBSCRIPTION;
DROP TABLE IF EXISTS user_bskBASKET;
DROP TABLE IF EXISTS usergroup_bskBASKET;
DROP TABLE IF EXISTS collection_externalcollection;
DROP TABLE IF EXISTS externalcollection;
DROP TABLE IF EXISTS collectiondetailedrecordpagetabs;
DROP TABLE IF EXISTS staEVENT;
DROP TABLE IF EXISTS clsMETHOD;
DROP TABLE IF EXISTS collection_clsMETHOD;
DROP TABLE IF EXISTS jrnJOURNAL;
DROP TABLE IF EXISTS jrnISSUE;
DROP TABLE IF EXISTS hstRECORD;
DROP TABLE IF EXISTS hstDOCUMENT;
DROP TABLE IF EXISTS hstTASK;
DROP TABLE IF EXISTS hstBATCHUPLOAD;
DROP TABLE IF EXISTS crcBORROWER;
DROP TABLE IF EXISTS crcILLREQUEST;
DROP TABLE IF EXISTS crcITEM;
DROP TABLE IF EXISTS crcLIBRARY;
DROP TABLE IF EXISTS crcLOAN;
DROP TABLE IF EXISTS crcLOANREQUEST;
DROP TABLE IF EXISTS crcPURCHASE;
DROP TABLE IF EXISTS crcVENDOR;
DROP TABLE IF EXISTS expJOB;
DROP TABLE IF EXISTS expQUERY;
DROP TABLE IF EXISTS expJOB_expQUERY;
DROP TABLE IF EXISTS expQUERYRESULT;
DROP TABLE IF EXISTS expJOBRESULT;
DROP TABLE IF EXISTS expJOBRESULT_expQUERYRESULT;
DROP TABLE IF EXISTS user_expJOB;
DROP TABLE IF EXISTS swrREMOTESERVER;
DROP TABLE IF EXISTS swrCLIENTDATA;
DROP TABLE IF EXISTS hstEXCEPTION;
DROP TABLE IF EXISTS aidUSERINPUTLOG;
DROP TABLE IF EXISTS aidCACHE;
DROP TABLE IF EXISTS aidPERSONIDDATA;
DROP TABLE IF EXISTS aidPERSONIDPAPERS;
DROP TABLE IF EXISTS aidRESULTS;
-DROP TABLE IF EXISTS aidPROBCACHE;
DROP TABLE IF EXISTS xtrJOB;
DROP TABLE IF EXISTS bsrMETHOD;
DROP TABLE IF EXISTS bsrMETHODNAME;
DROP TABLE IF EXISTS bsrMETHODDATA;
DROP TABLE IF EXISTS bsrMETHODDATABUCKET;
DROP TABLE IF EXISTS collection_bsrMETHOD;
DROP TABLE IF EXISTS lnkENTRY;
DROP TABLE IF EXISTS lnkENTRYURLTITLE;
DROP TABLE IF EXISTS lnkENTRYLOG;
DROP TABLE IF EXISTS lnkLOG;
DROP TABLE IF EXISTS lnkADMINURL;
DROP TABLE IF EXISTS lnkADMINURLLOG;
+DROP TABLE IF EXISTS webapikey;
-- end of file
diff --git a/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc b/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc
index b67bf559c..61ca08d9b 100644
--- a/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc
+++ b/modules/webaccess/doc/admin/webaccess-admin-guide.webdoc
@@ -1,1136 +1,1144 @@
## -*- mode: html; coding: utf-8; -*-
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
Contents
1. Introduction, using roles
2. WebAccess admin interface
3. Example pages, illustrating snapshots
4. Managing accounts / Access policy
5. Managing login methods
6. Firewall-like role definitions
1. INTRODUCTION, USING ROLES
WebAccess is a common RBAC, role based access control, for all of
Invenio. This means that users are connected to roles that cover
different areas of access. I.e administrator of the photo
collection or system librarian. Users can be active in
different areas and of course connected to as many roles as needed.
The roles are connected to actions. An action identifies a task you
can perform in Invenio. It can be defined to take any number of
arguments in order to more clearly describe what you are allowing
connected users to do.
For example the system librarian can be allowed to run bibindex on
the different indexes. To allow system librarians to run the
bibindex indexing on the field author we connect role system
librarian with action runbibindex using the argument
index='author'.
Additionally, roles could have firewall-like role
definitions. A definition is a formal description of which
users are entitled to belong to the role. So you have two ways for
connecting users to roles. Either linking explicitly a user with the
role or describing the characteristics that makes users belong to
the role.
WebAccess is based on allowing users to perform actions. This means
that only allowed actions are stored in the access control engine's
database.
2. WEBACCESS ADMIN INTERFACE
All the WebAccess Administration web pages have certain
features/design choices in common
- Divided into steps
The process of adding new authorizations/information is
stepwise. The subtitle contains information about wich step you are
on and what you are supposed to do.
- Restart from any wanted step
You can always start from an earlier step by simply clicking the
wanted button. This is not a way to undo changes! No information
about previous database is kept, so all changes are definite.
- Change or new entry must confirmed
On all the pages you will be asked to confirm the change, with
information about what kind of change you are about to perform.
- Links to other relevant admin areas on the right side
To make it easier to perform your administration tasks, we have
added a menu area on the right hand side of these pages. The menu
contain links to other relevant admin pages and change according to
the page you are on and the information you have selected.
3. EXAMPLE PAGES
I. Role area
II. Example - connecting role and user
I. Role area
Administration tasks starts in one of the administration areas. The
role area is the main area from where you can perform all your
managing tasks. The other admin areas are just other ways of
entering.
Home > Admin Area > WebAccess Admin > Manage WebAccess > Role Administration
Role Administration
administration with roles as access point
- Users:
- add or remove users from the access to a role and its priviliges.
- Authorizations/Actions:
- these terms means almost the same, but an authorization is a
connection between a role and an action (possibly) containing arguments.
- Roles:
- see all the information attached to a role and decide if you want to
delete it.
id
name
description
definition
users
authorizations / actions
role
2
photoadmin
Photo collection administrator
None
add / delete
add / modify / remove
modify / delete
show details
1
superadmin
superuser with all rights
allow email /.*@cern.ch/
add / delete
add / modify / remove
modify / delete
show details
3
webaccessadmin
WebAccess administrator
allow nickname 'jekyll'
add / delete
add / modify / remove
modify / delete
show details
- Create new role
- go here to add a new role.
- Create new action
- go here to add a new action.
II. Example - connecting role and user
One of the important tasks that can be handled via the WebAccess Admin Web Interface
is the delegation of access rights to users. This is done by connecting them to the
different roles offered.
The task is divided into 5 simple and comprehensive steps. Below follows the pages from
the different steps with comments on the ongoing procedure.
- step 1 - select a role
You must first select the role you want to connect users to. All the available roles are
listed alfabetically in a select box. Just find the wanted role and select it. Then click on
the button saying "select role".
If you start from the Role Area, this step is already done, and you start directly on step 2.
Home > Admin Area > WebAccess Admin >
Manage WebAccess > Role Administration > Connect user to role
Connect user to role
step 1 - select a role
- Create new role
- go here to add a new role.
- step 2 - search for users
As you can see, the subtitle of the page has now changed. The subtitle always tells you
which step you are on and what your current task is.
There can be possibly thousands of users using your online library, therefore it is important
to make it easier to identify the user you are looking for. Give part of, or the entire search
string and all users with partly matching e-mails will be listed on the next step.
You can also see that the right hand menu has changed. This area is always updated with links
to related admin areas.
Home > Admin Area > WebAccess Admin >
Manage WebAccess > Role Administration > Connect user to role
Connect user to role
step 2 - search for users
- Create new role
- go here to add a new role.
- Remove users
- remove users from role superadmin.
- Connected users
- show all users connected to role superadmin.
- Add authorization
- start adding new authorizations to role superadmin.
- step 3 - select a user.
The select box contains all users with partly matching e-mail addresses. Select the one
you want to connect to the role and continue.
Notice the navigation trail that tells you were on the Administrator pages you are currently
working.
Home > Admin Area > WebAccess Admin >
Manage WebAccess > Role Administration > Connect user to role
Connect user to role
step 3 - select a user
- Create new role
- go here to add a new role.
- Remove users
- remove users from role superadmin.
- Connected users
- show all users connected to role superadmin.
- Add authorization
- start adding new authorizations to role superadmin.
- step 4 - confirm to add user
All WebAccess Administrator web pages display the action you are about to peform, this
means explaining what kind of addition, change or update will be done to your access control
data.
If you are happy with your decision, simply confirm it.
Home > Admin Area > WebAccess Admin >
Manage WebAccess > Role Administration > Connect user to role
Connect user to role
step 4 - confirm to add user
- Create new role
- go here to add a new role.
- Remove users
- remove users from role superadmin.
- Connected users
- show all users connected to role superadmin.
- Add authorization
- start adding new authorizations to role superadmin.
- step 5 - confirm user added.
The user has now been added to this role. You can easily continue adding more users to this
role be restarting from step 2 or 3. You can also go directly to another area and keep working
on the same role.
Home > Admin Area > WebAccess Admin >
Manage WebAccess > Role Administration > Connect user to role
Connect user to role
step 5 - confirm user added
confirm: user mikael.vik@cern.ch added to role superadmin.
- Create new role
- go here to add a new role.
- Remove users
- remove users from role superadmin.
- Connected users
- show all users connected to role superadmin.
- Add authorization
- start adding new authorizations to role superadmin.
- we are done
This example is very similar to all the other pages where you administrate WebAccess. The pages
are an easy gateway to maintaing access control rights and share a lot of features.
- divided into steps
- restart from any wanted step (not undo)
- changes must be confirmed
- link to other relevant areas
- prevent unwanted input
As an administrator with access to these pages you are free to manage the rights any way you want.
IV. Managing accounts and access policy
Here you can administrate the accounts and the access policy for your Invenio installation.
- Access policy:
To change the access policy, the general config file (or
access_control_config.py) must be edited manually in a text
editor. The site can there be defined as opened or closed, you can
edit the access policy level for guest accounts, registered
accounts and decide when to warn the owner of the account when
something happens with it, either when it is created, deleted or
approved. The Apache server must be restarted after modifying
these settings.
The two levels for guest account, are:
0 - Allow guest accounts
1 - Do not allow guest accounts
The five levels for normal accounts, are:
0 - Allow user to create account, automatically activate new accounts
1 - Allow user to create account, administrator must activate account
2 - Only administrators can create account. User cannot edit the email address.
3 - Users cannot register or update account information (email/password)
4 - User cannot change default login method
You can configure Invenio to send an email:
1. To an admin email-address when an account is created
2. To the owner of an account when it is created
3. To the owner of an account when it is activated
4. To the owner of an account when it is deleted
Define how open the site is:
0 = normal operation of the site
1 = read-only site, all write operations temporarily closed
2 = site fully closed
3 = database connections disabled
CFG_ACCESS_CONTROL_LEVEL_SITE = 0
Access policy for guests:
0 = Allow guests to search,
1 = Guests cannot search (all users must login)
CFG_ACCESS_CONTROL_LEVEL_GUESTS = 0
Access policy for accounts:
0 = Users can register, automatically acticate accounts
1 = Users can register, but admin must activate the accounts
2 = Users cannot register or change email address, only admin can register accounts.
3 = Users cannot register or update email address or password, only admin can register accounts.
4 = Same as 3, but user cannot change login method.
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS = 0
Limit email addresses available to use when register a new account (example: cern.ch):
CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN = ""
Send an email when a new account is created by an user:
CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS = 0
Send an email to the user notifying when the account is created:
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT = 0
Send an email to the user notifying when the account is activated:
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION = 0
Send an email to the user notifying when the account is deleted/rejected:
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION = 0
- Account overview:
Here you find an overview of the number of guest accounts, registered accounts and accounts
awaiting activation, with a link to the activation page.
- Create account:
For creating new accounts, the email address must be unique. If configured to do so, an email
will be sent to the given address when an account is created.
- Edit accounts:
For activating or rejecting accounts in addition to modifying them. An activated account can be
inactivated for a short period of time, but this will not warn the account owner. To find accounts
enter a part of the email address of the account and then search. This may take some time. If there
are more than the selected number of accounts per page, you can use the next/prev links to switch
pages. The accounts to search in can also be limited to only activated or not activated accounts.
- Edit account:
When editing one account, you can change the email address, password, delete the account, or modify
the baskets or alerts belonging to one account. Which login method should be the default for this
account can also be selected. To modify baskets or alerts, you need to login as the user, and
modify the desired data as a normal user. Remember to log out as the user when you are finished
editing.
+ Here you can also edit the user's REST API keys changing its status to the desired one. This status
+ is specified by CFG_REST_API_KEY_STATUS (only the first three are mandatory):
+ 0 = OK
+ 1 = REMOVED
+ 2 = REVOKED
+ 3 = WARNING1
+ 4 = WARNING2
+ 5 = WARNING3
V. Managing login methods
Invenio supports using external login systems to authenticate users.
When a user wants to login, the username and password given by the user is checked against the selected
system, if the user is authenticated by the external system, a valid email-address is returned to
Invenio and used to recognize the user within Invenio.
If a new user is trying to login without having an account, using an external login system, an account
is automatically created in Invenio to recognize and store the users settings. The password for the
local account is randomly generated.
If you want the user to be unable to change login method and account username / password, forcing use
of certain external systems, set CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS to 4 as mentioned in the last paragraph.
If a user is changing login method from an external one to the internal, he also need to either change the
password before logging out, or set the password via the lost password email service.
If you are using CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS with a value greater than 1 note
that, even if the first login of a user through an external authentication technically means registering
the user into the system, this is not the semantic expected behaviour by the user. The user is already
registered at an authority that we trust, so there's no need to prevent the user from being imported
into the system. That's why for external authentication CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS is not
considered apart from what said above.
If a external login system is used, you may want to protect the users username / password using HTTPS.
To add new system, two changes must be made (for the time being):
- The name of the method, if it is default or not, and an instance of the class must be added to the variable
CFG_EXTERNAL_AUTHENTICATION in access_control_config.py.
- A class must be created derived from the class external_authentication inside file
external_authentication.py. This class must include at least the
function auth_user. This function returns a valid email-address in Invenio if the user
is authenticated, not necessarily the same entered by the user as username. If the user
is not authenticated, return None.
The class could also provide five more methods: fetch_user_preferences, user_exists,
fetch_user_groups_membership and fetch_all_users_groups_membership.
The first should take an email and eventually a password and should return a dictionary of keys
and value representing external preferences, infos or settings. If, for some reasons, you like
to force some kind of hiding for some particular field you should export the related key
prefixed by "HIDDEN_". Those fields won't be displayed in tables and pages regarding external
settings.
The second method should check through the external system if a particular email exists. If you
provide such a method then a user will be able to switch from and to this authorization method.
The third method should take an email and (if necessary) a password
and should return a dictionary of external_groups_names toghether with their description, for which
the user has a membership. Those groups will be merged into the groups system.
The user will be a member of those groups and will be able to use them in any place
where groups are useful, but won't be able to unsubscribe or to administrate them.
The fourth method should just return a dictionary of external groups as keys and tuples containing
a group description and a list of email of users belonging to each groups. Those memberships
will be merged into the database in the way done by the previous method, but could
provide batch synchronization of groups.
The fifth method should just return the nickname as is known by the external authentication
system, given the usual email/username and the password.
Note: if your system has more than one external login methods then incoherence in the groups
memberships could happen when a user switch his login method. This will be fixed some times in the
future.
If you add as an attribute of your class the enforce_external_nicknames and set it to True, this will enforce
the system to import external nicknames whenever the user login with the external login method for the
first time. Since a nickname is not changable this will stay fixed forever. If this nickname is
already registered in the system (suppose that is linked with a local account) then it will not be
imported. If this variable doesn't exist or is set to False then no nickname will be
imported and the user will be free to choose a nickname in the future (and then this will again
stay forever).
Note: every method will receive as last parameter the mod_python request object, that could
be used for particular purposes.
Example template:
from invenio.external_authentication import ExternalAuth, InvenioWebAccessExternalAuthError
class ExternalAuthFoo(ExternalAuth):
"""External authentication template example."""
def __init__ (self):
"""Initialize stuff here."""
self.name = None
self.enforce_external_nicknames = False
pass
def auth_user(self, username, password, req=None):
"""Authenticate user-supplied USERNAME and PASSWORD.
Return None if authentication failed, or the email address of the
person if the authentication was successful. In order to do
this you may perhaps have to keep a translation table between
usernames and email addresses.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
"""
raise NotImplementedError
#return None
def user_exists(self, email, req=None):
"""Checks against external_authentication for existance of email.
@return True if the user exists, False otherwise
"""
raise NotImplementedError
def fetch_user_groups_membership(self, username, password=None, req=None):
"""Given a username, returns a dictionary of groups
and their description to which the user is subscribed.
Raise InvenioWebAccessExternalAuthError in case of troubles.
"""
raise NotImplementedError
#return {}
def fetch_user_preferences(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of keys and
values, corresponding to external infos and settings.
userprefs = {"telephone": "2392489",
"address": "10th Downing Street"}
"""
raise NotImplementedError
#return {}
def fetch_all_users_groups_membership(self, req=None):
"""Fetch all the groups with a description, and users who belong to
each groups.
@return {'mygroup': ('description', ['email1', 'email2', ...]), ...}
"""
raise NotImplementedError
def fetch_user_nickname(self, username, password, req=None):
"""Given a username and a password, returns the right nickname belonging
to that user (username could be an email).
"""
raise NotImplementedError
#return Nickname
VI. FIREWALL-LIKE ROLE DEFINITIONS
The FireRole language description
In the WebAccess RBAC system, roles are built up from their names,
description and definition.
A definition is the way to formally implicitly define which users belong
to which roles.
A definition is expressed in a firewall like rules language. It's built up
by rows which are matched from top to bottom, in order to decide if the
current user (wethever he/she is logged in or not) may belong to a role.
Any row has this syntax:
ALLOW/DENY ANY/ALL
ALLOW/DENY FROM/UNTIL "YYYY-MM-DD"
or
ALLOW/DENY [NOT] field {one or more values}
The rows are parsed from top to bottom. If a row matches the user than the
user belongs to the role if the rule is an ALLOW rule, otherwise, if the
rule is a DENY one, the user doesn't belong to the role.
A rule of the kind ALLOW|DENY ANY always matches, regardless of the user.
Note, in place of ANY you can use the word ALL. The semantic is the same. The
system support both to let the user comply with the English grammar.
The second type of rule is interpreted as follows: given a date in the
form "YYYY-MM-DD" (double-)quoted), the rule is matched if, when using FROM,
the current date is either identical or 'bigger' than the given date, or if,
when using UNTIL, the current date is either identical or 'smaller' than the
given date. If the rule starts with ALLOW and is matched
then the next row is evaluated. If it is not matched, then the whole FireRole
will evaluate into a DENY ALL. If the rule starts with DENY and
is matched then the whole FireRole while evaluate as a DENY ALL. If it is
not matched then the next row is evaluated.
The third type of rule is interpreted as follows: given a dictionary
of keys:values describing a user (we will cover this below), the rule
considers the value associated with the key named in field, and checks
if it corresponds to at least one of the values in the "one or more values" list.
This is a list of comma separated strings, which can be literal
(double-)quoted strings or regexps (marked by `/' ... `/' signs). If at
least a value matches (literally or through the regexp language), the
whole rule is considered to match.
If the optional NOT keyword is specified than if at least a value of the
rule matches the rule is skipped, otherwise if all the value of the rules
don't match the whole rule matches.
A DENY ALL rule is implicitly added at the end of every definition. Note that
this imply that, if you are using e.g. a temporal rule (FROM/UNTIL), you
should explicitly add an additional row with value ALLOW ANY,
if you actually want to allow users in the specified timeframe.
Any field is valid, but only rules concerning fields which currently
exist in the user describing dictionary are checked. All the rules
with non existant fields are skipped.
The user describing dictionary (user_info) is built at runtime with all the informations
that can be gathered about the current user (and its session).
Currently valid fields are: uid, email, nickname, remote_ip,
remote_host, groups and all the external settings provided
by the external authentication systems (e.g. CERN SSO provides:
external_authmethod, external_building, external_department, external_email,
external_external, external_firstname, external_fullname, external_homdir,
external_homeinstitute, external_lastname, external_login, external_mobilenumber,
external_phonenumber).
Among those fields there are some special cases, which are remote_ip and
(apache_)groups. Rules can refer to remote_ip either using a literal
expression for specifing list of single ips, or a usual regexp (or list
of regexps), or, also, using the common network group/mask notation
(e.g. "127.0.0.0/24") as a literal string, which is a mix between literal
expressions and regexps. (apache_)groups are related to group memberships.
Since a user will probably belong to more than a group, then the rule
matches if there's at least one group to which the user belong, that matches
at least one of the expressions (NOT rules behave as you can imagine).
The dictionary is built using the current user session. If the user is
authenticated in some way (apache, locally, externally, SSO...) then more
infos could be provided to the firerole system in order to decide if the
user should belong to a role or not.
The default fields that are always there are:
- uid: an integer representing the user id
- nickname: the nickname of the user
- email: the email of the user
- group/groups: local or external group to which the user belong
- guest: 1 if the user is a guest (not logged), 0 otherwise
plus all the external setting retrieved by an external authentication system.
If the action to which the role defined is raised from the webinterface of
Invenio, then you will have those additional fields:
- remote_ip: the remote ip address of the user who is browsing
- remote_host: the remote hostname of the user who is browsing
- referer: the webpage from where the user is coming from
- uri: the uri the user is visiting
- agent: the agent string describing the user's browser
Note that you can specify either (apache_)group or (apache_)groups (with or
without the trailing s). They are semantically equal and are supported just
to let people comply with the English grammar.
Every rule is case-insensitive (apart values which must match literally
and regexp values which don't explicitly specify case-insesitive matches).
Every rule may contain comments preceded by the '#' character.
Any comment is discarded.
When you set a definition for a role, it is actually compiled and stored
in a binary compressed form inside the database. If the syntax isn't correct
this will be stated and the definition won't be set or updated.
Example of role definition:
allow not email /.*@gmail.com/,/.*@hotmail.com/
deny group badguys
allow remote_ip "127.0.0.0/24"
deny all
This definition would match all users whose emails don't end with @gmail.com and
@hotmail.com, or who don't belong to the group badguys and have remote_ip
in the 24bit mask network of 127.0.0.0. All the the other users don't belong
to the role which is being defined.
If you want to discover which keys are available on your system to build a FireRole
rule, just login with your account in your installation and visit your account page,
by activating verbose=9 variable. Under the tile you will se the available keys and
values that you can use to build a FireRole rule. All but fields prefixed with
precached_ are usuable.
diff --git a/modules/webaccess/lib/access_control_config.py b/modules/webaccess/lib/access_control_config.py
index 93a2e36f7..44efd2cdc 100644
--- a/modules/webaccess/lib/access_control_config.py
+++ b/modules/webaccess/lib/access_control_config.py
@@ -1,363 +1,378 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Invenio Access Control Config. """
__revision__ = \
"$Id$"
# pylint: disable=C0301
from invenio.config import CFG_SITE_NAME, CFG_SITE_URL, CFG_SITE_LANG, \
CFG_SITE_SECURE_URL, CFG_SITE_SUPPORT_EMAIL, CFG_CERN_SITE, \
CFG_OPENAIRE_SITE, CFG_SITE_RECORD, CFG_INSPIRE_SITE
from invenio.messages import gettext_set_language
class InvenioWebAccessFireroleError(Exception):
"""Just an Exception to discover if it's a FireRole problem"""
pass
# VALUES TO BE EXPORTED
# CURRENTLY USED BY THE FILES access_control_engine.py access_control_admin.py webaccessadmin_lib.py
# name of the role giving superadmin rights
SUPERADMINROLE = 'superadmin'
# name of the webaccess webadmin role
WEBACCESSADMINROLE = 'webaccessadmin'
# name of the action allowing roles to access the web administrator interface
WEBACCESSACTION = 'cfgwebaccess'
# name of the action allowing roles to access the web administrator interface
VIEWRESTRCOLL = 'viewrestrcoll'
# name of the action allowing roles to delegate the rights to other roles
# ex: libraryadmin to delegate libraryworker
DELEGATEADDUSERROLE = 'accdelegaterole'
# max number of users to display in the drop down selects
MAXSELECTUSERS = 25
# max number of users to display in a page (mainly for user area)
MAXPAGEUSERS = 25
# default role definition, source:
CFG_ACC_EMPTY_ROLE_DEFINITION_SRC = 'deny all'
# default role definition, compiled:
CFG_ACC_EMPTY_ROLE_DEFINITION_OBJ = (False, ())
# default role definition, compiled and serialized:
CFG_ACC_EMPTY_ROLE_DEFINITION_SER = None
# List of tags containing (multiple) emails of users who should authorize
# to access the corresponding record regardless of collection restrictions.
if CFG_CERN_SITE:
CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS = ['859__f', '270__m', '506__m']
else:
CFG_ACC_GRANT_AUTHOR_RIGHTS_TO_EMAILS_IN_TAGS = ['8560_f']
# Use external source for access control?
# CFG_EXTERNAL_AUTHENTICATION -- this is a dictionary with the enabled login method.
# The key is the name of the login method and the value is an instance of
# of the login method (see /help/admin/webaccess-admin-guide#5). Set the value
# to None if you wish to use the local Invenio authentication method.
# CFG_EXTERNAL_AUTH_DEFAULT -- set this to the key in CFG_EXTERNAL_AUTHENTICATION
# that should be considered as default login method
# CFG_EXTERNAL_AUTH_USING_SSO -- set this to the login method name of an SSO
# login method, if any, otherwise set this to None.
# CFG_EXTERNAL_AUTH_LOGOUT_SSO -- if CFG_EXTERNAL_AUTH_USING_SSO was not None
# set this to the URL that should be contacted to perform an SSO logout
from invenio.external_authentication_robot import ExternalAuthRobot
if CFG_CERN_SITE:
import external_authentication_sso as ea_sso
CFG_EXTERNAL_AUTH_USING_SSO = "CERN"
CFG_EXTERNAL_AUTH_DEFAULT = CFG_EXTERNAL_AUTH_USING_SSO
CFG_EXTERNAL_AUTH_LOGOUT_SSO = 'https://login.cern.ch/adfs/ls/?wa=wsignout1.0'
CFG_EXTERNAL_AUTHENTICATION = {
CFG_EXTERNAL_AUTH_USING_SSO : ea_sso.ExternalAuthSSO(),
}
elif CFG_OPENAIRE_SITE:
CFG_EXTERNAL_AUTH_DEFAULT = 'Local'
CFG_EXTERNAL_AUTH_USING_SSO = False
CFG_EXTERNAL_AUTH_LOGOUT_SSO = None
CFG_EXTERNAL_AUTHENTICATION = {
"Local": None,
- "OpenAIRE": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=False),
- "ZOpenAIRE": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=True)
+ "OpenAIRE": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=False, external_id_attribute_name="id"),
+ }
+elif CFG_INSPIRE_SITE:
+ CFG_EXTERNAL_AUTH_DEFAULT = 'arXiv'
+ CFG_EXTERNAL_AUTH_USING_SSO = False
+ CFG_EXTERNAL_AUTH_LOGOUT_SSO = None
+ CFG_EXTERNAL_AUTHENTICATION = {
+ "arXiv": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=True)
}
elif CFG_INSPIRE_SITE:
# INSPIRE specific robot configuration
CFG_EXTERNAL_AUTH_DEFAULT = 'Local'
CFG_EXTERNAL_AUTH_USING_SSO = False
CFG_EXTERNAL_AUTH_LOGOUT_SSO = None
CFG_EXTERNAL_AUTHENTICATION = {
"Local": None,
"Robot": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=False, check_user_ip=2),
"ZRobot": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=True, check_user_ip=2)
}
else:
CFG_EXTERNAL_AUTH_DEFAULT = 'Local'
CFG_EXTERNAL_AUTH_USING_SSO = False
CFG_EXTERNAL_AUTH_LOGOUT_SSO = None
CFG_EXTERNAL_AUTHENTICATION = {
"Local": None,
"Robot": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=False),
"ZRobot": ExternalAuthRobot(enforce_external_nicknames=True, use_zlib=True)
}
## If using SSO, this is the number of seconds after which the keep-alive
## SSO handler is pinged again to provide fresh SSO information.
CFG_EXTERNAL_AUTH_SSO_REFRESH = 600
# default data for the add_default_settings function
# Note: by default the definition is set to deny any. This won't be a problem
# because userid directly connected with roles will still be allowed.
# roles
# name description definition
DEF_ROLES = ((SUPERADMINROLE, 'superuser with all rights', 'deny any'),
(WEBACCESSADMINROLE, 'WebAccess administrator', 'deny any'),
('anyuser', 'Any user', 'allow any'),
('basketusers', 'Users who can use baskets', 'allow any'),
('loanusers', 'Users who can use loans', 'allow any'),
('groupusers', 'Users who can use groups', 'allow any'),
('alertusers', 'Users who can use alerts', 'allow any'),
('messageusers', 'Users who can use messages', 'allow any'),
('holdingsusers', 'Users who can view holdings', 'allow any'),
('statisticsusers', 'Users who can view statistics', 'allow any'),
('claimpaperusers', 'Users who can perform changes to their own paper attributions without the need for an operator\'s approval', 'allow any'),
('claimpaperoperators', 'Users who can perform changes to _all_ paper attributions without the need for an operator\'s approval', 'deny any'),
('paperclaimviewers', 'Users who can view "claim my paper" facilities.', 'allow all'),
('paperattributionviewers', 'Users who can view "attribute this paper" facilities', 'allow all'),
('paperattributionlinkviewers', 'Users who can see attribution links in the search', 'allow all'),
)
# Demo site roles
DEF_DEMO_ROLES = (('photocurator', 'Photo collection curator', 'deny any'),
('thesesviewer', 'Theses viewer', 'allow group "Theses viewers"'),
('thesescurator', 'Theses collection curator', 'deny any'),
('swordcurator', 'BibSword client curator', 'deny any'),
('referee_DEMOBOO_*', 'Book collection curator', 'deny any'),
('restrictedpicturesviewer', 'Restricted pictures viewer', 'deny any'),
('curator', 'Curator', 'deny any'),
('basketusers', 'Users who can use baskets', 'deny email "hyde@cds.cern.ch"\nallow any'),
('claimpaperusers', 'Users who can perform changes to their own paper attributions without the need for an operator\'s approval', 'deny email "hyde@cds.cern.ch"\nallow any'),
('submit_DEMOJRN_*', 'Users who can submit (and modify) "Atlantis Times" articles', 'deny all'),
('atlantiseditor', 'Users who can configure "Atlantis Times" journal', 'deny all'),
('commentmoderator', 'Users who can moderate comments', 'deny all'),
('poetrycommentreader', 'Users who can view comments in Poetry collection', 'deny all'))
DEF_DEMO_USER_ROLES = (('jekyll@cds.cern.ch', 'thesesviewer'),
('jekyll@cds.cern.ch', 'swordcurator'),
('jekyll@cds.cern.ch', 'claimpaperusers'),
('dorian.gray@cds.cern.ch', 'referee_DEMOBOO_*'),
('balthasar.montague@cds.cern.ch', 'curator'),
('romeo.montague@cds.cern.ch', 'restrictedpicturesviewer'),
('romeo.montague@cds.cern.ch', 'swordcurator'),
('romeo.montague@cds.cern.ch', 'thesescurator'),
('juliet.capulet@cds.cern.ch', 'restrictedpicturesviewer'),
('juliet.capulet@cds.cern.ch', 'photocurator'),
('romeo.montague@cds.cern.ch', 'submit_DEMOJRN_*'),
('juliet.capulet@cds.cern.ch', 'submit_DEMOJRN_*'),
('balthasar.montague@cds.cern.ch', 'atlantiseditor'),
('romeo.montague@cds.cern.ch', 'poetrycommentreader'))
# users
# list of e-mail addresses
DEF_USERS = []
# actions
# name desc allowedkeywords optional
DEF_ACTIONS = (
('cfgwebsearch', 'configure WebSearch', '', 'no'),
('cfgbibformat', 'configure BibFormat', '', 'no'),
('cfgbibknowledge', 'configure BibKnowledge', '', 'no'),
('cfgwebsubmit', 'configure WebSubmit', '', 'no'),
('cfgbibrank', 'configure BibRank', '', 'no'),
('cfgwebcomment', 'configure WebComment', '', 'no'),
('cfgweblinkback', 'configure WebLinkback' , '', 'no'),
('cfgoaiharvest', 'configure OAI Harvest', '', 'no'),
('cfgoairepository', 'configure OAI Repository', '', 'no'),
('cfgbibindex', 'configure BibIndex', '', 'no'),
('cfgbibexport', 'configure BibExport', '', 'no'),
('cfgrobotkeys', 'configure Robot keys', 'login_method,robot', 'yes'),
('cfgbibsort', 'configure BibSort', '', 'no'),
('runbibindex', 'run BibIndex', '', 'no'),
('runbibupload', 'run BibUpload', '', 'no'),
('runwebcoll', 'run webcoll', 'collection', 'yes'),
('runbibformat', 'run BibFormat', 'format', 'yes'),
('runbibclassify', 'run BibClassify', 'taxonomy', 'yes'),
('runbibtaskex', 'run BibTaskEx example', '', 'no'),
('runbibrank', 'run BibRank', '', 'no'),
('runoaiharvest', 'run oaiharvest task', '', 'no'),
('runoairepository', 'run oairepositoryupdater task', '', 'no'),
('runbibedit', 'run Record Editor', 'collection', 'yes'),
('runbibeditmulti', 'run Multi-Record Editor', '', 'no'),
('runbibdocfile', 'run Document File Manager', '', 'no'),
('runbibmerge', 'run Record Merger', '', 'no'),
('runbibswordclient', 'run BibSword client', '', 'no'),
('runwebstatadmin', 'run WebStadAdmin', '', 'no'),
('runinveniogc', 'run InvenioGC', '', 'no'),
('runbibexport', 'run BibExport', '', 'no'),
('referee', 'referee document type doctype/category categ', 'doctype,categ', 'yes'),
('submit', 'use webSubmit', 'doctype,act,categ', 'yes'),
('viewrestrdoc', 'view restricted document', 'status', 'no'),
('viewrestrcomment', 'view restricted comment', 'status', 'no'),
(WEBACCESSACTION, 'configure WebAccess', '', 'no'),
(DELEGATEADDUSERROLE, 'delegate subroles inside WebAccess', 'role', 'no'),
(VIEWRESTRCOLL, 'view restricted collection', 'collection', 'no'),
('cfgwebjournal', 'configure WebJournal', 'name,with_editor_rights', 'no'),
('viewcomment', 'view comments', 'collection', 'no'),
('viewlinkbacks', 'view linkbacks', 'collection', 'no'),
('sendcomment', 'send comments', 'collection', 'no'),
('attachcommentfile', 'attach files to comments', 'collection', 'no'),
('attachsubmissionfile', 'upload files to drop box during submission', '', 'no'),
('cfgbibexport', 'configure BibExport', '', 'no'),
('runbibexport', 'run BibExport', '', 'no'),
('usebaskets', 'use baskets', '', 'no'),
('useloans', 'use loans', '', 'no'),
('usegroups', 'use groups', '', 'no'),
('usealerts', 'use alerts', '', 'no'),
('usemessages', 'use messages', '', 'no'),
('viewholdings', 'view holdings', 'collection', 'yes'),
('viewstatistics', 'view statistics', 'collection', 'yes'),
('runbibcirculation', 'run BibCirculation', '', 'no'),
('moderatecomments', 'moderate comments', 'collection', 'no'),
('moderatelinkbacks', 'moderate linkbacks', 'collection', 'no'),
('runbatchuploader', 'run batchuploader', 'collection', 'yes'),
('runbibtasklet', 'run BibTaskLet', '', 'no'),
('claimpaper_view_pid_universe', 'View the Claim Paper interface', '', 'no'),
('claimpaper_claim_own_papers', 'Clam papers to his own personID', '', 'no'),
('claimpaper_claim_others_papers', 'Claim papers for others', '', 'no'),
('claimpaper_change_own_data', 'Change data associated to his own person ID', '', 'no'),
('claimpaper_change_others_data', 'Change data of any person ID', '', 'no'),
('runbibtasklet', 'run BibTaskLet', '', 'no'),
('cfgbibsched', 'configure BibSched', '', 'no')
)
# Default authorizations
# role action arguments
DEF_AUTHS = (('basketusers', 'usebaskets', {}),
('loanusers', 'useloans', {}),
('groupusers', 'usegroups', {}),
('alertusers', 'usealerts', {}),
('messageusers', 'usemessages', {}),
('holdingsusers', 'viewholdings', {}),
('statisticsusers', 'viewstatistics', {}),
('claimpaperusers', 'claimpaper_view_pid_universe', {}),
('claimpaperoperators', 'claimpaper_view_pid_universe', {}),
('claimpaperusers', 'claimpaper_claim_own_papers', {}),
('claimpaperoperators', 'claimpaper_claim_own_papers', {}),
('claimpaperoperators', 'claimpaper_claim_others_papers', {}),
('claimpaperusers', 'claimpaper_change_own_data', {}),
('claimpaperoperators', 'claimpaper_change_own_data', {}),
('claimpaperoperators', 'claimpaper_change_others_data', {}),
)
# Demo site authorizations
# role action arguments
DEF_DEMO_AUTHS = (
('photocurator', 'runwebcoll', {'collection': 'Pictures'}),
('restrictedpicturesviewer', 'viewrestrdoc', {'status': 'restricted_picture'}),
('thesesviewer', VIEWRESTRCOLL, {'collection': 'Theses'}),
('referee_DEMOBOO_*', 'referee', {'doctype': 'DEMOBOO', 'categ': '*'}),
('curator', 'cfgbibknowledge', {}),
('curator', 'runbibedit', {}),
('curator', 'runbibeditmulti', {}),
('curator', 'runbibmerge', {}),
('swordcurator', 'runbibswordclient', {}),
('thesescurator', 'runbibedit', {'collection': 'Theses'}),
('thesescurator', VIEWRESTRCOLL, {'collection': 'Theses'}),
('photocurator', 'runbibedit', {'collection': 'Pictures'}),
('referee_DEMOBOO_*', 'runbibedit', {'collection': 'Books'}),
('submit_DEMOJRN_*', 'submit', {'doctype': 'DEMOJRN', 'act': 'SBI', 'categ': '*'}),
('submit_DEMOJRN_*', 'submit', {'doctype': 'DEMOJRN', 'act': 'MBI', 'categ': '*'}),
('submit_DEMOJRN_*', 'cfgwebjournal', {'name': 'AtlantisTimes', 'with_editor_rights': 'no'}),
('atlantiseditor', 'cfgwebjournal', {'name': 'AtlantisTimes', 'with_editor_rights': 'yes'}),
('referee_DEMOBOO_*', 'runbatchuploader', {'collection': 'Books'}),
('poetrycommentreader', 'viewcomment', {'collection': 'Poetry'}),
('atlantiseditor', VIEWRESTRCOLL, {'collection': 'Atlantis Times Drafts'}),
('anyuser', 'submit', {'doctype': 'DEMOART', 'act': 'SBI', 'categ': 'ARTICLE'}),
)
_ = gettext_set_language(CFG_SITE_LANG)
# Activities (i.e. actions) for which exists an administrative web interface.
CFG_ACC_ACTIVITIES_URLS = {
'runbibedit' : (_("Run Record Editor"), "%s/%s/edit/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
'runbibeditmulti' : (_("Run Multi-Record Editor"), "%s/%s/multiedit/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
'runbibdocfile' : (_("Run Document File Manager"), "%s/submit/managedocfiles?ln=%%s" % CFG_SITE_URL),
'runbibmerge' : (_("Run Record Merger"), "%s/%s/merge/?ln=%%s" % (CFG_SITE_URL, CFG_SITE_RECORD)),
'runbibswordclient' : (_("Run BibSword client"), "%s/bibsword/?ln=%%s" % CFG_SITE_URL),
'cfgbibknowledge' : (_("Configure BibKnowledge"), "%s/kb?ln=%%s" % CFG_SITE_URL),
'cfgbibformat' : (_("Configure BibFormat"), "%s/admin/bibformat/bibformatadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgoaiharvest' : (_("Configure OAI Harvest"), "%s/admin/bibharvest/oaiharvestadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgoairepository' : (_("Configure OAI Repository"), "%s/admin/bibharvest/oairepositoryadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgbibindex' : (_("Configure BibIndex"), "%s/admin/bibindex/bibindexadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgbibrank' : (_("Configure BibRank"), "%s/admin/bibrank/bibrankadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgwebaccess' : (_("Configure WebAccess"), "%s/admin/webaccess/webaccessadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgwebcomment' : (_("Configure WebComment"), "%s/admin/webcomment/webcommentadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgweblinkback' : (_("Configure WebLinkback"), "%s/admin/weblinkback/weblinkbackadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgwebsearch' : (_("Configure WebSearch"), "%s/admin/websearch/websearchadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgwebsubmit' : (_("Configure WebSubmit"), "%s/admin/websubmit/websubmitadmin.py?ln=%%s" % CFG_SITE_URL),
'cfgwebjournal' : (_("Configure WebJournal"), "%s/admin/webjournal/webjournaladmin.py?ln=%%s" % CFG_SITE_URL),
'cfgbibsort' : (_("Configure BibSort"), "%s/admin/bibsort/bibsortadmin.py?ln=%%s" % CFG_SITE_URL),
'runbibcirculation' : (_("Run BibCirculation"), "%s/admin/bibcirculation/bibcirculationadmin.py?ln=%%s" % CFG_SITE_URL),
'runbatchuploader' : (_("Run Batch Uploader"), "%s/batchuploader/metadata?ln=%%s" % CFG_SITE_URL),
'claimpaper_claim_others_papers' : (_("Run Person/Author Manager"), "%s/person/search?ln=%%s" % CFG_SITE_URL)
}
CFG_WEBACCESS_MSGS = {
0: 'Try to login with another account.' % (CFG_SITE_SECURE_URL),
1: '
If you think this is not correct, please contact: %s' % (CFG_SITE_SUPPORT_EMAIL, CFG_SITE_SUPPORT_EMAIL),
2: '
If you have any questions, please write to %s' % (CFG_SITE_SUPPORT_EMAIL, CFG_SITE_SUPPORT_EMAIL),
3: 'Guest users are not allowed, please login.' % CFG_SITE_SECURE_URL,
4: 'The site is temporarily closed for maintenance. Please come back soon.',
5: 'Authorization failure',
6: '%s temporarily closed' % CFG_SITE_NAME,
7: 'This functionality is temporarily closed due to server maintenance. Please use only the search engine in the meantime.',
8: 'Functionality temporarily closed'
}
CFG_WEBACCESS_WARNING_MSGS = {
0: 'Authorization granted',
1: 'You are not authorized to perform this action.',
2: 'You are not authorized to perform any action.',
3: 'The action %s does not exist.',
4: 'Unexpected error occurred.',
5: 'Missing mandatory keyword argument(s) for this action.',
6: 'Guest accounts are not authorized to perform this action.',
7: 'Not enough arguments, user ID and action name required.',
8: 'Incorrect keyword argument(s) for this action.',
9: """Account '%s' is not yet activated.""",
10: """You were not authorized by the authentication method '%s'.""",
11: """The selected login method '%s' is not the default method for this account, please try another one.""",
12: """Selected login method '%s' does not exist.""",
13: """Could not register '%s' account.""",
14: """Could not login using '%s', because this user is unknown.""",
15: """Could not login using your '%s' account, because you have introduced a wrong password.""",
16: """External authentication troubles using '%s' (maybe temporary network problems).""",
17: """You have not yet confirmed the email address for the '%s' authentication method.""",
18: """The administrator has not yet activated your account for the '%s' authentication method.""",
19: """The site is having troubles in sending you an email for confirming your email address. The error has been logged and will be taken care of as soon as possible.""",
20: """No roles are authorized to perform action %s with the given parameters."""
}
+
+#There are three status key that must be here: OK, REMOVED and REVOKED
+#the value doesn't matter at all
+CFG_WEB_API_KEY_STATUS = {
+ 'OK':'OK',
+ 'REMOVED':'REMOVED',
+ 'REVOKED':'REVOKED',
+ 'WARNING':'WARNING'
+ }
diff --git a/modules/webaccess/lib/external_authentication.py b/modules/webaccess/lib/external_authentication.py
index 78d9bcd83..209348b5c 100644
--- a/modules/webaccess/lib/external_authentication.py
+++ b/modules/webaccess/lib/external_authentication.py
@@ -1,105 +1,106 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""External user authentication for Invenio."""
__revision__ = \
"$Id$"
class InvenioWebAccessExternalAuthError(Exception):
"""Exception to signaling general external trouble."""
pass
class ExternalAuth:
"""External authentication template example."""
def __init__(self, enforce_external_nicknames=False):
"""Initialize stuff here"""
self.name = None
# Set the following variable to True in order to import the externally
# provided nickname into Invenio during the first login of a user
# through this external authentication system.
# If the nickname is already taken into Invenio, then it won't be
# considered.
self.enforce_external_nicknames = enforce_external_nicknames
pass
def auth_user(self, username, password, req=None):
"""Authenticate user-supplied USERNAME and PASSWORD. Return
- None if authentication failed, or the email address of the
- person if the authentication was successful. In order to do
+ (None, None) if authentication failed, or the (email, ext_id), where
+ email is the email address of the person and ext_id is the external
+ identifier, if the authentication was successful. In order to do
this you may perhaps have to keep a translation table between
usernames and email addresses.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
"""
raise NotImplementedError()
- #return None
+ #return None, None
def user_exists(self, email, req=None):
"""Check the external authentication system for existance of email.
@return: True if the user exists, False otherwise
"""
raise NotImplementedError()
def fetch_user_groups_membership(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of groups
and their description to which the user is subscribed.
Raise InvenioWebAccessExternalAuthError in case of troubles.
"""
raise NotImplementedError()
#return {}
def fetch_user_nickname(self, username, password=None, req=None):
"""Given a username and a password, returns the right nickname belonging
to that user (username could be an email).
"""
raise NotImplementedError()
#return Nickname
def fetch_user_preferences(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of keys and
values, corresponding to external infos and settings.
userprefs = {"telephone": "2392489",
"address": "10th Downing Street"}
(WEBUSER WILL erase all prefs that starts by EXTERNAL_ and will
store: "EXTERNAL_telephone"; all internal preferences can use whatever
name but starting with EXTERNAL). If a pref begins with HIDDEN_ it will
be ignored.
"""
raise NotImplementedError()
#return {}
def fetch_all_users_groups_membership(self, req=None):
"""Fetch all the groups with a description, and users who belong to
each groups.
@return: {'mygroup': ('description', ['email1', 'email2', ...]), ...}
"""
raise NotImplementedError()
def robot_login_method_p():
"""Return True if this method is dedicated to robots and should
not therefore be available as a choice to regular users upon login.
"""
return False
robot_login_method_p = staticmethod(robot_login_method_p)
diff --git a/modules/webaccess/lib/external_authentication_cern.py b/modules/webaccess/lib/external_authentication_cern.py
index 21c8af622..631aaad9d 100644
--- a/modules/webaccess/lib/external_authentication_cern.py
+++ b/modules/webaccess/lib/external_authentication_cern.py
@@ -1,195 +1,196 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""External user authentication for CERN NICE/CRA Invenio."""
__revision__ = \
"$Id$"
import httplib
import socket
import re
from invenio.errorlib import register_exception
from invenio.external_authentication import ExternalAuth, \
InvenioWebAccessExternalAuthError
from invenio.external_authentication_cern_wrapper import AuthCernWrapper
# Tunable list of settings to be hidden
CFG_EXTERNAL_AUTH_CERN_HIDDEN_SETTINGS = ['auth', 'respccid', 'ccid']
# Tunable list of groups to be hidden
CFG_EXTERNAL_AUTH_HIDDEN_GROUPS = (
'All Exchange People',
'CERN Users',
'cern-computing-postmasters@cern.ch',
'cern-nice2000-postmasters@cern.ch',
'CMF FrontEnd Users',
'CMF_NSC_259_NSU',
'Domain Users',
'GP Apply Favorites Redirection',
'GP Apply NoAdmin',
'info-terminalservices@cern.ch',
'info-terminalservices-members@cern.ch',
'IT Web IT',
'NICE Deny Enforce Password-protected Screensaver',
'NICE Enforce Password-protected Screensaver',
'NICE LightWeight Authentication WS Users',
'NICE MyDocuments Redirection (New)',
'NICE Profile Redirection',
'NICE Terminal Services Users',
'NICE Users',
'NICE VPN Users',
)
CFG_EXTERNAL_AUTH_HIDDEN_GROUPS_RE = (
re.compile(r'Users by Letter [A-Z]'),
re.compile(r'building-[\d]+'),
re.compile(r'Users by Home CERNHOME[A-Z]'),
)
class ExternalAuthCern(ExternalAuth):
"""
External authentication example for a custom HTTPS-based
authentication service (called "CERN NICE").
"""
def __init__(self):
"""Initialize stuff here"""
ExternalAuth.__init__(self)
try:
self.connection = AuthCernWrapper()
except (httplib.CannotSendRequest, socket.error, AttributeError,
IOError, TypeError), msg: # Let the user note that
# no connection is available
register_exception(alert_admin=True)
raise InvenioWebAccessExternalAuthError, msg
def _try_twice(self, funct, params):
"""Try twice to execute funct on self.connection passing it params.
If for various reason the connection doesn't work it's restarted
"""
try:
ret = funct(self.connection, **params)
except (httplib.CannotSendRequest, socket.error, AttributeError,
IOError, TypeError):
try:
self.connection = AuthCernWrapper()
ret = funct(self.connection, **params)
except (httplib.CannotSendRequest, socket.error, AttributeError,
IOError, TypeError):
register_exception(alert_admin=True)
self.connection = None
raise InvenioWebAccessExternalAuthError
return ret
def auth_user(self, username, password, req=None):
"""
Check USERNAME and PASSWORD against CERN NICE/CRA database.
- Return None if authentication failed, or the email address of the
+ Return (None, None) if authentication failed, or the
+ (email address, nickname) of the
person if the authentication was successful. In order to do
this you may perhaps have to keep a translation table between
usernames and email addresses.
If it is the first time the user logs in Invenio the nickname is
stored alongside the email. If this nickname is unfortunatly already
in use it is discarded. Otherwise it is ignored.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
"""
infos = self._try_twice(funct=AuthCernWrapper.get_user_info, \
params={"user_name":username, "password":password})
if "email" in infos:
- return infos["email"]
+ return infos["email"], infos["login"]
else:
- return None
+ return None, None
def user_exists(self, email, req=None):
"""Checks against CERN NICE/CRA for existance of email.
@return: True if the user exists, False otherwise
"""
users = self._try_twice(funct=AuthCernWrapper.list_users, \
params={"display_name":email})
return email.upper() in [user['email'].upper() for user in users]
def fetch_user_groups_membership(self, email, password=None, req=None):
"""Fetch user groups membership from the CERN NICE/CRA account.
@return: a dictionary of groupname, group description
"""
groups = self._try_twice(funct=AuthCernWrapper.get_groups_for_user, \
params={"user_name":email})
# Filtering out uncomfortable groups
groups = [group for group in groups if group not in
CFG_EXTERNAL_AUTH_HIDDEN_GROUPS]
for regexp in CFG_EXTERNAL_AUTH_HIDDEN_GROUPS_RE:
for group in groups:
if regexp.match(group):
groups.remove(group)
# Produce list of double value: group/mailing list(with stripped
# @cern.ch) name, and group/description built from the name.
return dict(map(lambda x: (x.find('@') > -1 and x[:x.find('@')] or x,
'@' in x and x + ' (CERN Mailing list)'
or x + ' (CERN Group)'), groups))
def fetch_user_nickname(self, username, password, req=None):
"""Given a username and a password, returns the right nickname
belonging to that user (username could be an email).
"""
infos = self._try_twice(funct=AuthCernWrapper.get_user_info,
params={"user_name":username, "password":password})
if "login" in infos:
return infos["login"]
else:
return None
def fetch_user_preferences(self, username, password=None, req=None):
"""Fetch user preferences/settings from the CERN Nice account.
the external key will be '1' if the account is external to NICE/CRA,
otherwise 0
@return: a dictionary. Note: auth and respccid are hidden
"""
prefs = self._try_twice(funct=AuthCernWrapper.get_user_info, \
params={"user_name":username, "password":password})
ret = {}
try:
if int(prefs['auth']) == 3 \
and (int(prefs['respccid']) > 0 \
or not prefs['email'].endswith('@cern.ch')):
ret['external'] = '1'
else:
ret['external'] = '0'
except KeyError:
ret['external'] = '1'
for key, value in prefs.items():
if key in CFG_EXTERNAL_AUTH_CERN_HIDDEN_SETTINGS:
ret['HIDDEN_' + key] = value
else:
ret[key] = value
## Hack to be forward-compatible with CERN SSO implementation
if ret.has_key('company'):
ret['homeinstitute'] = ret['company']
del ret['company']
if ret.has_key('name'):
ret['fullname'] = ret['name']
del ret['name']
return ret
diff --git a/modules/webaccess/lib/external_authentication_ldap.py b/modules/webaccess/lib/external_authentication_ldap.py
index 0087b9058..2bcbc38ef 100644
--- a/modules/webaccess/lib/external_authentication_ldap.py
+++ b/modules/webaccess/lib/external_authentication_ldap.py
@@ -1,234 +1,234 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""External user authentication for EPFL's LDAP instance.
This LDAP external authentication system relies on a collaborative LDAP
organized like this:
o=EPFL, c=CH
|
|
+--ou=groups
| |
| |
| +--- cn=xxx
| displayName= name of the group
| uniqueIdentifier= some local id for groups
|
|
|
+--ou=users
| |
| |
| +---uid= some local id for users (ex: grfavre)
| uniqueIdentifier= another local id (ex: 128933)
| mail=xxx@xxx.xx
| memberOf= id of a group
| memberOf= id of another group
|
+
This example of an LDAP authentication should help you develop yours in your
specific installation.
"""
__revision__ = \
"$Id$"
import ldap
from invenio.external_authentication import ExternalAuth, \
InvenioWebAccessExternalAuthError
CFG_EXTERNAL_AUTH_LDAP_SERVERS = ['ldap://scoldap.epfl.ch']
CFG_EXTERNAL_AUTH_LDAP_CONTEXT = "o=EPFL,c=CH"
CFG_EXTERNAL_AUTH_LDAP_USER_UID = ["uid", "uniqueIdentifier", "mail"]
CFG_EXTERNAL_AUTH_LDAP_MAIL_ENTRY = 'mail'
CFG_EXTERNAL_AUTH_LDAP_GROUP_MEMBERSHIP = 'memberOf'
CFG_EXTERNAL_AUTH_LDAP_GROUP_UID = 'uniqueIdentifier'
CFG_EXTERNAL_AUTH_LDAP_GROUP_NAME = 'displayName'
CFG_EXTERNAL_AUTH_LDAP_HIDDEN_GROUPS = ['EPFL-unit', 'users']
class ExternalAuthLDAP(ExternalAuth):
"""
External authentication example for a custom LDAP-based
authentication service.
"""
def __init__(self):
"""Initialize stuff here"""
ExternalAuth.__init__(self)
self.enforce_external_nicknames = True
def _ldap_try (self, command):
""" Try to run the specified command on the first LDAP server that
is not down."""
for server in CFG_EXTERNAL_AUTH_LDAP_SERVERS:
try:
connection = ldap.initialize(server)
return command(connection)
except ldap.SERVER_DOWN, error_message:
continue
raise InvenioWebAccessExternalAuthError
def auth_user(self, username, password, req=None):
"""
Check USERNAME and PASSWORD against the LDAP system.
- Return None if authentication failed, or the email address of the
+ Return (None, None) if authentication failed, or the (email address, user_dn) of the
person if the authentication was successful.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
Note: for SSO the parameter are discarded and overloaded by Shibboleth
variables
"""
if not password:
- return None
+ return None, None
query = '(|' + ''.join (['(%s=%s)' % (attrib, username)
for attrib in
CFG_EXTERNAL_AUTH_LDAP_USER_UID]) \
+ ')'
def _check (connection):
users = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query)
# We pick the first result, as all the data we are interested
# in should be the same in all the entries.
if len(users):
user_dn, user_info = users [0]
else:
- return None
+ return None, None
try:
connection.simple_bind_s(user_dn, password)
except ldap.INVALID_CREDENTIALS:
# It is enough to fail on one server to consider the credential
# to be invalid
- return None
- return user_info[CFG_EXTERNAL_AUTH_LDAP_MAIL_ENTRY][0]
+ return None, None
+ return user_info[CFG_EXTERNAL_AUTH_LDAP_MAIL_ENTRY][0], user_dn
return self._ldap_try(_check)
def user_exists(self, email, req=None):
"""Check the external authentication system for existance of email.
@return: True if the user exists, False otherwise
"""
query = '(%s=%s)' % (CFG_EXTERNAL_AUTH_LDAP_MAIL_ENTRY, email)
def _check (connection):
users = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query)
return len(users) != 0
return self._ldap_try(_check)
def fetch_user_nickname(self, username, password=None, req=None):
"""Given a username and a password, returns the right nickname belonging
to that user (username could be an email).
"""
query = '(|' + ''.join (['(%s=%s)' % (attrib, username)
for attrib in
CFG_EXTERNAL_AUTH_LDAP_USER_UID]) \
+ ')'
def _get_nickname(connection):
users = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query)
# We pick the first result, as all the data we are interested
# in should be the same in all the entries.
if len(users):
user_dn, user_info = users [0]
else:
return None
emails = user_info[CFG_EXTERNAL_AUTH_LDAP_MAIL_ENTRY]
if len(emails):
email = emails[0]
else:
return False
(left_part, right_part) = email.split('@')
nickname = left_part.replace('.', ' ').title()
if right_part != 'epfl.ch':
nickname += ' - ' + right_part
return nickname
return self._ldap_try(_get_nickname)
def fetch_user_groups_membership(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of groups
and their description to which the user is subscribed.
Raise InvenioWebAccessExternalAuthError in case of troubles.
"""
query_person = '(|' + ''.join (['(%s=%s)' % (attrib, username)
for attrib in
CFG_EXTERNAL_AUTH_LDAP_USER_UID]) \
+ ')'
def _get_groups(connection):
users = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query_person)
if len(users):
user_dn, user_info = users [0]
else:
return {}
groups = {}
group_ids = user_info[CFG_EXTERNAL_AUTH_LDAP_GROUP_MEMBERSHIP]
for group_id in group_ids:
query_group = '(%s=%s)' % (CFG_EXTERNAL_AUTH_LDAP_GROUP_UID,
group_id)
ldap_group = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query_group)
if len(ldap_group):
group_dn, group_infos = ldap_group[0]
group_name = group_infos[CFG_EXTERNAL_AUTH_LDAP_GROUP_NAME][0]
if group_name in CFG_EXTERNAL_AUTH_LDAP_HIDDEN_GROUPS:
continue
groups[group_id] = group_name
return groups
return self._ldap_try(_get_groups)
def fetch_user_preferences(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of keys and
values, corresponding to external infos and settings.
userprefs = {"telephone": "2392489",
"address": "10th Downing Street"}
(WEBUSER WILL erase all prefs that starts by EXTERNAL_ and will
store: "EXTERNAL_telephone"; all internal preferences can use whatever
name but starting with EXTERNAL). If a pref begins with HIDDEN_ it will
be ignored.
"""
query = '(|' + ''.join (['(%s=%s)' % (attrib, username)
for attrib in
CFG_EXTERNAL_AUTH_LDAP_USER_UID]) \
+ ')'
def _get_personal_infos(connection):
users = connection.search_s(CFG_EXTERNAL_AUTH_LDAP_CONTEXT,
ldap.SCOPE_SUBTREE,
query)
if len(users):
user_dn, user_info = users [0]
return user_info
else:
return {}
return self._ldap_try(_get_personal_infos)
diff --git a/modules/webaccess/lib/external_authentication_robot.py b/modules/webaccess/lib/external_authentication_robot.py
index 1fc4a5a51..bea3609a6 100644
--- a/modules/webaccess/lib/external_authentication_robot.py
+++ b/modules/webaccess/lib/external_authentication_robot.py
@@ -1,407 +1,413 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""External user authentication for simple robots
This implement an external authentication system suitable for robots usage.
User attributes are retrieved directly from the form dictionary of the request
object.
"""
import os
import sys
import hmac
import time
import base64
if sys.hexversion < 0x2050000:
import sha as sha1
else:
from hashlib import sha1
from cPickle import dumps
from zlib import decompress, compress
from invenio.jsonutils import json, json_unicode_to_utf8
from invenio.shellutils import mymkdir
from invenio.external_authentication import ExternalAuth, InvenioWebAccessExternalAuthError
from invenio.config import CFG_ETCDIR, CFG_SITE_URL, CFG_SITE_SECURE_URL
CFG_ROBOT_EMAIL_ATTRIBUTE_NAME = 'email'
CFG_ROBOT_NICKNAME_ATTRIBUTE_NAME = 'nickname'
CFG_ROBOT_GROUPS_ATTRIBUTE_NAME = 'groups'
CFG_ROBOT_TIMEOUT_ATTRIBUTE_NAME = '__timeout__'
CFG_ROBOT_USERIP_ATTRIBUTE_NAME = '__userip__'
CFG_ROBOT_GROUPS_SEPARATOR = ';'
CFG_ROBOT_URL_TIMEOUT = 3600
CFG_ROBOT_KEYS_PATH = os.path.join(CFG_ETCDIR, 'webaccess', 'robot_keys.dat')
def normalize_ip(ip, up_to_bytes=4):
"""
@param up_to_bytes: set this to the number of bytes that should
be considered in the normalization. E.g. is this is set two 2, only the
first two bytes will be considered, while the remaining two will be set
to 0.
@return: a normalized IP, e.g. 123.02.12.12 -> 123.2.12.12
"""
try:
ret = []
for i, number in enumerate(ip.split(".")):
if i < up_to_bytes:
ret.append(str(int(number)))
else:
ret.append("0")
return '.'.join(ret)
except ValueError:
## e.g. if it's IPV6 ::1
return ip
def load_robot_keys():
"""
@return: the robot key dictionary.
"""
from cPickle import loads
from zlib import decompress
try:
robot_keys = loads(decompress(open(CFG_ROBOT_KEYS_PATH).read()))
if not isinstance(robot_keys, dict):
return {}
else:
return robot_keys
except:
return {}
class ExternalAuthRobot(ExternalAuth):
"""
This class implement an external authentication method suitable to be
used by an external service that, after having authenticated a user,
will provide a URL to the user that, once followed, will successfully
login the user into Invenio, with any detail the external service
decided to provide to the Invenio installation.
Such URL should be built as follows:
BASE?QUERY
where BASE is CFG_SITE_SECURE_URL/youraccount/robotlogin
and QUERY is a urlencoded mapping of the following key->values:
- assertion: an assertion, i.e. a piece of information describing the
user, see below for more details.
- robot: the identifier of the external service providing the assertion
- login_method: the name of the login method as defined in CFG_EXTERNAL_AUTHENTICATION.
- digest: the digest of the signature as detailed below.
- referer: the URL where the user should be redirected after successful
login (it is called referer as, for historical reasons, this is the
original URL of the page on which, a human-user has clicked "login".
the "assertion" should be a JSON serialized mapping with the following
keys:
- email: the email of the user (i.e. its identifier).
- nickname: optional nickname of the user.
- groups: an optional ';'-separated list of groups to which the user
belongs to.
- __timeout__: the number of seconds (floating point) from the Epoch,
after which the URL will no longer be valid. (expressed in UTC)
- __userip__: the IP address of the user for whom this URL has been
created. (if the user will follow this URL using a different URL the
request will not be valid)
- any other key can be added and will be merged in the external user
settings.
If L{use_zlib} is True the assertion is a base64-url-flavour encoding
of the zlib compression of the original assertion (useful for shortening
the URL while make it easy to type).
The "digest" is the hexadecimal representation of the digest using the
HMAC-SHA1 method to sign the assertion with the secret key associated
with the robot for the given login_method.
@param enforce_external_nicknames: whether to trust nicknames provided by
the external service and use them (if possible) as unique identifier
in the system.
@type enforce_external_nicknames: boolean
@param email_attribute_name: the actual key in the assertion that will
contain the email.
@type email_attribute_name: string
@param nickname_attribute_name: the actual key in the assertion that will
contain the nickname.
@type nickname_attribute_name: string
@param groups_attribute_name: the actual key in the assertion that will
contain the groups.
@type groups_attribute_name: string
@param groups_separator: the string used to separate groups.
@type groups_separator: string
@param timeout_attribute_name: the actual key in the assertion that will
contain the timeout.
@type timeout_attribute_name: string
@param userip_attribute_name: the actual key in the assertion that will
contain the user IP.
@type userip_attribute_name: string
+ @param external_id_attribute_name: the actual string that identifies the
+ user in the external authentication system. By default this is set
+ to be the same as the nickname, but this can be configured.
@param check_user_ip: whether to check for the IP address of the user
using the given URL, against the IP address stored in the assertion
to be identical. If 0, no IP check will be performed, if 1, only the
1st byte will be compared, if 2, only the first two bytes will be
compared, if 3, only the first three bytes, and if 4, the whole IP
address will be checked.
@type check_user_ip: int
@param use_zlib: whether to use base64-url-flavour encoding of the zlib
compression of the json serialization of the assertion or simply
the json serialization of the assertion.
@type use_zlib: boolean
"""
def __init__(self, enforce_external_nicknames=False,
email_attribute_name=CFG_ROBOT_EMAIL_ATTRIBUTE_NAME,
nickname_attribute_name=CFG_ROBOT_NICKNAME_ATTRIBUTE_NAME,
groups_attribute_name=CFG_ROBOT_GROUPS_ATTRIBUTE_NAME,
groups_separator=CFG_ROBOT_GROUPS_SEPARATOR,
timeout_attribute_name=CFG_ROBOT_TIMEOUT_ATTRIBUTE_NAME,
userip_attribute_name=CFG_ROBOT_USERIP_ATTRIBUTE_NAME,
check_user_ip=4,
+ external_id_attribute_name=CFG_ROBOT_NICKNAME_ATTRIBUTE_NAME,
use_zlib=True,
):
ExternalAuth.__init__(self, enforce_external_nicknames=enforce_external_nicknames)
self.email_attribute_name = email_attribute_name
self.nickname_attribute_name = nickname_attribute_name
self.groups_attribute_name = groups_attribute_name
self.groups_separator = groups_separator
self.timeout_attribute_name = timeout_attribute_name
self.userip_attribute_name = userip_attribute_name
+ self.external_id_attribute_name = external_id_attribute_name
self.check_user_ip = check_user_ip
self.use_zlib = use_zlib
def __extract_attribute(self, req):
"""
Load from the request the given assertion, extract all the attribute
to properly login the user, and verify that the data are actually
both well formed and signed correctly.
"""
from invenio.webinterface_handler import wash_urlargd
args = wash_urlargd(req.form, {
'assertion': (str, ''),
'robot': (str, ''),
'digest': (str, ''),
'login_method': (str, '')})
assertion = args['assertion']
digest = args['digest']
robot = args['robot']
login_method = args['login_method']
shared_key = load_robot_keys().get(login_method, {}).get(robot)
if shared_key is None:
raise InvenioWebAccessExternalAuthError("A key does not exist for robot: %s, login_method: %s" % (robot, login_method))
if not self.verify(shared_key, assertion, digest):
raise InvenioWebAccessExternalAuthError("The provided assertion does not validate against the digest %s for robot %s" % (repr(digest), repr(robot)))
if self.use_zlib:
try:
## Workaround to Perl implementation that does not add
## any padding to the base64 encoding.
needed_pad = (4 - len(assertion) % 4) % 4
assertion += needed_pad * '='
assertion = decompress(base64.urlsafe_b64decode(assertion))
except:
raise InvenioWebAccessExternalAuthError("The provided assertion is corrupted")
data = json_unicode_to_utf8(json.loads(assertion))
if not isinstance(data, dict):
raise InvenioWebAccessExternalAuthError("The provided assertion is invalid")
timeout = data[self.timeout_attribute_name]
if timeout < time.time():
raise InvenioWebAccessExternalAuthError("The provided assertion is expired")
userip = data.get(self.userip_attribute_name)
if not self.check_user_ip or (normalize_ip(userip, self.check_user_ip) == normalize_ip(req.remote_ip, self.check_user_ip)):
return data
else:
raise InvenioWebAccessExternalAuthError("The provided assertion has been issued for a different IP address (%s instead of %s)" % (userip, req.remote_ip))
def auth_user(self, username, password, req=None):
"""Authenticate user-supplied USERNAME and PASSWORD. Return
None if authentication failed, or the email address of the
person if the authentication was successful. In order to do
this you may perhaps have to keep a translation table between
usernames and email addresses.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
"""
data = self.__extract_attribute(req)
email = data.get(self.email_attribute_name)
+ ext_id = data.get(self.external_id_attribute_name, email)
if email:
if isinstance(email, str):
- return email.strip().lower()
+ return email.strip().lower(), ext_id.strip()
else:
raise InvenioWebAccessExternalAuthError("The email provided in the assertion is invalid: %s" % (repr(email)))
else:
- return None
+ return None, None
def fetch_user_groups_membership(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of groups
and their description to which the user is subscribed.
Raise InvenioWebAccessExternalAuthError in case of troubles.
"""
if self.groups_attribute_name:
data = self.__extract_attribute(req)
groups = data.get(self.groups_attribute_name)
if groups:
if isinstance(groups, str):
groups = [group.strip() for group in groups.split(self.groups_separator)]
return dict(zip(groups, groups))
else:
raise InvenioWebAccessExternalAuthError("The groups provided in the assertion are invalid: %s" % (repr(groups)))
return {}
def fetch_user_nickname(self, username, password=None, req=None):
"""Given a username and a password, returns the right nickname belonging
to that user (username could be an email).
"""
if self.nickname_attribute_name:
data = self.__extract_attribute(req)
nickname = data.get(self.nickname_attribute_name)
if nickname:
if isinstance(nickname, str):
return nickname.strip().lower()
else:
raise InvenioWebAccessExternalAuthError("The nickname provided in the assertion is invalid: %s" % (repr(nickname)))
return None
def fetch_user_preferences(self, username, password=None, req=None):
"""Given a username and a password, returns a dictionary of keys and
values, corresponding to external infos and settings.
userprefs = {"telephone": "2392489",
"address": "10th Downing Street"}
(WEBUSER WILL erase all prefs that starts by EXTERNAL_ and will
store: "EXTERNAL_telephone"; all internal preferences can use whatever
name but starting with EXTERNAL). If a pref begins with HIDDEN_ it will
be ignored.
"""
data = self.__extract_attribute(req)
for key in (self.email_attribute_name, self.groups_attribute_name, self.nickname_attribute_name, self.timeout_attribute_name, self.userip_attribute_name):
if key and key in data:
del data[key]
return data
def robot_login_method_p():
"""Return True if this method is dedicated to robots and should
not therefore be available as a choice to regular users upon login.
"""
return True
robot_login_method_p = staticmethod(robot_login_method_p)
def sign(secret, assertion):
"""
@return: a signature of the given assertion.
@rtype: string
@note: override this method if you want to change the signature
algorithm (e.g. to use GPG).
@see: L{verify}
"""
return hmac.new(secret, assertion, sha1).hexdigest()
sign = staticmethod(sign)
def verify(secret, assertion, signature):
"""
@return: True if the signature is valid
@rtype: boolean
@note: override this method if you want to change the signature
algorithm (e.g. to use GPG)
@see: L{sign}
"""
return hmac.new(secret, assertion, sha1).hexdigest() == signature
verify = staticmethod(verify)
def test_create_example_url(self, email, login_method, robot, ip, assertion=None, timeout=None, referer=None, groups=None, nickname=None):
"""
Create a test URL to test the robot login.
@param email: email of the user we want to login as.
@type email: string
@param login_method: the login_method name as specified in CFG_EXTERNAL_AUTHENTICATION.
@type login_method: string
@param robot: the identifier of this robot.
@type robot: string
@param assertion: any further data we want to send to.
@type: json serializable mapping
@param ip: the IP of the user.
@type: string
@param timeout: timeout when the URL will expire (in seconds from the Epoch)
@type timeout: float
@param referer: the URL where to land after successful login.
@type referer: string
@param groups: the list of optional group of the user.
@type groups: list of string
@param nickname: the optional nickname of the user.
@type nickname: string
@return: the URL to login as the user.
@rtype: string
"""
from invenio.access_control_config import CFG_EXTERNAL_AUTHENTICATION
from invenio.urlutils import create_url
if assertion is None:
assertion = {}
assertion[self.email_attribute_name] = email
if nickname:
assertion[self.nickname_attribute_name] = nickname
if groups:
assertion[self.groups_attribute_name] = self.groups_separator.join(groups)
if timeout is None:
timeout = time.time() + CFG_ROBOT_URL_TIMEOUT
assertion[self.timeout_attribute_name] = timeout
if referer is None:
referer = CFG_SITE_URL
if login_method is None:
for a_login_method, details in CFG_EXTERNAL_AUTHENTICATION.iteritems():
if details[2]:
login_method = a_login_method
break
robot_keys = load_robot_keys()
assertion[self.userip_attribute_name] = ip
assertion = json.dumps(assertion)
if self.use_zlib:
assertion = base64.urlsafe_b64encode(compress(assertion))
shared_key = robot_keys[login_method][robot]
digest = self.sign(shared_key, assertion)
return create_url("%s%s" % (CFG_SITE_SECURE_URL, "/youraccount/robotlogin"), {
'assertion': assertion,
'robot': robot,
'login_method': login_method,
'digest': digest,
'referer': referer})
def update_robot_key(login_method, robot, key=None):
"""
Utility to update the robot key store.
@param login_method: the login_method name as per L{CFG_EXTERNAL_AUTHENTICATION}.
It should correspond to a robot-enable login method.
@type: string
@param robot: the robot identifier
@type robot: string
@param key: the secret
@type key: string
@note: if the secret is empty the corresponding key will be removed.
"""
robot_keys = load_robot_keys()
if key is None and login_method in robot_keys and robot in robot_keys[login_method]:
del robot_keys[login_method][robot]
if not robot_keys[login_method]:
del robot_keys[login_method]
else:
if login_method not in robot_keys:
robot_keys[login_method] = {}
robot_keys[login_method][robot] = key
mymkdir(os.path.join(CFG_ETCDIR, 'webaccess'))
open(CFG_ROBOT_KEYS_PATH, 'w').write(compress(dumps(robot_keys, -1)))
diff --git a/modules/webaccess/lib/external_authentication_sso.py b/modules/webaccess/lib/external_authentication_sso.py
index 942da99f9..493019cd0 100644
--- a/modules/webaccess/lib/external_authentication_sso.py
+++ b/modules/webaccess/lib/external_authentication_sso.py
@@ -1,201 +1,202 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""External user authentication for CERN NICE/CRA Invenio."""
__revision__ = \
"$Id$"
import re
from invenio.external_authentication import ExternalAuth
# Tunable list of settings to be hidden
CFG_EXTERNAL_AUTH_HIDDEN_SETTINGS = ('auth', 'respccid', 'personid')
# Tunable list of groups to be hidden
CFG_EXTERNAL_AUTH_HIDDEN_GROUPS = (
'All Exchange People',
'CERN Users',
'cern-computing-postmasters',
'cern-nice2000-postmasters',
'CMF FrontEnd Users',
'CMF_NSC_259_NSU',
'Domain Users',
'GP Apply Favorites Redirection',
'GP Apply NoAdmin',
'info-terminalservices',
'info-terminalservices-members',
'IT Web IT',
'NICE Deny Enforce Password-protected Screensaver',
'NICE Enforce Password-protected Screensaver',
'NICE LightWeight Authentication WS Users',
'NICE MyDocuments Redirection (New)',
'NICE Profile Redirection',
'NICE Terminal Services Users',
'NICE Users',
'NICE VPN Users',
)
CFG_EXTERNAL_AUTH_HIDDEN_GROUPS_RE = (
re.compile(r'Users by Letter [A-Z]'),
re.compile(r'building-[\d]+'),
re.compile(r'Users by Home CERNHOME[A-Z]'),
)
# Prefix name for Shibboleth variables
CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME = 'ADFS_'
# Name of the variable containing groups
CFG_EXTERNAL_AUTH_SSO_GROUP_VARIABLE = CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME+'GROUP'
# Name of the variable containing login name
CFG_EXTERNAL_AUTH_SSO_LOGIN_VARIABLE = CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME+'LOGIN'
# Name of the variable containing email
CFG_EXTERNAL_AUTH_SSO_EMAIL_VARIABLE = CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME+'EMAIL'
# Name of the variable containing groups
CFG_EXTERNAL_AUTH_SSO_GROUP_VARIABLE = CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME+'GROUP'
# Separator character for group variable
CFG_EXTERNAL_AUTH_SSO_GROUPS_SEPARATOR = ';'
class ExternalAuthSSO(ExternalAuth):
"""
External authentication example for a custom SSO-based
authentication service.
"""
def __init__(self, enforce_external_nicknames=False):
"""Initialize stuff here"""
ExternalAuth.__init__(self, enforce_external_nicknames)
self.egroup_cache = None
def in_shibboleth(self, req):
"""
Return True if the current request handler is actually under
Shibboleth control.
"""
return req.subprocess_env.has_key(CFG_EXTERNAL_AUTH_SSO_EMAIL_VARIABLE)
def auth_user(self, username, password, req=None):
"""
Check USERNAME and PASSWORD against the SSO system.
- Return None if authentication failed, or the email address of the
+ Return (None, None) if authentication failed, or the
+ (email address, nickname) of the
person if the authentication was successful. In order to do
this you may perhaps have to keep a translation table between
usernames and email addresses.
If it is the first time the user logs in Invenio the nickname is
stored alongside the email. If this nickname is unfortunatly already
in use it is discarded. Otherwise it is ignored.
Raise InvenioWebAccessExternalAuthError in case of external troubles.
Note: for SSO the parameter are discarded and overloaded by Shibboleth
variables
"""
if req:
req.add_common_vars()
if req.subprocess_env.has_key(CFG_EXTERNAL_AUTH_SSO_EMAIL_VARIABLE):
- return req.subprocess_env[CFG_EXTERNAL_AUTH_SSO_EMAIL_VARIABLE]
- return None
+ return req.subprocess_env[CFG_EXTERNAL_AUTH_SSO_EMAIL_VARIABLE], req.subprocess_env[CFG_EXTERNAL_AUTH_SSO_LOGIN_VARIABLE]
+ return None, None
#def user_exists(self, email, req=None):
#"""Checks against CERN NICE/CRA for existance of email.
#@return: True if the user exists, False otherwise
#"""
#users = self._try_twice(funct=AuthCernWrapper.list_users, \
#params={"display_name":email})
#return email.upper() in [user['email'].upper() for user in users]
def fetch_user_groups_membership(self, email, password=None, req=None):
"""Fetch user groups membership from the SSO system.
@return: a dictionary of groupname, group description
Note: for SSO the parameter are discarded and overloaded by Shibboleth
variables
"""
return self._fetch_egroups(req)
def fetch_user_nickname(self, username, password=None, req=None):
"""Given a username and a password, returns the right nickname belonging
to that user (username could be an email).
Note: for SSO the parameter are discarded and overloaded by Shibboleth
variables
"""
if req:
req.add_common_vars()
if req.subprocess_env.has_key(CFG_EXTERNAL_AUTH_SSO_LOGIN_VARIABLE):
return req.subprocess_env[CFG_EXTERNAL_AUTH_SSO_LOGIN_VARIABLE]
else:
return None
def _fetch_egroups(self, req=None):
if False: #self.egroup_cache is not None:
return self.egroup_cache
elif req:
req.add_common_vars()
if req.subprocess_env.has_key(CFG_EXTERNAL_AUTH_SSO_GROUP_VARIABLE):
groups = req.subprocess_env[CFG_EXTERNAL_AUTH_SSO_GROUP_VARIABLE].split(CFG_EXTERNAL_AUTH_SSO_GROUPS_SEPARATOR)
# Filtering out uncomfortable groups
groups = [group for group in groups if group not in CFG_EXTERNAL_AUTH_HIDDEN_GROUPS]
for regexp in CFG_EXTERNAL_AUTH_HIDDEN_GROUPS_RE:
for group in groups:
if regexp.match(group):
groups.remove(group)
self.egroup_cache = dict(map(lambda x: (x, '@' in x and x + ' (Mailing list)' \
or x + ' (Group)'), groups))
return self.egroup_cache
return {}
def _fetch_particular_preferences(self, req=None):
"""This hidden method is there to be overwritten in order to get some
particular value from non standard variables.
"""
if req:
ret = {}
req.add_common_vars()
if req.subprocess_env.has_key('HTTP_SHIB_AUTHENTICATION_METHOD'):
ret['authmethod'] = req.subprocess_env['HTTP_SHIB_AUTHENTICATION_METHOD']
egroups = self._fetch_egroups(req)
ret['external'] = 'CERN External Users' in egroups and '1' or '0'
return ret
return {}
def fetch_user_preferences(self, username, password=None, req=None):
"""Fetch user preferences/settings from the SSO account.
the external key will be '1' if the account is external to SSO,
otherwise 0
@return: a dictionary.
Note: for SSO the parameter are discarded and overloaded by Shibboleth
variables
"""
if req:
req.add_common_vars()
ret = {}
prefs = self._fetch_particular_preferences(req)
for key, value in req.subprocess_env.items():
if key.startswith(CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME) and not key == CFG_EXTERNAL_AUTH_SSO_GROUP_VARIABLE:
prefs[key[len(CFG_EXTERNAL_AUTH_SSO_PREFIX_NAME):].lower()] = value
for key, value in prefs.items():
if key in CFG_EXTERNAL_AUTH_HIDDEN_SETTINGS:
ret['HIDDEN_' + key] = value
else:
ret[key] = value
return ret
return {}
diff --git a/modules/webaccess/lib/webaccess_regression_tests.py b/modules/webaccess/lib/webaccess_regression_tests.py
index 3a9b24bbd..d96292cf4 100644
--- a/modules/webaccess/lib/webaccess_regression_tests.py
+++ b/modules/webaccess/lib/webaccess_regression_tests.py
@@ -1,241 +1,305 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""WebAccess Regression Test Suite."""
__revision__ = "$Id$"
import unittest
import socket
import time
import cgi
from urlparse import urlparse, urlunparse
from urllib import urlopen, urlencode
from invenio.access_control_admin import acc_add_role, acc_delete_role, \
acc_get_role_definition
from invenio.access_control_firerole import compile_role_definition, \
serialize, deserialize
from invenio.config import CFG_SITE_URL, CFG_SITE_SECURE_URL, CFG_DEVEL_SITE
from invenio.testutils import make_test_suite, run_test_suite, \
test_web_page_content, merge_error_messages
+from invenio.dbquery import run_sql
class WebAccessWebPagesAvailabilityTest(unittest.TestCase):
"""Check WebAccess web pages whether they are up or not."""
def test_webaccess_admin_interface_availability(self):
"""webaccess - availability of WebAccess Admin interface pages"""
baseurl = CFG_SITE_URL + '/admin/webaccess/webaccessadmin.py/'
_exports = ['', 'delegate_startarea', 'manageaccounts']
error_messages = []
for url in [baseurl + page for page in _exports]:
# first try as guest:
error_messages.extend(test_web_page_content(url,
username='guest',
expected_text=
'Authorization failure'))
# then try as admin:
error_messages.extend(test_web_page_content(url,
username='admin'))
if error_messages:
self.fail(merge_error_messages(error_messages))
return
def test_webaccess_admin_guide_availability(self):
"""webaccess - availability of WebAccess Admin guide pages"""
url = CFG_SITE_URL + '/help/admin/webaccess-admin-guide'
error_messages = test_web_page_content(url,
expected_text="WebAccess Admin Guide")
if error_messages:
self.fail(merge_error_messages(error_messages))
return
class WebAccessFireRoleTest(unittest.TestCase):
"""Check WebAccess behaviour WRT FireRole."""
def setUp(self):
"""Create a fake role."""
self.role_name = 'test'
self.role_description = 'test role'
self.role_definition = 'allow email /.*@cern.ch/'
self.role_id, dummy, dummy, dummy = acc_add_role(self.role_name,
self.role_description,
serialize(compile_role_definition(self.role_definition)),
self.role_definition)
def tearDown(self):
"""Drop the fake role."""
acc_delete_role(self.role_id)
def test_webaccess_firerole_serialization(self):
"""webaccess - firerole role definition correctly serialized"""
def_ser = compile_role_definition(self.role_definition)
tmp_def_ser = acc_get_role_definition(self.role_id)
self.assertEqual(def_ser, deserialize(tmp_def_ser))
class WebAccessUseBasketsTest(unittest.TestCase):
"""
Check WebAccess behaviour WRT enabling/disabling web modules such
as baskets.
"""
def test_precached_area_authorization(self):
"""webaccess - login-time precached authorizations for usebaskets"""
error_messages = test_web_page_content(CFG_SITE_SECURE_URL + '/youraccount/display?ln=en', username='jekyll', password='j123ekyll', expected_text='Your Baskets')
error_messages.extend(test_web_page_content(CFG_SITE_SECURE_URL + '/youraccount/display?ln=en', username='hyde', password='h123yde', unexpected_text='Your Baskets'))
if error_messages:
self.fail(merge_error_messages(error_messages))
if CFG_DEVEL_SITE:
class WebAccessRobotLoginTest(unittest.TestCase):
"""
Check whether robot login functionality is OK.
"""
def _erase_example_user_and_groups(self):
- from invenio.dbquery import run_sql
- uid = run_sql("SELECT id FROM user WHERE email=%s", (self.a_email, ))
- if uid:
- run_sql("DELETE FROM user WHERE id=%s", (uid[0][0], ))
- run_sql("DELETE FROM user_usergroup WHERE id_user=%s", (uid[0][0], ))
- for method_name in self.robot_login_methods:
- for group in self.some_groups:
- run_sql("DELETE FROM usergroup WHERE name=%s", (group + " [" + method_name + "]",))
+ for email in (self.a_email, self.another_email):
+ uid = run_sql("SELECT id FROM user WHERE email=%s", (email, ))
+ if uid:
+ run_sql("DELETE FROM user WHERE id=%s", (uid[0][0], ))
+ run_sql("DELETE FROM user_usergroup WHERE id_user=%s", (uid[0][0], ))
+ run_sql("DELETE FROM userEXT WHERE id_user=%s", (uid[0][0], ))
+ for method_name in self.robot_login_methods:
+ for group in self.some_groups:
+ run_sql("DELETE FROM usergroup WHERE name=%s", ("%s [%s]" % (group, method_name), ))
+ for nickname in (self.a_nickname, self.another_nickname):
+ run_sql("DELETE FROM userEXT WHERE id=%s", (nickname, ))
def setUp(self):
from invenio.access_control_config import CFG_EXTERNAL_AUTHENTICATION
self.robot_login_methods = dict([(method_name, CFG_EXTERNAL_AUTHENTICATION[method_name]) for method_name in CFG_EXTERNAL_AUTHENTICATION if CFG_EXTERNAL_AUTHENTICATION[method_name] and CFG_EXTERNAL_AUTHENTICATION[method_name].robot_login_method_p()])
self.a_robot = "regression-test"
self.a_password = "123"
self.a_email = "foo.bar@example.org"
+ self.another_email = "baz@example.org"
self.a_nickname = "foo-bar"
+ self.another_nickname = "baz"
self.some_groups = ["a group for regression test", "another group for regression test"]
self.myip = urlopen(CFG_SITE_URL + "/httptest/whatismyip").read()
from invenio.external_authentication_robot import update_robot_key
for method_name in self.robot_login_methods:
update_robot_key(method_name, self.a_robot, self.a_password)
from invenio.external_authentication_robot import load_robot_keys
def tearDown(self):
from invenio.external_authentication_robot import update_robot_key
#for method_name in self.robot_login_methods:
#update_robot_key(method_name, self.a_robot)
from invenio.external_authentication_robot import load_robot_keys
self._erase_example_user_and_groups()
def test_normal_robot_login_method(self):
"""webaccess - robot login method"""
for method_name, method in self.robot_login_methods.iteritems():
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip)
try:
error_messages = test_web_page_content(url, expected_text=self.a_email)
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
def test_robot_login_method_with_nickname(self):
"""webaccess - robot login method with nickname"""
for method_name, method in self.robot_login_methods.iteritems():
if method.enforce_external_nicknames:
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, nickname=self.a_nickname)
try:
error_messages = test_web_page_content(url, expected_text=self.a_nickname)
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
def test_robot_login_method_with_groups(self):
"""webaccess - robot login method with groups"""
for method_name, method in self.robot_login_methods.iteritems():
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, groups=self.some_groups, referer=CFG_SITE_SECURE_URL + "/yourgroups/display")
try:
for group in self.some_groups:
error_messages = test_web_page_content(url, expected_text="%s [%s]" % (group, method_name))
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
def test_robot_login_method_wrong_ip(self):
"""webaccess - robot login method wrong IP"""
for method_name, method in self.robot_login_methods.iteritems():
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, '123.123.123.123')
try:
error_messages = test_web_page_content(url, expected_text="The provided assertion has been issued for a different IP address")
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
def test_robot_login_method_expired_assertion(self):
"""webaccess - robot login method with expired assertion"""
for method_name, method in self.robot_login_methods.iteritems():
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, timeout=time.time())
time.sleep(1)
try:
error_messages = test_web_page_content(url, expected_text="The provided assertion is expired")
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
def test_robot_login_method_with_invalid_signature(self):
"""webaccess - robot login method with invalid signature"""
for method_name, method in self.robot_login_methods.iteritems():
url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip)
url = list(urlparse(url))
query = cgi.parse_qs(url[4])
for key, value in query.items():
query[key] = value[0]
digest = query['digest']
digest0 = digest[0]
if digest0 == '0':
digest0 = '1'
else:
digest0 = '0'
digest = digest0 + digest[1:]
query['digest'] = digest
url[4] = urlencode(query)
url = urlunparse(url)
try:
error_messages = test_web_page_content(url, expected_text="does not validate against the digest")
if error_messages:
self.fail(merge_error_messages(error_messages))
finally:
self._erase_example_user_and_groups()
+ def test_robot_login_method_changed_email(self):
+ """webaccess - robot login method changed email"""
+ for method_name, method in self.robot_login_methods.iteritems():
+ url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, nickname=self.a_nickname)
+ url2 = method.test_create_example_url(self.another_email, method_name, self.a_robot, self.myip, nickname=self.a_nickname)
+ try:
+ error_messages = test_web_page_content(url, expected_text=self.a_nickname)
+ if error_messages:
+ self.fail(merge_error_messages(error_messages))
+ id_user = run_sql("SELECT id FROM user WHERE email=%s", (self.a_email, ))[0][0]
+ self.failUnless(run_sql("SELECT * FROM userEXT WHERE id=%s AND id_user=%s AND method=%s", (self.a_nickname, id_user, method_name)), "Can't find id %s for user %s with metod %s. userEXT contains: %s" % (self.a_nickname, id_user, method_name, run_sql("SELECT * FROM userEXT")))
+ error_messages = test_web_page_content(url2, expected_text=self.a_nickname)
+ if error_messages:
+ self.fail(merge_error_messages(error_messages))
+ id_user2 = run_sql("SELECT id FROM user WHERE email=%s", (self.another_email, ))[0][0]
+ self.assertEqual(id_user, id_user2)
+ self.failUnless(run_sql("SELECT * FROM userEXT WHERE id=%s AND id_user=%s AND method=%s", (self.a_nickname, id_user2, method_name)))
+ ## The old email should not exist any longer.
+ self.failIf(run_sql("SELECT * FROM user WHERE email=%s", (self.a_email, )))
+ finally:
+ self._erase_example_user_and_groups()
+
+ def test_robot_login_method_merging_accounts(self):
+ """webaccess - robot login method merging accounts"""
+ for method_name, method in self.robot_login_methods.iteritems():
+ url = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, nickname=self.a_nickname)
+ url2 = method.test_create_example_url(self.another_email, method_name, self.a_robot, self.myip, nickname=self.another_nickname)
+ url3 = method.test_create_example_url(self.a_email, method_name, self.a_robot, self.myip, nickname=self.another_nickname)
+ try:
+ error_messages = test_web_page_content(url, expected_text=self.a_nickname)
+ if error_messages:
+ self.fail(merge_error_messages(error_messages))
+ id_user = run_sql("SELECT id FROM user WHERE email=%s", (self.a_email, ))[0][0]
+ self.failUnless(run_sql("SELECT * FROM userEXT WHERE id=%s AND id_user=%s AND method=%s", (self.a_nickname, id_user, method_name)))
+ error_messages = test_web_page_content(url2, expected_text=self.another_nickname)
+ if error_messages:
+ self.fail(merge_error_messages(error_messages))
+ id_user2 = run_sql("SELECT id FROM user WHERE email=%s", (self.another_email, ))[0][0]
+ self.failIfEqual(id_user, id_user2)
+ self.failUnless(run_sql("SELECT * FROM userEXT WHERE id=%s AND id_user=%s AND method=%s", (self.another_nickname, id_user2, method_name)), "Can't find id %s for user %s with metod %s. userEXT contains: %s" % (self.another_nickname, id_user2, method_name, run_sql("SELECT * FROM userEXT")))
+ ## The first email should still exists
+ self.failUnless(run_sql("SELECT * FROM user WHERE email=%s", (self.a_email, )))
+ ## We log in with the 1st email but with the second nickname.
+ ## That means the 1st user should be merged into the second.
+ ## However we still check for the 1st nickname, as in Invenio,
+ ## once a nickname is assigned it will never change!
+ error_messages = test_web_page_content(url3, expected_text=self.a_nickname)
+ if error_messages:
+ self.fail(merge_error_messages(error_messages))
+ ## The another_email should not exist any longer.
+ self.failIf(run_sql("SELECT * FROM user WHERE email=%s", (self.another_email, )), "%s still exists! while it should have been merged into %s: %s, userEXT contains: %s" % (self.another_email, self.a_email, run_sql("SELECT * FROM user WHERE email=%s", (self.another_email, )), run_sql("SELECT * FROM userEXT")))
+ ## And the corresponding user should not exist anymore as it has been
+ ## merged into id_user
+ self.failIf(run_sql("SELECT * FROM user WHERE id=%s", (id_user2, )))
+ self.failUnless(run_sql("SELECT * FROM user WHERE id=%s AND email=%s", (id_user, self.a_email)))
+ finally:
+ self._erase_example_user_and_groups()
+
TEST_SUITE = make_test_suite(WebAccessWebPagesAvailabilityTest,
WebAccessFireRoleTest,
WebAccessUseBasketsTest,
WebAccessRobotLoginTest)
else:
TEST_SUITE = make_test_suite(WebAccessWebPagesAvailabilityTest,
WebAccessFireRoleTest,
WebAccessUseBasketsTest)
if __name__ == "__main__":
run_test_suite(TEST_SUITE, warn_user=True)
diff --git a/modules/webaccess/lib/webaccessadmin_lib.py b/modules/webaccess/lib/webaccessadmin_lib.py
index 239434ce1..033f370f6 100644
--- a/modules/webaccess/lib/webaccessadmin_lib.py
+++ b/modules/webaccess/lib/webaccessadmin_lib.py
@@ -1,3838 +1,3884 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Invenio WebAccess Administrator Interface."""
__revision__ = "$Id$"
__lastupdated__ = """$Date$"""
## fill config variables:
import re
import random
import getopt
import sys
import time
from invenio.config import \
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS, \
CFG_ACCESS_CONTROL_LEVEL_GUESTS, \
CFG_ACCESS_CONTROL_LEVEL_SITE, \
CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN, \
CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS, \
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION, \
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION, \
CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_SITE_ADMIN_EMAIL, \
CFG_SITE_SECURE_URL
import invenio.access_control_engine as acce
import invenio.access_control_admin as acca
from invenio.mailutils import send_email
from invenio.errorlib import register_exception
from invenio.bibrankadminlib import addadminbox, tupletotable, \
tupletotable_onlyselected, addcheckboxes, createhiddenform
from invenio.access_control_firerole import compile_role_definition, \
repair_role_definitions, serialize
from invenio.messages import gettext_set_language
from invenio.dbquery import run_sql, OperationalError, wash_table_column_name
from invenio.webpage import page
from invenio.webuser import getUid, isGuestUser, page_not_authorized, collect_user_info
from invenio.webuser import email_valid_p, get_user_preferences, \
set_user_preferences, update_Uid
from invenio.urlutils import redirect_to_url, wash_url_argument
from invenio.access_control_config import DEF_DEMO_USER_ROLES, \
DEF_DEMO_ROLES, DEF_DEMO_AUTHS, WEBACCESSACTION, MAXPAGEUSERS, \
SUPERADMINROLE, CFG_EXTERNAL_AUTHENTICATION, DELEGATEADDUSERROLE, \
CFG_ACC_EMPTY_ROLE_DEFINITION_SRC, InvenioWebAccessFireroleError, \
- MAXSELECTUSERS, CFG_EXTERNAL_AUTH_DEFAULT
+ MAXSELECTUSERS, CFG_EXTERNAL_AUTH_DEFAULT, CFG_WEB_API_KEY_STATUS
from invenio.bibtask import authenticate
from cgi import escape
def index(req, title='', body='', subtitle='', adminarea=2, authorized=0, ln=CFG_SITE_LANG):
"""main function to show pages for webaccessadmin.
1. if user not logged in and administrator, show the mustlogin page
2. if used without body argument, show the startpage
3. show admin page with title, body, subtitle and navtrail.
authorized - if 1, don't check if the user is allowed to be webadmin """
navtrail_previous_links = 'Admin Area' \
'' % (CFG_SITE_SECURE_URL,)
if body:
if adminarea == 1:
navtrail_previous_links += '> ' \
'Delegate Rights ' % (CFG_SITE_SECURE_URL, )
if adminarea >= 2 and adminarea < 7:
navtrail_previous_links += '> ' \
'' \
'WebAccess Admin ' % (CFG_SITE_SECURE_URL, )
if adminarea == 3:
navtrail_previous_links += '> ' \
'Role Administration ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 4:
navtrail_previous_links += '> ' \
'Action Administration ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 5:
navtrail_previous_links += '> ' \
'User Administration ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 6:
navtrail_previous_links += '> ' \
'Reset Authorizations ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 7:
navtrail_previous_links += '> ' \
'Manage Accounts ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 8:
navtrail_previous_links += '> ' \
'List Groups ' % (CFG_SITE_SECURE_URL, )
elif adminarea == 9:
navtrail_previous_links += '> ' \
'Manage Robot Login ' % (CFG_SITE_SECURE_URL, )
id_user = getUid(req)
(auth_code, auth_message) = is_adminuser(req)
if not authorized and auth_code != 0:
return mustloginpage(req, auth_message)
elif not body:
title = 'WebAccess Admin'
body = startpage()
elif type(body) != str: body = addadminbox(subtitle, datalist=body)
return page(title=title,
uid=id_user,
req=req,
body=body,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__)
def mustloginpage(req, message):
"""show a page asking the user to login."""
navtrail_previous_links = '' \
'Admin Area > ' \
'WebAccess Admin ' % (CFG_SITE_SECURE_URL, CFG_SITE_SECURE_URL)
return page_not_authorized(req=req, text=message,
navtrail=navtrail_previous_links)
def is_adminuser(req):
"""check if user is a registered administrator. """
return acce.acc_authorize_action(req, WEBACCESSACTION)
def perform_managerobotlogin(req, robot_name='', new_pwd1='', new_pwd2='', login_method='', timeout='', referer='', ip='', action='', confirm=0, email='', groups='', nickname='', json_assertion='', url_only=0):
robot_name = wash_url_argument(robot_name, 'str')
new_pwd1 = wash_url_argument(new_pwd1, 'str')
new_pwd2 = wash_url_argument(new_pwd2, 'str')
login_method = wash_url_argument(login_method, 'str')
timeout = wash_url_argument(timeout, 'int')
referer = wash_url_argument(referer, 'str')
ip = wash_url_argument(ip, 'str')
action = wash_url_argument(action, 'str')
confirm = wash_url_argument(confirm, 'int')
email = wash_url_argument(email, 'str')
groups = wash_url_argument(groups, 'str')
nickname = wash_url_argument(nickname, 'str')
url_only = wash_url_argument(url_only, 'int')
json_assertion = wash_url_argument(json_assertion, 'str')
from invenio.external_authentication_robot import update_robot_key, load_robot_keys, json
(auth_code, auth_message) = acce.acc_authorize_action(req, 'cfgrobotkeys', login_method='*', robot='*')
if auth_code != 0: return mustloginpage(req, auth_message)
available_robot_login_methods = [name for (name, method) in CFG_EXTERNAL_AUTHENTICATION.iteritems() if method and method.robot_login_method_p()]
errors = []
warnings = []
messages = []
if not available_robot_login_methods:
errors.append("""
You should enable at least on robot based login method in access_control_config.py in the variable CFG_EXTERNAL_AUTHENTICATION.
""")
forms = ""
else:
robot_keys = load_robot_keys()
if not login_method:
login_method = available_robot_login_methods[0]
if not timeout:
timeout = 60 * 60
if not ip:
ip = req.remote_ip
user_info = collect_user_info(req)
if not email:
email = user_info['email']
if not nickname:
nickname = user_info['nickname']
if not robot_name:
if login_method in robot_keys and robot_keys[login_method]:
robot_name = robot_keys[login_method].keys()[0]
if not referer:
referer = CFG_SITE_SECURE_URL
if action == 'changepwd':
if acce.acc_authorize_action(user_info, 'cfgrobotkeys', login_method=login_method, robot=robot_name)[0]:
errors.append("""You don't have proper authorization to modify robot %s for login_method %s.""" % (escape(robot_name), escape(login_method)))
if login_method not in available_robot_login_methods:
errors.append("""The login method must be one among the available_robot_login_methods (%s).""" % escape(', '.join(available_robot_login_methods)))
if new_pwd1 != new_pwd2:
errors.append("""The two passwords are not equal.""")
new_pwd1 = ''
new_pwd2 = ''
if not robot_name:
errors.append("""The robot name must be specified.""")
if int(confirm) == 1:
if not errors:
update_robot_key(login_method, robot_name, new_pwd1)
robot_keys = load_robot_keys()
if new_pwd1:
messages.append("""The password for robot %s has been successfully updated.""" % escape(robot_name))
else:
messages.append("""The password for robot %s has been erased, and hence the robot %s does not exist anymore.""" % (escape(robot_name), escape(robot_name)))
action = ''
confirm = 0
robot_name = ''
new_pwd1 = ''
new_pwd2 = ''
else:
if not new_pwd1:
warnings.append("""By setting an empty password you will actually erase the robot %s""" % escape(robot_name))
elif action == 'createurl':
if acce.acc_authorize_action(user_info, 'cfgrobotkeys', login_method=login_method, robot=robot_name)[0]:
errors.append("""You don't have proper authorization to create a URL for robot %s for login_method %s.""" % (escape(robot_name), escape(login_method)))
if login_method not in available_robot_login_methods:
errors.append("""The login method must be one among the available_robot_login_methods (%s).""" % escape(', '.join(available_robot_login_methods)))
if robot_name not in robot_keys.get(login_method, {}):
errors.append("""The robot name does not correspond to a valid robot name (for %s these are: %s).""" % (escape(login_method), escape(', '.join(robot_keys.get(login_method, {}).keys()))))
if json_assertion.strip():
try:
assertion = json.loads(json_assertion)
assert(isinstance(assertion, dict))
except Exception, err:
errors.append("""The assertion is not a valid json serializable mapping: %s""" % (err))
else:
assertion = None
if not email:
errors.append("""The email is mandatory.""")
if not ip:
errors.append("""The IP address is mandatory.""")
if not errors:
url = CFG_EXTERNAL_AUTHENTICATION[login_method].test_create_example_url(email, login_method=login_method, robot=robot_name, ip=ip, timeout=time.time() + timeout, referer=referer, groups=groups.splitlines(), nickname=nickname, assertion=assertion)
if url_only:
req.content_type = 'text/plain'
return url
messages.append("""The corresponding URL is: %(url)s""" % {
'url_escape': escape(url, True),
'url': escape(url)
})
action = ''
forms = """Existing login_method:
%s
""" % ''.join(["%s (robots: %s) " % (CFG_SITE_SECURE_URL, method, method, ', '.join(robot_keys.get(method, {}))) for method in available_robot_login_methods])
forms += """Existing robot names (for login_method %s):
%s
""" % (escape(login_method), ''.join(["%s " % name for name in robot_keys.get(login_method, {})]))
confirm_field = """"""
if action == 'changepwd':
confirm_field = """
Please confirm once more you want to change this password."""
login_method_boxes = ""
for login_method_name in available_robot_login_methods:
if login_method_name == login_method:
login_method_boxes += """
""" % {'name': escape(login_method_name, True)}
else:
login_method_boxes += """
""" % {'name': escape(login_method_name, True)}
forms += """
""" % {
'login_method_boxes': login_method_boxes,
'robot_name': escape(robot_name, True),
'timeout': escape(str(timeout), True),
'referer': escape(referer, True),
'ip': escape(ip, True),
'email': escape(email, True),
'nickname': escape(nickname, True),
'groups': escape(groups),
'json_assertion': escape(json_assertion)
}
out = ""
if errors:
out += "ERRORS
"
for error in errors:
out += '- %s
' % error
out += "
"
if warnings:
out += "WARNINGS
"
for warning in warnings:
out += '- %s
' % warning
out += "
"
if messages:
out += "INFORMATION
"
for message in messages:
out += '- %s
' % message
out += "
"
out += forms
return index(req=req,
title='Manage Robot Login',
subtitle='Are to manage robot-based authentiation',
body=out,
adminarea=2)
def perform_listgroups(req):
"""List all the existing groups."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
header = ['name']
groups = run_sql('select name from usergroup')
output = tupletotable(header, groups, highlight_rows_p=True,
alternate_row_colors_p=True)
extra = """
- Create new role
- go here to add a new role.
"""
return index(req=req,
title='Group list',
subtitle='All the groups registered in the system',
body=[output, extra],
adminarea=2)
def perform_rolearea(req, grep=""):
"""create the role area menu page."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
header = ['id', 'name', 'description', 'firewall like role definition',
'users', 'authorizations / actions', 'role', '']
roles = acca.acc_get_all_roles()
roles2 = []
if grep:
try:
re_grep = re.compile(grep)
except Exception, err:
re_grep = None
grep = ''
else:
re_grep = None
for (id, name, desc, dummy, firerole_def_src) in roles:
if not firerole_def_src:
firerole_def_src = '' ## Workaround for None.
if re_grep and not re_grep.search(name) and not re_grep.search(desc) and not re_grep.search(firerole_def_src):
## We're grepping for some word.
## Let's dig into the authorization then.
all_actions = acca.acc_find_possible_actions_all(id)
## FIXME: the acc_find_possible_actions_all is really an ugly
## function, but is the closest to what it's needed in order
## to retrieve all the authorization of a role.
for idx, row in enumerate(all_actions):
grepped = False
if idx % 2 == 0:
## even lines contains headers like in:
## ['role', 'action', '#', 'collection']
## the only useful text to grep is from index 3 onwards
for keyword in row[3:]:
if re_grep.search(keyword):
grepped = True
break
if grepped:
break
else:
## odd lines contains content like in:
## [1, 18L, 1, 'Theses']
## the useful text to grep is indirectly index 1
## which is indeed the id_action (needed to retrieve the
## action name) and from column 3 onwards.
if re_grep.search(acca.acc_get_action_name(row[1])):
break
for value in row[3:]:
if re_grep.search(value):
grepped = True
break
if grepped:
break
else:
## We haven't grepped anything!
## Let's skip to the next role then...
continue
if len(desc) > 30:
desc = desc[:30] + '...'
if firerole_def_src and len(firerole_def_src) > 30:
firerole_def_src = firerole_def_src[:30] + '...'
roles2.append([id, name, desc, firerole_def_src])
for col in [(('add', 'adduserrole'),
('delete', 'deleteuserrole'),),
(('add', 'addauthorization'),
('modify', 'modifyauthorizations'),
('remove', 'deleteroleaction')),
(('modify', 'modifyrole'),
('delete', 'deleterole')),
(('show details', 'showroledetails'), )]:
roles2[-1].append('%s' %
(col[0][1], id, col[0][0]))
for (str, function) in col[1:]:
roles2[-1][-1] += ' / %s' % \
(function, id, str)
output = """
- Users:
- add or remove users from the access to a role and its priviliges.
- Authorizations/Actions:
- these terms means almost the same, but an authorization is a
connection between a role and an action (possibly) containing arguments.
- Roles:
- see all the information attached to a role and decide if you want
to
delete it.
""" % escape(grep)
output += tupletotable(header=header, tuple=roles2, highlight_rows_p=True,
alternate_row_colors_p=True)
extra = """
- Create new role
- go here to add a new role.
"""
return index(req=req,
title='Role Administration',
subtitle='administration with roles as access point',
body=[output, extra],
adminarea=2)
def perform_actionarea(req, grep=''):
"""create the action area menu page."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
if grep:
try:
re_grep = re.compile(grep)
except Exception, err:
re_grep = None
grep = ''
else:
re_grep = None
header = ['name', 'authorizations/roles', '']
actions = acca.acc_get_all_actions()
actions2 = []
roles2 = []
for (id, name, description) in actions:
if re_grep and not re_grep.search(name) and not re_grep.search(description):
grepped = False
roles = acca.acc_get_action_roles(id)
for id_role, role_name, role_description in roles:
if re_grep.search(role_name) or re_grep.search(role_description):
grepped = True
break
elif re_grep.search(acca.acc_get_role_details(id_role)[3] or ''):
## Found in FireRole
grepped = True
break
else:
details = acca.acc_find_possible_actions(id_role, id)
if details:
for argument in details[0][1:]:
if re_grep.search(argument):
grepped = True
break
for values in details[1:]:
for value in values[1:]:
if re_grep.search(value):
grepped = True
break
if grepped:
break
if grepped:
break
if not grepped:
continue
actions2.append([name, description])
for col in [(('add', 'addauthorization'),
('modify', 'modifyauthorizations'),
('remove', 'deleteroleaction')),
(('show details', 'showactiondetails'), )]:
actions2[-1].append('%s'
'' % (col[0][1], id, col[0][0]))
for (str, function) in col[1:]:
actions2[-1][-1] += ' / %s' % (function, id, str)
output = """
- Authorizations/Roles:
- these terms means almost the same, but an authorization is a
connection between a role and an action (possibly) containing
arguments.
- Actions:
- see all the information attached to an action.
""" % escape(grep)
output += tupletotable(header=header, tuple=actions2, highlight_rows_p=True,
alternate_row_colors_p=True)
extra = """
- Create new role
- go here to add a new role.
"""
return index(req=req,
title='Action Administration',
subtitle='administration with actions as access point',
body=[output, extra],
adminarea=2)
def perform_userarea(req, email_user_pattern=''):
"""create area to show info about users. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = 'step 1 - search for users'
output = """
search for users to display.
"""
# remove letters not allowed in an email
email_user_pattern = cleanstring_email(email_user_pattern)
text = ' 1. search for user\n'
text += ' \n' % (email_user_pattern, )
output += createhiddenform(action="userarea",
text=text,
button="search for users")
if email_user_pattern:
try:
users1 = run_sql("""SELECT id, email FROM user WHERE email<>'' AND email RLIKE %s
ORDER BY email LIMIT %s""", (email_user_pattern, MAXPAGEUSERS+1))
except OperationalError:
users1 = ()
if not users1:
output += 'no matching users
'
else:
subtitle = 'step 2 - select what to do with user'
users = []
for (id, email) in users1[:MAXPAGEUSERS]:
users.append([id, email])
for col in [(('add', 'addroleuser'),
('remove', 'deleteuserrole')),
(('show details', 'showuserdetails'), )]:
users[-1].append('%s' % (col[0][1],
email_user_pattern, id, col[0][0]))
for (str, function) in col[1:]:
users[-1][-1] += ' / %s' % \
(function, email_user_pattern, id, str)
output += 'found %s matching users:
' % \
(len(users1), )
output += tupletotable(header=['id', 'email', 'roles', ''],
tuple=users, highlight_rows_p=True, alternate_row_colors_p=True)
if len(users1) > MAXPAGEUSERS:
output += 'only showing the first %s users, ' \
'narrow your search...
' % (MAXPAGEUSERS, )
return index(req=req,
title='User Administration',
subtitle=subtitle,
body=[output],
adminarea=2)
def perform_resetarea(req):
"""create the reset area menu page."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
output = """
- Reset to Default Authorizations
- remove all changes that has been done to the roles and
add only the default authorization settings.
- Add Default Authorizations
- keep all changes and add the default authorization settings.
"""
return index(req=req,
title='Reset Authorizations',
subtitle='reseting to or adding default authorizations',
body=[output],
adminarea=2)
def perform_resetdefaultsettings(req, superusers=[], confirm=0):
"""delete all roles, actions and authorizations presently in the database
and add only the default roles.
only selected users will be added to superadmin, rest is blank """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
# cleaning input
if type(superusers) == str: superusers = [superusers]
# remove not valid e-mails
for email in superusers:
if not check_email(email): superusers.remove(email)
# instructions
output = """
before you reset the settings, we need some users
to connect to %s.
enter as many e-mail addresses you want and press reset.
confirm reset settings when you have added enough e-mails.
%s is added as default.
""" % (SUPERADMINROLE, CFG_SITE_ADMIN_EMAIL)
# add more superusers
output += """
enter user e-mail addresses:
"""
if superusers:
# remove emails
output += """
"""
# superusers confirm table
start = ''
output += 'reset default settings with the users below?
'
output += tupletotable(header=['e-mail address'],
tuple=superusers,
start=start,
extracolumn=extra,
end=end,
highlight_rows_p=True,
alternate_row_colors_p=True)
if confirm in [1, "1"]:
res = acca.acc_reset_default_settings(superusers)
if res:
output += 'successfully reset default settings
'
else:
output += 'sorry, could not reset default settings
'
return index(req=req,
title='Reset Default Settings',
subtitle='reset settings',
body=[output],
adminarea=6)
def perform_adddefaultsettings(req, superusers=[], confirm=0):
"""add the default settings, and keep everything else.
probably nothing will be deleted, except if there has been made changes to the defaults."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
# cleaning input
if type(superusers) == str: superusers = [superusers]
# remove not valid e-mails
for email in superusers:
if not check_email(email): superusers.remove(email)
# instructions
output = """
before you add the settings, we need some users
to connect to %s.
enter as many e-mail addresses you want and press add.
confirm add settings when you have added enough e-mails.
%s is added as default.
""" % (SUPERADMINROLE, CFG_SITE_ADMIN_EMAIL)
# add more superusers
output += """
enter user e-mail addresses:
"""
if superusers:
# remove emails
output += """
"""
# superusers confirm table
start = ''
output += 'add default settings with the users below?
'
output += tupletotable(header=['e-mail address'],
tuple=superusers,
start=start,
extracolumn=extra,
end=end)
if confirm in [1, "1"]:
res = acca.acc_add_default_settings(superusers)
if res:
output += 'successfully added default settings
'
else:
output += 'sorry, could not add default settings
'
return index(req=req,
title='Add Default Settings',
subtitle='add settings',
body=[output],
adminarea=6)
def perform_manageaccounts(req, mtype='', content='', confirm=0):
"""start area for managing accounts."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = 'Overview'
fin_output = ''
fin_output += """
Menu
0. Show all
1. Access policy
2. Account overview
3. Create account
4. Edit accounts
""" % (CFG_SITE_SECURE_URL, CFG_SITE_SECURE_URL, CFG_SITE_SECURE_URL, CFG_SITE_SECURE_URL, CFG_SITE_SECURE_URL)
if mtype == "perform_accesspolicy" and content:
fin_output += content
elif mtype == "perform_accesspolicy" or mtype == "perform_showall":
fin_output += perform_accesspolicy(req, callback='')
fin_output += "
"
if mtype == "perform_accountoverview" and content:
fin_output += content
elif mtype == "perform_accountoverview" or mtype == "perform_showall":
fin_output += perform_accountoverview(req, callback='')
fin_output += "
"
if mtype == "perform_createaccount" and content:
fin_output += content
elif mtype == "perform_createaccount" or mtype == "perform_showall":
fin_output += perform_createaccount(req, callback='')
fin_output += "
"
if mtype == "perform_modifyaccounts" and content:
fin_output += content
elif mtype == "perform_modifyaccounts" or mtype == "perform_showall":
fin_output += perform_modifyaccounts(req, callback='')
fin_output += "
"
if mtype == "perform_becomeuser" and content:
fin_output += content
elif mtype == "perform_becomeuser" or mtype == "perform_showall":
fin_output += perform_becomeuser(req, callback='')
fin_output += "
"
return index(req=req,
title='Manage Accounts',
subtitle=subtitle,
body=[fin_output],
adminarea=0,
authorized=1)
def perform_accesspolicy(req, callback='yes', confirm=0):
"""Modify default behaviour of a guest user or if new accounts should automatically/manually be modified."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """1. Access policy. [?]""" % CFG_SITE_SECURE_URL
account_policy = {}
account_policy[0] = "Users can register new accounts. New accounts automatically activated."
account_policy[1] = "Users can register new accounts. Admin users must activate the accounts."
account_policy[2] = "Only admin can register new accounts. User cannot edit email address."
account_policy[3] = "Only admin can register new accounts. User cannot edit email address or password."
account_policy[4] = "Only admin can register new accounts. User cannot edit email address, password or login method."
account_policy[5] = "Only admin can register new accounts. User cannot edit email address, password or login method and information about how to get an account is hidden from the login page."
site_policy = {}
site_policy[0] = "Normal operation of the site."
site_policy[1] = "Read-only site, all write operations temporarily closed."
site_policy[2] = "Site fully closed."
site_policy[3] = "Site fully closed. Database connection disabled."
output = "(Modifications must be done in access_control_config.py)
"
output += "
Current settings:
"
output += "Site status: %s
" % (site_policy[CFG_ACCESS_CONTROL_LEVEL_SITE])
output += "Guest accounts allowed: %s
" % (CFG_ACCESS_CONTROL_LEVEL_GUESTS == 0 and "Yes" or "No")
output += "Account policy: %s
" % (account_policy[CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS])
output += "Allowed email addresses limited: %s
" % (CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN and CFG_ACCESS_CONTROL_LIMIT_REGISTRATION_TO_DOMAIN or "Not limited")
output += "Send email to admin when new account: %s
" % (CFG_ACCESS_CONTROL_NOTIFY_ADMIN_ABOUT_NEW_ACCOUNTS == 1 and "Yes" or "No")
output += "Send email to user after creating new account: %s
" % (CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT == 1 and "Yes" or "No")
output += "Send email to user when account is activated: %s
" % (CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION == 1 and "Yes" or "No")
output += "Send email to user when account is deleted/rejected: %s
" % (CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION == 1 and "Yes" or "No")
output += "
"
output += "Available 'login via' methods:
"
methods = CFG_EXTERNAL_AUTHENTICATION.keys()
methods.sort()
for system in methods:
output += """%s %s
""" % (system, (CFG_EXTERNAL_AUTH_DEFAULT == system and "(Default)" or ""))
output += "
Changing the settings:
"
output += "Currently, all changes must be done using your favourite editor, and the webserver restarted for changes to take effect. For the settings to change, either look in the guide or in access_control_config.py ."
body = [output]
if callback:
return perform_manageaccounts(req, "perform_accesspolicy", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_accountoverview(req, callback='yes', confirm=0):
"""Modify default behaviour of a guest user or if new accounts should automatically/manually be modified."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """2. Account overview. [?]""" % CFG_SITE_SECURE_URL
output = ""
res = run_sql("SELECT COUNT(*) FROM user WHERE email=''")
output += "Guest accounts: %s
" % res[0][0]
res = run_sql("SELECT COUNT(*) FROM user WHERE email!=''")
output += "Registered accounts: %s
" % res[0][0]
res = run_sql("SELECT COUNT(*) FROM user WHERE email!='' AND note='0' OR note IS NULL")
output += "Inactive accounts: %s " % res[0][0]
if res[0][0] > 0:
output += ' [Activate/Reject accounts]'
res = run_sql("SELECT COUNT(*) FROM user")
output += "
Total nr of accounts: %s
" % res[0][0]
body = [output]
if callback:
return perform_manageaccounts(req, "perform_accountoverview", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_createaccount(req, email='', password='', callback='yes', confirm=0):
"""Modify default behaviour of a guest user or if new accounts should automatically/manually be modified."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """3. Create account. [?]""" % CFG_SITE_SECURE_URL
output = ""
text = ' Email:\n'
text += '
' % (email, )
text += ' Password:\n'
text += '
' % (password, )
output += createhiddenform(action="createaccount",
text=text,
confirm=1,
button="Create")
if confirm in [1, "1"] and email and email_valid_p(email):
res = run_sql("SELECT email FROM user WHERE email=%s", (email,))
if not res:
res = run_sql("INSERT INTO user (email,password, note) values(%s,AES_ENCRYPT(email,%s), '1')", (email, password))
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT == 1:
emailsent = send_new_user_account_warning(email, email, password) == 0
if password:
output += 'Account created with password and activated.'
else:
output += 'Account created without password and activated.'
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_NEW_ACCOUNT == 1:
if emailsent:
output += '
An email has been sent to the owner of the account.'
else:
output += '
Could not send an email to the owner of the account.'
else:
output += 'An account with the same email already exists.'
elif confirm in [1, "1"]:
output += 'Please specify an valid email-address.'
body = [output]
if callback:
return perform_manageaccounts(req, "perform_createaccount", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_modifyaccountstatus(req, userID, email_user_pattern, limit_to, maxpage, page, callback='yes', confirm=0):
"""set a disabled account to enabled and opposite"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
res = run_sql("SELECT id, email, note FROM user WHERE id=%s", (userID, ))
subtitle = ""
output = ""
if res:
if res[0][2] in [0, "0", None]:
res2 = run_sql("UPDATE user SET note=1 WHERE id=%s", (userID, ))
output += """The account '%s' has been activated.""" % res[0][1]
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_ACTIVATION == 1:
emailsent = send_account_activated_message(res[0][1], res[0][1], '*****')
if emailsent:
output += """
An email has been sent to the owner of the account."""
else:
output += """
Could not send an email to the owner of the account."""
elif res[0][2] in [1, "1"]:
res2 = run_sql("UPDATE user SET note=0 WHERE id=%s", (userID, ))
output += """The account '%s' has been set inactive.""" % res[0][1]
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_modifyaccounts(req, email_user_pattern, limit_to, maxpage, page, content=output, callback='yes')
else:
return addadminbox(subtitle, body)
def perform_editaccount(req, userID, mtype='', content='', callback='yes', confirm=-1):
"""form to modify an account. this method is calling other methods which again is calling this and sending back the output of the method.
if callback, the method will call perform_editcollection, if not, it will just return its output.
userID - id of the user
mtype - the method that called this method.
content - the output from that method."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
res = run_sql("SELECT id, email FROM user WHERE id=%s", (userID, ))
if not res:
if mtype == "perform_deleteaccount":
text = """The selected account has been deleted, to continue editing, go back to 'Manage Accounts'."""
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION == 1:
text += """
An email has been sent to the owner of the account."""
else:
text = """The selected accounts does not exist, please go back and select an account to edit."""
return index(req=req,
title='Edit Account',
subtitle="Edit account",
body=[text],
adminarea=7,
authorized=1)
fin_output = """
Menu
0. Show all
1. Modify login-data
2. Modify preferences
3. Delete account
+ 4. Edit REST API Key
- """ % (CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID)
+ """ % (CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID, CFG_SITE_SECURE_URL, userID)
if mtype == "perform_modifylogindata" and content:
fin_output += content
elif mtype == "perform_modifylogindata" or not mtype:
fin_output += perform_modifylogindata(req, userID, callback='')
if mtype == "perform_modifypreferences" and content:
fin_output += content
elif mtype == "perform_modifypreferences" or not mtype:
fin_output += perform_modifypreferences(req, userID, callback='')
if mtype == "perform_deleteaccount" and content:
fin_output += content
elif mtype == "perform_deleteaccount" or not mtype:
fin_output += perform_deleteaccount(req, userID, callback='')
+ if mtype == "perform_modifyapikeydata" and content:
+ fin_output += content
+ elif mtype == "perform_modifyapikeydata" or not mtype:
+ fin_output += perform_modifyapikeydata(req, userID, callback='')
+
return index(req=req,
title='Edit Account',
subtitle="Edit account '%s'" % res[0][1],
body=[fin_output],
adminarea=7,
authorized=1)
def perform_becomeuser(req, userID='', callback='yes', confirm=0):
"""modify email and password of an account"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """5. Became user. [?]""" % CFG_SITE_SECURE_URL
res = run_sql("SELECT email FROM user WHERE id=%s", (userID, ))
output = ""
if res:
update_Uid(req, res[0][0])
redirect_to_url(req, CFG_SITE_SECURE_URL)
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_editaccount(req, userID, mtype='perform_becomeuser', content=addadminbox(subtitle, body), callback='yes')
else:
return addadminbox(subtitle, body)
def perform_modifylogindata(req, userID, nickname='', email='', password='', callback='yes', confirm=0):
"""modify email and password of an account"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """1. Edit login-data. [?]""" % CFG_SITE_SECURE_URL
res = run_sql("SELECT id, email, nickname FROM user WHERE id=%s", (userID, ))
output = ""
if res:
if not email and not password:
email = res[0][1]
nickname = res[0][2]
text = ' Account id:%s
\n' % userID
text = ' Nickname:\n'
text += '
' % (nickname, )
text += ' Email:\n'
text += '
' % (email, )
text += ' Password:\n'
text += '
' % (password, )
output += createhiddenform(action="modifylogindata",
text=text,
userID=userID,
confirm=1,
button="Modify")
if confirm in [1, "1"] and email and email_valid_p(email):
res = run_sql("SELECT nickname FROM user WHERE nickname=%s AND id<>%s", (nickname, userID))
if res:
output += 'Sorry, the specified nickname is already used.'
else:
res = run_sql("UPDATE user SET email=%s WHERE id=%s", (email, userID))
if password:
res = run_sql("UPDATE user SET password=AES_ENCRYPT(email,%s) WHERE id=%s", (password, userID))
else:
output += 'Password not modified. '
res = run_sql("UPDATE user SET nickname=%s WHERE id=%s", (nickname, userID))
output += 'Nickname/email and/or password modified.'
elif confirm in [1, "1"]:
output += 'Please specify an valid email-address.'
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_editaccount(req, userID, mtype='perform_modifylogindata', content=addadminbox(subtitle, body), callback='yes')
else:
return addadminbox(subtitle, body)
def perform_modifypreferences(req, userID, login_method='', callback='yes', confirm=0):
"""modify email and password of an account"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """2. Modify preferences. [?]""" % CFG_SITE_SECURE_URL
res = run_sql("SELECT id, email FROM user WHERE id=%s", (userID, ))
output = ""
if res:
user_pref = get_user_preferences(userID)
if confirm in [1, "1"]:
if login_method:
user_pref['login_method'] = login_method
set_user_preferences(userID, user_pref)
output += "Select default login method:
"
text = ""
methods = CFG_EXTERNAL_AUTHENTICATION.keys()
methods.sort()
for system in methods:
text += """%s
""" % (system, (user_pref['login_method'] == system and "checked" or ""), system)
output += createhiddenform(action="modifypreferences",
text=text,
confirm=1,
userID=userID,
button="Select")
if confirm in [1, "1"]:
if login_method:
output += """The login method has been changed"""
else:
output += """Nothing to update"""
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_editaccount(req, userID, mtype='perform_modifypreferences', content=addadminbox(subtitle, body), callback='yes')
else:
return addadminbox(subtitle, body)
def perform_deleteaccount(req, userID, callback='yes', confirm=0):
"""delete account"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """3. Delete account. [?]""" % CFG_SITE_SECURE_URL
res = run_sql("SELECT id, email FROM user WHERE id=%s", (userID, ))
output = ""
if res:
if confirm in [0, "0"]:
text = 'Are you sure you want to delete the account with email: "%s"?' % res[0][1]
output += createhiddenform(action="deleteaccount",
text=text,
userID=userID,
confirm=1,
button="Delete")
elif confirm in [1, "1"]:
res2 = run_sql("DELETE FROM user WHERE id=%s", (userID, ))
output += 'Account deleted.'
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION == 1:
emailsent = send_account_deleted_message(res[0][1], res[0][1])
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_editaccount(req, userID, mtype='perform_deleteaccount', content=addadminbox(subtitle, body), callback='yes')
else:
return addadminbox(subtitle, body)
+def perform_modifyapikeydata(req, userID, keyID='', status='' , callback='yes', confirm=0):
+ """modify REST API keys of an account"""
+
+ (auth_code, auth_message) = is_adminuser(req)
+ if auth_code != 0: return mustloginpage(req, auth_message)
+
+ subtitle = """4. Edit REST API Keys. [?]""" % CFG_SITE_SECURE_URL
+
+ if confirm in [1, "1"]:
+ run_sql("UPDATE apikey SET status=%s WHERE id=%s", (status, keyID))
+
+ res = run_sql("SELECT id, description, status FROM apikey WHERE id_user=%s", (userID, ))
+ output = ""
+ if res:
+ for key_info in res:
+ text = ''
+ text += ' Key: %s
\n' % key_info[0]
+ text += ' ' % key_info[0]
+ text += ' Description: %s
\n' % key_info[1]
+ text += '
\n'
+ if key_info[0] == keyID:
+ text += 'Key status modified'
+ output += createhiddenform(action="modifyapikeydata",
+ text=text,
+ userID=userID,
+ confirm=1,
+ button="Modify")
+ else:
+ output += 'The account id given does not have REST API Keys.'
+
+ body = [output]
+
+ if callback:
+ return perform_editaccount(req, userID, mtype='perform_modifyapikeydata', content=addadminbox(subtitle, body), callback='yes')
+ else:
+ return addadminbox(subtitle, body)
+
def perform_rejectaccount(req, userID, email_user_pattern, limit_to, maxpage, page, callback='yes', confirm=0):
"""Delete account and send an email to the owner."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
res = run_sql("SELECT id, email, note FROM user WHERE id=%s", (userID, ))
output = ""
subtitle = ""
if res:
res2 = run_sql("DELETE FROM user WHERE id=%s", (userID, ))
output += 'Account rejected and deleted.'
if CFG_ACCESS_CONTROL_NOTIFY_USER_ABOUT_DELETION == 1:
if not res[0][2] or res[0][2] == "0":
emailsent = send_account_rejected_message(res[0][1], res[0][1])
elif res[0][2] == "1":
emailsent = send_account_deleted_message(res[0][1], res[0][1])
if emailsent:
output += """
An email has been sent to the owner of the account."""
else:
output += """
Could not send an email to the owner of the account."""
else:
output += 'The account id given does not exist.'
body = [output]
if callback:
return perform_modifyaccounts(req, email_user_pattern, limit_to, maxpage, page, content=output, callback='yes')
else:
return addadminbox(subtitle, body)
def perform_modifyaccounts(req, email_user_pattern='', limit_to=-1, maxpage=MAXPAGEUSERS, page=1, content='', callback='yes', confirm=0):
"""Modify default behaviour of a guest user or if new accounts should automatically/manually be modified."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
subtitle = """4. Edit accounts. [?]""" % CFG_SITE_SECURE_URL
output = ""
# remove letters not allowed in an email
email_user_pattern = cleanstring_email(email_user_pattern)
try:
maxpage = int(maxpage)
except:
maxpage = MAXPAGEUSERS
try:
page = int(page)
if page < 1:
page = 1
except:
page = 1
text = ' Email (part of):\n'
text += '
' % (email_user_pattern, )
text += """Limit to:
""" % ((limit_to=="all" and "selected" or ""), (limit_to=="enabled" and "selected" or ""), (limit_to=="disabled" and "selected" or ""))
text += """Accounts per page:
""" % ((maxpage==25 and "selected" or ""), (maxpage==50 and "selected" or ""), (maxpage==100 and "selected" or ""), (maxpage==250 and "selected" or ""), (maxpage==500 and "selected" or ""), (maxpage==1000 and "selected" or ""))
output += createhiddenform(action="modifyaccounts",
text=text,
button="search for accounts")
if limit_to not in [-1, "-1"] and maxpage:
options = []
users1 = "SELECT id,email,note FROM user WHERE "
if limit_to == "enabled":
users1 += " email!='' AND note=1"
elif limit_to == "disabled":
users1 += " email!='' AND note=0 OR note IS NULL"
elif limit_to == "guest":
users1 += " email=''"
else:
users1 += " email!=''"
if email_user_pattern:
users1 += " AND email RLIKE %s"
options += [email_user_pattern]
users1 += " ORDER BY email LIMIT %s"
options += [maxpage * page + 1]
try:
users1 = run_sql(users1, tuple(options))
except OperationalError:
users1 = ()
if not users1:
output += 'There are no accounts matching the email given.'
else:
users = []
if maxpage * (page - 1) > len(users1):
page = len(users1) / maxpage + 1
for (id, email, note) in users1[maxpage * (page - 1):(maxpage * page)]:
users.append(['', id, email, (note=="1" and 'Active' or 'Inactive')])
for col in [(((note=="1" and 'Inactivate' or 'Activate'), 'modifyaccountstatus'), ((note == "0" and 'Reject' or 'Delete'), 'rejectaccount'), ),
(('Edit account', 'editaccount'), ),]:
users[-1].append('%s' % (col[0][1], id, email_user_pattern, limit_to, maxpage, page, random.randint(0, 1000), col[0][0]))
for (str, function) in col[1:]:
users[-1][-1] += ' / %s' % (function, id, email_user_pattern, limit_to, maxpage, page, random.randint(0, 1000), str)
users[-1].append('%s' % ('becomeuser', id, email_user_pattern, limit_to, maxpage, page, random.randint(0, 1000), 'Become user'))
last = ""
next = ""
if len(users1) > maxpage:
if page > 1:
last += 'Last Page' % (email_user_pattern, limit_to, maxpage, (page - 1))
if len(users1[maxpage * (page - 1):(maxpage * page)]) == maxpage:
next += 'Next page' % (email_user_pattern, limit_to, maxpage, (page + 1))
output += 'Showing accounts %s-%s:' % (1 + maxpage * (page - 1), maxpage * page)
else:
output += '%s matching account(s):' % len(users1)
output += tupletotable(header=[last, 'id', 'email', 'Status', '', '', next], tuple=users)
else:
output += 'Please select which accounts to find and how many to show per page.'
if content:
output += "
%s" % content
body = [output]
if callback:
return perform_manageaccounts(req, "perform_modifyaccounts", addadminbox(subtitle, body))
else:
return addadminbox(subtitle, body)
def perform_delegate_startarea(req):
"""start area for lower level delegation of rights."""
# refuse access to guest users:
uid = getUid(req)
if isGuestUser(uid):
return index(req=req,
title='Delegate Rights',
adminarea=0,
authorized=0)
subtitle = 'select what to do'
output = ''
if is_adminuser(req)[0] == 0:
output += """
You are also allowed to be in the Main Admin Area which gives you
the access to the full functionality of WebAccess.
"""
output += """
- Connect users to roles
- add users to the roles you have delegation rights to.
- Remove users from roles
- remove users from the roles you have delegation rights to.
- Set up delegation rights
- specialized area to set up the delegation rights used in the areas above.
you need to be a web administrator to access the area.
"""
return index(req=req,
title='Delegate Rights',
subtitle=subtitle,
body=[output],
adminarea=0,
authorized=1)
def perform_delegate_adminsetup(req, id_role_admin=0, id_role_delegate=0, confirm=0):
"""lets the webadmins set up the delegation rights for the other roles
id_role_admin - the role to be given delegation rights
id_role_delegate - the role over which the delegation rights are given
confirm - make the connection happen """
subtitle = 'step 1 - select admin role'
admin_roles = acca.acc_get_all_roles()
output = """
This is a specialized area to handle a task that also can be handled
from the "add authorization" interface.
By handling the delegation rights here you get the advantage of
not having to select the correct action (%s) or
remembering the names of available roles.
""" % (DELEGATEADDUSERROLE, )
output += createroleselect(id_role=id_role_admin,
step=1,
button='select admin role',
name='id_role_admin',
action='delegate_adminsetup',
roles=admin_roles)
if str(id_role_admin) != '0':
subtitle = 'step 2 - select delegate role'
name_role_admin = acca.acc_get_role_name(id_role=id_role_admin)
delegate_roles_old = acca.acc_find_delegated_roles(id_role_admin=id_role_admin)
delegate_roles = []
delegate_roles_old_names = []
for role in admin_roles:
if (role,) not in delegate_roles_old:
delegate_roles.append(role)
else:
delegate_roles_old_names.append(role[1])
if delegate_roles_old_names:
delegate_roles_old_names.sort()
names_str = ''
for name in delegate_roles_old_names:
if names_str: names_str += ', '
names_str += name
output += 'previously selected roles: %s.
' % (names_str, )
extra = """
- Remove delegated roles
- use the standard administration area to remove delegation rights
you no longer want to be available.
""" % (id_role_admin, acca.acc_get_action_id(name_action=DELEGATEADDUSERROLE))
else:
output += 'no previously selected roles.
'
output += createroleselect(id_role=id_role_delegate,
step=2,
button='select delegate role',
name='id_role_delegate',
action='delegate_adminsetup',
roles=delegate_roles,
id_role_admin=id_role_admin)
if str(id_role_delegate) != '0':
subtitle = 'step 3 - confirm to add delegation right'
name_role_delegate = acca.acc_get_role_name(id_role=id_role_delegate)
output += """
Warning: don't hand out delegation rights that can harm the system (e.g. delegating superrole).
"""
output += createhiddenform(action="delegate_adminsetup",
text='let role %s delegate rights over role %s?' % (name_role_admin, name_role_delegate),
id_role_admin=id_role_admin,
id_role_delegate=id_role_delegate,
confirm=1)
if int(confirm):
subtitle = 'step 4 - confirm delegation right added'
# res1 = acca.acc_add_role_action_arguments_names(name_role=name_role_admin,
# name_action=DELEGATEADDUSERROLE,
# arglistid=-1,
# optional=0,
# role=name_role_delegate)
res1 = acca.acc_add_authorization(name_role=name_role_admin,
name_action=DELEGATEADDUSERROLE,
optional=0,
role=name_role_delegate)
if res1:
output += 'confirm: role %s delegates role %s.' % (name_role_admin, name_role_delegate)
else: output += '
sorry, delegation right could not be added,
it probably already exists.
'
# see if right hand menu is available
try: body = [output, extra]
except NameError: body = [output]
return index(req=req,
title='Delegate Rights',
subtitle=subtitle,
body=body,
adminarea=1)
def perform_delegate_adduserrole(req, id_role=0, email_user_pattern='', id_user=0, confirm=0):
"""let a lower level web admin add users to a limited set of roles.
id_role - the role to connect to a user
id_user - the user to connect to a role
confirm - make the connection happen """
# finding the allowed roles for this user
id_admin = getUid(req)
id_action = acca.acc_get_action_id(name_action=DELEGATEADDUSERROLE)
actions = acca.acc_find_possible_actions_user(id_user=id_admin, id_action=id_action)
allowed_roles = []
allowed_id_roles = []
for (id, arglistid, name_role_help) in actions[1:]:
id_role_help = acca.acc_get_role_id(name_role=name_role_help)
if id_role_help and [id_role_help, name_role_help, ''] not in allowed_roles:
allowed_roles.append([id_role_help, name_role_help, ''])
allowed_id_roles.append(str(id_role_help))
output = ''
if not allowed_roles:
subtitle = 'no delegation rights'
output += """
You do not have the delegation rights over any roles.
If you think you should have such rights, contact a WebAccess Administrator.
"""
extra = ''
else:
subtitle = 'step 1 - select role'
output += """
Lower level delegation of access rights to roles.
An administrator with all rights have to give you these rights.
"""
email_out = acca.acc_get_user_email(id_user=id_user)
name_role = acca.acc_get_role_name(id_role=id_role)
output += createroleselect(id_role=id_role, step=1, name='id_role',
action='delegate_adduserrole', roles=allowed_roles)
if str(id_role) != '0' and str(id_role) in allowed_id_roles:
subtitle = 'step 2 - search for users'
# remove letters not allowed in an email
email_user_pattern = cleanstring_email(email_user_pattern)
text = ' 2. search for user \n'
text += ' \n' % (email_user_pattern, )
output += createhiddenform(action="delegate_adduserrole",
text=text,
button="search for users",
id_role=id_role)
# pattern is entered
if email_user_pattern:
# users with matching email-address
try:
users1 = run_sql("""SELECT id, email FROM user WHERE email<>'' AND email RLIKE %s ORDER BY email """, (email_user_pattern, ))
except OperationalError:
users1 = ()
# users that are connected
try:
users2 = run_sql("""SELECT DISTINCT u.id, u.email
FROM user u LEFT JOIN user_accROLE ur ON u.id = ur.id_user
WHERE ur.id_accROLE = %s AND u.email RLIKE %s
ORDER BY u.email """, (id_role, email_user_pattern))
except OperationalError:
users2 = ()
# no users that match the pattern
if not (users1 or users2):
output += 'no qualified users, try new search.
'
# too many matching users
elif len(users1) > MAXSELECTUSERS:
output += '%s hits, too many qualified users, specify more narrow search. (limit %s)
' % (len(users1), MAXSELECTUSERS)
# show matching users
else:
subtitle = 'step 3 - select a user'
users = []
extrausers = []
for (id, email) in users1:
if (id, email) not in users2: users.append([id,email,''])
for (id, email) in users2:
extrausers.append([-id, email,''])
output += createuserselect(id_user=id_user,
action="delegate_adduserrole",
step=3,
users=users,
extrausers=extrausers,
button="add this user",
id_role=id_role,
email_user_pattern=email_user_pattern)
try: id_user = int(id_user)
except ValueError: pass
# user selected already connected to role
if id_user < 0:
output += 'users in brackets are already attached to the role, try another one...
'
# a user is selected
elif email_out:
subtitle = "step 4 - confirm to add user"
output += createhiddenform(action="delegate_adduserrole",
text='add user %s to role %s?' % (email_out, name_role),
id_role=id_role,
email_user_pattern=email_user_pattern,
id_user=id_user,
confirm=1)
# it is confirmed that this user should be added
if confirm:
# add user
result = acca.acc_add_user_role(id_user=id_user, id_role=id_role)
if result and result[2]:
subtitle = 'step 5 - confirm user added'
output += 'confirm: user %s added to role %s.
' % (email_out, name_role)
else:
subtitle = 'step 5 - user could not be added'
output += 'sorry, but user could not be added.
'
extra = """
- Remove users from role
- remove users from the roles you have delegating rights to.
""" % (id_role, )
return index(req=req,
title='Connect users to roles',
subtitle=subtitle,
body=[output, extra],
adminarea=1,
authorized=1)
def perform_delegate_deleteuserrole(req, id_role=0, id_user=0, confirm=0):
"""let a lower level web admin remove users from a limited set of roles.
id_role - the role to connect to a user
id_user - the user to connect to a role
confirm - make the connection happen """
subtitle = 'in progress...'
output = 'in progress...
'
# finding the allowed roles for this user
id_admin = getUid(req)
id_action = acca.acc_get_action_id(name_action=DELEGATEADDUSERROLE)
actions = acca.acc_find_possible_actions_user(id_user=id_admin, id_action=id_action)
output = ''
if not actions:
subtitle = 'no delegation rights'
output += """
You do not have the delegation rights over any roles.
If you think you should have such rights, contact a WebAccess Administrator.
"""
extra = ''
else:
subtitle = 'step 1 - select role'
output += """
Lower level delegation of access rights to roles.
An administrator with all rights have to give you these rights.
"""
email_out = acca.acc_get_user_email(id_user=id_user)
name_role = acca.acc_get_role_name(id_role=id_role)
# create list of allowed roles
allowed_roles = []
allowed_id_roles = []
for (id, arglistid, name_role_help) in actions[1:]:
id_role_help = acca.acc_get_role_id(name_role=name_role_help)
if id_role_help and [id_role_help, name_role_help, ''] not in allowed_roles:
allowed_roles.append([id_role_help, name_role_help, ''])
allowed_id_roles.append(str(id_role_help))
output += createroleselect(id_role=id_role, step=1,
action='delegate_deleteuserrole', roles=allowed_roles)
if str(id_role) != '0' and str(id_role) in allowed_id_roles:
subtitle = 'step 2 - select user'
users = acca.acc_get_role_users(id_role)
output += createuserselect(id_user=id_user,
step=2,
action='delegate_deleteuserrole',
users=users,
id_role=id_role)
if str(id_user) != '0':
subtitle = 'step 3 - confirm delete of user'
email_user = acca.acc_get_user_email(id_user=id_user)
output += createhiddenform(action="delegate_deleteuserrole",
text='delete user %s from %s?'
% (headerstrong(user=id_user), headerstrong(role=id_role)),
id_role=id_role,
id_user=id_user,
confirm=1)
if confirm:
res = acca.acc_delete_user_role(id_user=id_user, id_role=id_role)
if res:
subtitle = 'step 4 - confirm user deleted from role'
output += 'confirm: deleted user %s from role %s.
' % (email_user, name_role)
else:
subtitle = 'step 4 - user could not be deleted'
output += 'sorry, but user could not be deleted
user is probably already deleted.'
extra = """
- Connect users to role
- add users to the roles you have delegating rights to.
""" % (id_role, )
return index(req=req,
title='Remove users from roles',
subtitle=subtitle,
body=[output, extra],
adminarea=1,
authorized=1)
def perform_showactiondetails(req, id_action):
"""show the details of an action. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
output = createactionselect(id_action=id_action,
action="showactiondetails",
step=1,
actions=acca.acc_get_all_actions(),
button="select action")
if id_action not in [0, '0']:
output += actiondetails(id_action=id_action)
extra = """
- Add new authorization
- add an authorization.
- Modify authorizations
- modify existing authorizations.
- Remove role
- remove all authorizations from action and a role.
""" % (id_action, id_action, id_action)
body = [output, extra]
else:
output += 'no details to show
'
body = [output]
return index(req=req,
title='Show Action Details',
subtitle='show action details',
body=body,
adminarea=4)
def actiondetails(id_action=0):
"""show details of given action. """
output = ''
if id_action not in [0, '0']:
name_action = acca.acc_get_action_name(id_action=id_action)
output += 'action details:
'
output += tupletotable(header=['id', 'name', 'description', 'allowedkeywords', 'optional'],
tuple=[acca.acc_get_action_details(id_action=id_action)])
roleshlp = acca.acc_get_action_roles(id_action=id_action)
if roleshlp:
roles = []
for (id, name, dummy) in roleshlp:
res = acca.acc_find_possible_actions(id, id_action)
if res:
authorization_details = tupletotable(header=res[0], tuple=res[1:])
else:
authorization_details = 'no details to show'
roles.append([id, name,
authorization_details,
'show connected users' % (id, )])
roletable = tupletotable(header=['id', 'name', 'authorization details', ''], tuple=roles)
output += 'roles connected to %s:
\n' % (headerstrong(action=name_action, query=0), )
output += roletable
else:
output += 'no roles connected to %s.
\n' % (headerstrong(action=name_action, query=0), )
else:
output += 'no details to show
'
return output
def perform_addrole(req, id_role=0, name_role='', description='put description here.', firerole_def_src=CFG_ACC_EMPTY_ROLE_DEFINITION_SRC, confirm=0):
"""form to add a new role with these values:
name_role - name of the new role
description - optional description of the role """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
name_role = cleanstring(name_role)
title='Add Role'
subtitle = 'step 1 - give values to the requested fields'
output = """
""" % (escape(name_role, '"'), escape(description), escape(firerole_def_src))
if name_role:
# description must be changed before submitting
subtitle = 'step 2 - confirm to add role'
internaldesc = ''
if description != 'put description here.':
internaldesc = description
try:
firerole_def_ser = serialize(compile_role_definition(firerole_def_src))
except InvenioWebAccessFireroleError, msg:
output += "%s" % msg
else:
text = """
add role with:
\n
name: %s
""" % (name_role, )
if internaldesc:
text += 'description: %s?\n' % (description, )
output += createhiddenform(action="addrole",
text=text,
name_role=escape(name_role, '"'),
description=escape(description, '"'),
firerole_def_src=escape(firerole_def_src, '"'),
confirm=1)
if confirm not in ["0", 0]:
result = acca.acc_add_role(name_role=name_role,
description=internaldesc,
firerole_def_ser=firerole_def_ser,
firerole_def_src=firerole_def_src)
if result:
subtitle = 'step 3 - role added'
output += 'role added:
'
result = list(result)
result[3] = result[3].replace('\n', '
')
result = tuple(result)
output += tupletotable(header=['id', 'role name', 'description', 'firewall like role definition'],
tuple=[result])
else:
subtitle = 'step 3 - role could not be added'
output += 'sorry, could not add role,
role with the same name probably exists.
'
id_role = acca.acc_get_role_id(name_role=name_role)
extra = """
- Add authorization
- start adding new authorizations to role %s.
Connect user
connect a user to role %s.
""" % (id_role, name_role, id_role, name_role)
try: body = [output, extra]
except NameError: body = [output]
return index(req=req,
title=title,
body=body,
subtitle=subtitle,
adminarea=3)
def perform_modifyrole(req, id_role='0', name_role='', description='put description here.', firerole_def_src='', modified='0', confirm=0):
"""form to add a new role with these values:
name_role - name of the role to be changed
description - optional description of the role
firerole_def_src - optional firerole like definition of the role
"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
ret = acca.acc_get_role_details(id_role)
if ret and modified =='0':
name_role = ret[1]
description = ret[2]
firerole_def_src = ret[3]
if not firerole_def_src or firerole_def_src == '' or firerole_def_src is None:
firerole_def_src = 'deny any'
name_role = cleanstring(name_role)
title='Modify Role'
subtitle = 'step 1 - give values to the requested fields and confirm to modify role'
output = """
""" % (id_role, escape(name_role), escape(description), escape(firerole_def_src))
if modified in [1, '1']:
# description must be changed before submitting
internaldesc = ''
if description != 'put description here.':
internaldesc = description
text = """
modify role with:
\n
name: %s
""" % (name_role, )
if internaldesc:
text += 'description: %s?
' % (description, )
text += 'firewall like role definition: %s' % firerole_def_src.replace('\n', '
')
try:
firerole_def_ser = serialize(compile_role_definition(firerole_def_src))
except InvenioWebAccessFireroleError, msg:
subtitle = 'step 2 - role could not be modified'
output += 'sorry, could not modify role because of troubles with its definition:
%s
' % msg
else:
output += createhiddenform(action="modifyrole",
text=text,
id_role = id_role,
name_role=escape(name_role, True),
description=escape(description, True),
firerole_def_src=escape(firerole_def_src, True),
modified=1,
confirm=1)
if confirm not in ["0", 0]:
result = acca.acc_update_role(id_role, name_role=name_role,
description=internaldesc, firerole_def_ser=firerole_def_ser, firerole_def_src=firerole_def_src)
if result:
subtitle = 'step 2 - role modified'
output += 'role modified:
'
output += tupletotable(header=['id', 'role name',
'description', 'firewall like role definition'],
tuple=[(id_role, name_role, description, firerole_def_src.replace('\n', '
'))])
else:
subtitle = 'step 2 - role could not be modified'
output += 'sorry, could not modify role,
please contact the administrator.
'
body = [output]
return index(req=req,
title=title,
body=body,
subtitle=subtitle,
adminarea=3)
def perform_deleterole(req, id_role="0", confirm=0):
"""select a role and show all connected information,
users - users that can access the role.
actions - actions with possible authorizations."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
title = 'Delete role'
subtitle = 'step 1 - select role to delete'
name_role = acca.acc_get_role_name(id_role=id_role)
output = createroleselect(id_role=id_role,
action="deleterole",
step=1,
roles=acca.acc_get_all_roles(),
button="delete role")
if id_role != "0" and name_role:
subtitle = 'step 2 - confirm delete of role'
output += roledetails(id_role=id_role)
output += createhiddenform(action="deleterole",
text='delete role %s and all connections?' % (name_role, ),
id_role=id_role,
confirm=1)
if confirm:
res = acca.acc_delete_role(id_role=id_role)
subtitle = 'step 3 - confirm role deleted'
if res:
output += "confirm: role %s deleted.
" % (name_role, )
output += "%s entries were removed.
" % (res, )
else:
output += "sorry, the role could not be deleted.
"
elif id_role != "0":
output += 'the role has been deleted...
'
return index(req=req,
title=title,
subtitle=subtitle,
body=[output],
adminarea=3)
def perform_showroledetails(req, id_role):
"""show the details of a role."""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
output = createroleselect(id_role=id_role,
action="showroledetails",
step=1,
roles=acca.acc_get_all_roles(),
button="select role")
if id_role not in [0, '0']:
name_role = acca.acc_get_role_name(id_role=id_role)
output += roledetails(id_role=id_role)
extra = """
- Modify role
-
- modify the role you are seeing
- Add new authorization
- add an authorization.
- Modify authorizations
- modify existing authorizations.
- Connect user
- connect a user to role %(name_role)s.
- Remove user
- remove a user from role %(name_role)s.
""" % {'id_role' : id_role, 'name_role' : name_role}
body = [output, extra]
else:
output += 'no details to show
'
body = [output]
return index(req=req,
title='Show Role Details',
subtitle='show role details',
body=body,
adminarea=3)
def roledetails(id_role=0):
"""create the string to show details about a role. """
name_role = acca.acc_get_role_name(id_role=id_role)
usershlp = acca.acc_get_role_users(id_role)
users = []
for (id, email, dummy) in usershlp:
users.append([id, email, 'show user details' % (id, )])
usertable = tupletotable(header=['id', 'email'], tuple=users, highlight_rows_p=True,
alternate_row_colors_p=True)
actionshlp = acca.acc_get_role_actions(id_role)
actions = []
for (action_id, name, dummy) in actionshlp:
res = acca.acc_find_possible_actions(id_role, action_id)
if res:
authorization_details = tupletotable(header=res[0], tuple=res[1:])
else:
authorization_details = 'no details to show'
actions.append([action_id, name, authorization_details,
'show action details' % (id_role, action_id)])
actiontable = tupletotable(header=['id', 'name', 'parameters', ''], tuple=actions)
# show role details
details = 'role details:
'
role_details = acca.acc_get_role_details(id_role=id_role)
if role_details[3] is None:
role_details[3] = ''
role_details[3] = role_details[3].replace('\n', '
') # Hack for preformatting firerole rules
details += tupletotable(header=['id', 'name', 'description', 'firewall like role definition'],
tuple=[role_details])
# show connected users
details += 'users connected to %s:
' % (headerstrong(role=name_role, query=0), )
if users:
details += usertable
else:
details += 'no users connected.
'
# show connected authorizations
details += 'authorizations for %s:
' % (headerstrong(role=name_role, query=0), )
if actions:
details += actiontable
else:
details += 'no authorizations connected
'
return details
def perform_adduserrole(req, id_role='0', email_user_pattern='', id_user='0', confirm=0):
"""create connection between user and role.
id_role - id of the role to add user to
email_user_pattern - search for users using this pattern
id_user - id of user to add to the role. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
email_out = acca.acc_get_user_email(id_user=id_user)
name_role = acca.acc_get_role_name(id_role=id_role)
title = 'Connect user to role '
subtitle = 'step 1 - select a role'
output = createroleselect(id_role=id_role,
action="adduserrole",
step=1,
roles=acca.acc_get_all_roles())
# role is selected
if id_role != "0":
title += name_role
subtitle = 'step 2 - search for users'
# remove letters not allowed in an email
email_user_pattern = cleanstring_email(email_user_pattern)
text = ' 2. search for user \n'
text += ' \n' % (email_user_pattern, )
output += createhiddenform(action="adduserrole",
text=text,
button="search for users",
id_role=id_role)
# pattern is entered
if email_user_pattern:
# users with matching email-address
try:
users1 = run_sql("""SELECT id, email FROM user WHERE email<>'' AND email RLIKE %s ORDER BY email """, (email_user_pattern, ))
except OperationalError:
users1 = ()
# users that are connected
try:
users2 = run_sql("""SELECT DISTINCT u.id, u.email
FROM user u LEFT JOIN user_accROLE ur ON u.id = ur.id_user
WHERE ur.id_accROLE = %s AND u.email RLIKE %s
ORDER BY u.email """, (id_role, email_user_pattern))
except OperationalError:
users2 = ()
# no users that match the pattern
if not (users1 or users2):
output += 'no qualified users, try new search.
'
elif len(users1) > MAXSELECTUSERS:
output += '%s hits, too many qualified users, specify more narrow search. (limit %s)
' % (len(users1), MAXSELECTUSERS)
# show matching users
else:
subtitle = 'step 3 - select a user'
users = []
extrausers = []
for (user_id, email) in users1:
if (user_id, email) not in users2: users.append([user_id,email,''])
for (user_id, email) in users2:
extrausers.append([-user_id, email,''])
output += createuserselect(id_user=id_user,
action="adduserrole",
step=3,
users=users,
extrausers=extrausers,
button="add this user",
id_role=id_role,
email_user_pattern=email_user_pattern)
try: id_user = int(id_user)
except ValueError: pass
# user selected already connected to role
if id_user < 0:
output += 'users in brackets are already attached to the role, try another one...
'
# a user is selected
elif email_out:
subtitle = "step 4 - confirm to add user"
output += createhiddenform(action="adduserrole",
text='add user %s to role %s?' % (email_out, name_role),
id_role=id_role,
email_user_pattern=email_user_pattern,
id_user=id_user,
confirm=1)
# it is confirmed that this user should be added
if confirm:
# add user
result = acca.acc_add_user_role(id_user=id_user, id_role=id_role)
if result and result[2]:
subtitle = 'step 5 - confirm user added'
output += 'confirm: user %s added to role %s.
' % (email_out, name_role)
else:
subtitle = 'step 5 - user could not be added'
output += 'sorry, but user could not be added.
'
extra = """
- Create new role
- go here to add a new role.
"""
if str(id_role) != "0":
extra += """
- Remove users
- remove users from role %s.
- Connected users
- show all users connected to role %s.
- Add authorization
- start adding new authorizations to role %s.
""" % (id_role, name_role, id_role, name_role, id_role, name_role)
return index(req=req,
title=title,
subtitle=subtitle,
body=[output, extra],
adminarea=3)
def perform_addroleuser(req, email_user_pattern='', id_user='0', id_role='0', confirm=0):
"""delete connection between role and user.
id_role - id of role to disconnect
id_user - id of user to disconnect. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
email_out = acca.acc_get_user_email(id_user=id_user)
name_role = acca.acc_get_role_name(id_role=id_role)
# used to sort roles, and also to determine right side links
con_roles = []
not_roles = []
title = 'Connect user to roles'
subtitle = 'step 1 - search for users'
# clean email search string
email_user_pattern = cleanstring_email(email_user_pattern)
text = ' 1. search for user \n'
text += ' \n' % (email_user_pattern, )
output = createhiddenform(action='addroleuser',
text=text,
button='search for users',
id_role=id_role)
if email_user_pattern:
subtitle = 'step 2 - select user'
try:
users1 = run_sql("""SELECT id, email FROM user WHERE email<>'' AND email RLIKE %s ORDER BY email """, (email_user_pattern, ))
except OperationalError:
users1 = ()
users = []
for (id, email) in users1: users.append([id, email, ''])
# no users
if not users:
output += 'no qualified users, try new search.
'
# too many users
elif len(users) > MAXSELECTUSERS:
output += '%s hits, too many qualified users, specify more narrow search. (limit %s)
' % (len(users), MAXSELECTUSERS)
# ok number of users
else:
output += createuserselect(id_user=id_user,
action='addroleuser',
step=2,
users=users,
button='select user',
email_user_pattern=email_user_pattern)
if int(id_user):
subtitle = 'step 3 - select role'
# roles the user is connected to
role_ids = acca.acc_get_user_roles(id_user=id_user)
# all the roles, lists are sorted on the background of these...
all_roles = acca.acc_get_all_roles()
# sort the roles in connected and not connected roles
for (id, name, description, dummy, dummy) in all_roles:
if id in role_ids: con_roles.append([-id, name, description])
else: not_roles.append([id, name, description])
# create roleselect
output += createroleselect(id_role=id_role,
action='addroleuser',
step=3,
roles=not_roles,
extraroles=con_roles,
extrastamp='(connected)',
button='add this role',
email_user_pattern=email_user_pattern,
id_user=id_user)
if int(id_role) < 0:
name_role = acca.acc_get_role_name(id_role=-int(id_role))
output += 'role %s already connected to the user, try another one...
' % (name_role, )
elif int(id_role):
subtitle = 'step 4 - confirm to add role to user'
output += createhiddenform(action='addroleuser',
text='add role %s to user %s?' % (name_role, email_out),
email_user_pattern=email_user_pattern,
id_user=id_user,
id_role=id_role,
confirm=1)
if confirm:
# add role
result = acca.acc_add_user_role(id_user=id_user, id_role=id_role)
if result and result[2]:
subtitle = 'step 5 - confirm role added'
output += '
confirm: role %s added to user %s.
' % (name_role, email_out)
else:
subtitle = 'step 5 - role could not be added'
output += 'sorry, but role could not be added
'
extra = """
- Create new role
- go here to add a new role.
"""
if int(id_user) and con_roles:
extra += """
- Remove roles
- disconnect roles from user %s.
""" % (id_user, email_out)
if int(id_role):
if int(id_role) < 0: id_role = -int(id_role)
extra += """
- Remove users
- disconnect users from role %s.
-
""" % (id_role, name_role)
return index(req=req,
title=title,
subtitle=subtitle,
body=[output, extra],
adminarea=5)
def perform_deleteuserrole(req, id_role='0', id_user='0', reverse=0, confirm=0):
"""delete connection between role and user.
id_role - id of role to disconnect
id_user - id of user to disconnect. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
title = 'Remove user from role'
email_user = acca.acc_get_user_email(id_user=id_user)
name_role = acca.acc_get_role_name(id_role=id_role)
output = ''
if reverse in [0, '0']:
adminarea = 3
subtitle = 'step 1 - select the role'
output += createroleselect(id_role=id_role,
action="deleteuserrole",
step=1,
roles=acca.acc_get_all_roles())
if id_role != "0":
subtitle = 'step 2 - select the user'
output += createuserselect(id_user=id_user,
action="deleteuserrole",
step=2,
users=acca.acc_get_role_users(id_role=id_role),
id_role=id_role)
else:
adminarea = 5
# show only if user is connected to a role, get users connected to roles
users = run_sql("""SELECT DISTINCT(u.id), u.email, u.note
FROM user u LEFT JOIN user_accROLE ur
ON u.id = ur.id_user
WHERE ur.id_accROLE != 'NULL' AND u.email != ''
ORDER BY u.email """)
has_roles = 1
# check if the user is connected to any roles
for (id, email, note) in users:
if str(id) == str(id_user): break
# user not connected to a role
else:
subtitle = 'step 1 - user not connected'
output += 'no need to remove roles from user %s,
user is not connected to any roles.
' % (email_user, )
has_roles, id_user = 0, '0' # stop the rest of the output below...
# user connected to roles
if has_roles:
output += createuserselect(id_user=id_user,
action="deleteuserrole",
step=1,
users=users,
reverse=reverse)
if id_user != "0":
subtitle = 'step 2 - select the role'
role_ids = acca.acc_get_user_roles(id_user=id_user)
all_roles = acca.acc_get_all_roles()
roles = []
for (id, name, desc, dummy, dummy) in all_roles:
if id in role_ids: roles.append([id, name, desc])
output += createroleselect(id_role=id_role,
action="deleteuserrole",
step=2,
roles=roles,
id_user=id_user,
reverse=reverse)
if id_role != '0' and id_user != '0':
subtitle = 'step 3 - confirm delete of user'
output += createhiddenform(action="deleteuserrole",
text='delete user %s from %s?' % (headerstrong(user=id_user), headerstrong(role=id_role)),
id_role=id_role,
id_user=id_user,
reverse=reverse,
confirm=1)
if confirm:
res = acca.acc_delete_user_role(id_user=id_user, id_role=id_role)
if res:
subtitle = 'step 4 - confirm delete of user'
output += 'confirm: deleted user %s from role %s.
' % (email_user, name_role)
else:
subtitle = 'step 4 - user could not be deleted'
output += 'sorry, but user could not be deleted
user is probably already deleted.'
extra = ''
if str(id_role) != "0":
extra += """
- Connect user
- add users to role %s.
""" % (id_role, name_role)
if int(reverse):
extra += """
- Remove user
- remove users from role %s.
""" % (id_role, name_role)
extra += '
'
if str(id_user) != "0":
extra += """
- Connect role
- add roles to user %s.
""" % (email_user, id_user, email_user)
if not int(reverse):
extra += """
- Remove role
- remove roles from user %s.
""" % (id_user, email_user, email_user)
extra += '
'
if extra: body = [output, extra]
else: body = [output]
return index(req=req,
title=title,
subtitle=subtitle,
body=body,
adminarea=adminarea)
def perform_showuserdetails(req, id_user=0):
"""show the details of a user. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
if id_user not in [0, '0']:
output = userdetails(id_user=id_user)
email_user = acca.acc_get_user_email(id_user=id_user)
extra = """
- Connect role
- connect a role to user %s.
- Remove role
- remove a role from user %s.
""" % (id_user, email_user, email_user, id_user, email_user)
body = [output, extra]
else:
body = ['no details to show
']
return index(req=req,
title='Show User Details',
subtitle='show user details',
body=body,
adminarea=5)
def userdetails(id_user=0):
"""create the string to show details about a user. """
# find necessary details
email_user = acca.acc_get_user_email(id_user=id_user)
userroles = acca.acc_get_user_roles(id_user=id_user)
conn_roles = []
# find connected roles
for (id, name, desc, dummy, dummy) in acca.acc_get_all_roles():
if id in userroles:
conn_roles.append([id, name, desc])
conn_roles[-1].append('show details' % (id, ))
if conn_roles:
# print details
details = 'roles connected to user %s
' % (email_user, )
details += tupletotable(header=['id', 'name', 'description', ''], tuple=conn_roles)
else:
details = 'no roles connected to user %s.
' % (email_user, )
return details
def perform_addauthorization(req, id_role="0", id_action="0", optional=0, reverse="0", confirm=0, **keywords):
""" form to add new connection between user and role:
id_role - role to connect
id_action - action to connect
reverse - role or action first? """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
# values that might get used
name_role = acca.acc_get_role_name(id_role=id_role) or id_role
name_action = acca.acc_get_action_name(id_action=id_action) or id_action
optional = optional == 'on' and 1 or int(optional)
extra = """
- Create new role
- go here to add a new role.
"""
# create the page according to which step the user is on
# role -> action -> arguments
if reverse in ["0", 0]:
adminarea = 3
subtitle = 'step 1 - select role'
output = createroleselect(id_role=id_role,
action="addauthorization",
step=1,
roles=acca.acc_get_all_roles(),
reverse=reverse)
if str(id_role) != "0":
subtitle = 'step 2 - select action'
rolacts = acca.acc_get_role_actions(id_role)
allhelp = acca.acc_get_all_actions()
allacts = []
for r in allhelp:
if r not in rolacts: allacts.append(r)
output += createactionselect(id_action=id_action,
action="addauthorization",
step=2,
actions=rolacts,
extraactions=allacts,
id_role=id_role,
reverse=reverse)
# action -> role -> arguments
else:
adminarea = 4
subtitle = 'step 1 - select action'
output = createactionselect(id_action=id_action,
action="addauthorization",
step=1,
actions=acca.acc_get_all_actions(),
reverse=reverse)
if str(id_action) != "0":
subtitle = 'step 2 - select role'
actroles = acca.acc_get_action_roles(id_action)
allhelp = acca.acc_get_all_roles()
allroles = []
for r in allhelp:
if r not in actroles: allroles.append(r)
output += createroleselect(id_role=id_role,
action="addauthorization",
step=2,
roles=actroles,
extraroles=allroles,
id_action=id_action,
reverse=reverse)
# ready for step 3 no matter which direction we took to get here
if id_action != "0" and id_role != "0":
# links to adding authorizations in the other direction
if str(reverse) == "0":
extra += """
- Add authorization
- add authorizations to action %s.
""" % (id_action, name_action)
else:
extra += """
- Add authorization
- add authorizations to role %s.
""" % (id_role, name_role)
subtitle = 'step 3 - enter values for the keywords\n'
output += """
""" % (headerstrong(role=name_role, action=name_action, query=0), )
else:
output += 'connection without arguments is already created.
'
else:
# action with arguments
optionalargs = acca.acc_get_action_is_optional(id_action=id_action)
output += '3. authorized arguments
'
if optionalargs:
# optional arguments
output += """
connect %s to %s for any arguments
connect %s to %s for only these argument cases:
""" % (optional and 'checked="checked"' or '', name_role, name_action, not optional and 'checked="checked"' or '', name_role, name_action)
# list the arguments
allkeys = 1
for key in res_keys:
output += '%s \n \n'
output += '\n'
# ask for confirmation
if str(allkeys) != "0" or optional:
keys = keywords.keys()
keys.reverse()
subtitle = 'step 4 - confirm add of authorization\n'
text = """
create connection between
%s
""" % (headerstrong(role=name_role, action=name_action, query=0), )
if optional:
text += 'withouth arguments'
keywords = {}
else:
for key in keys:
text += '%s: %s \n' % (escape(key), escape(keywords[key]))
output += createhiddenform(action="addauthorization",
text=text,
id_role=id_role,
id_action=id_action,
reverse=reverse,
confirm=1,
optional=optional,
**keywords)
# show existing authorizations, found authorizations further up in the code...
# res_auths = acca.acc_find_possible_actions(id_role, id_action)
output += 'existing authorizations:
'
if res_auths:
output += tupletotable(header=res_auths[0], tuple=res_auths[1:])
# shortcut to modifying authorizations
extra += """
- Modify authorizations
- modify the existing authorizations.
""" % (id_role, id_action, reverse)
else: output += 'no details to show
'
# user confirmed to add entries
if confirm:
subtitle = 'step 5 - confirm authorization added'
res1 = acca.acc_add_authorization(name_role=name_role,
name_action=name_action,
optional=optional,
**keywords)
if res1:
res2 = acca.acc_find_possible_actions(id_role, id_action)
arg = res1[0][3] # the arglistid
new = [res2[0]]
for row in res2[1:]:
if int(row[0]) == int(arg): new.append(row)
newauths = tupletotable(header=new[0], tuple=new[1:])
newentries = tupletotable(header=['role id', 'action id', 'argument id', '#'], tuple=res1)
st = 'style="vertical-align: top"'
output += """
new authorization and entries:
%s
%s
""" % (st, newauths, st, newentries)
else: output += 'sorry, authorization could not be added,
it probably already exists
'
# trying to put extra link on the right side
try: body = [output, extra]
except NameError: body = [output]
return index(req=req,
title = 'Create entry for new authorization',
subtitle=subtitle,
body=body,
adminarea=adminarea)
def perform_deleteroleaction(req, id_role="0", id_action="0", reverse=0, confirm=0):
"""delete all connections between a role and an action.
id_role - id of the role
id_action - id of the action
reverse - 0: ask for role first
1: ask for action first"""
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
title = 'Remove action from role '
if reverse in ["0", 0]:
# select role -> action
adminarea = 3
subtitle = 'step 1 - select a role'
output = createroleselect(id_role=id_role,
action="deleteroleaction",
step=1,
roles=acca.acc_get_all_roles(),
reverse=reverse)
if id_role != "0":
rolacts = acca.acc_get_role_actions(id_role=id_role)
subtitle = 'step 2 - select the action'
output += createactionselect(id_action=id_action,
action="deleteroleaction",
step=2,
actions=rolacts,
reverse=reverse,
id_role=id_role,
button="remove connection and all authorizations")
else:
# select action -> role
adminarea = 4
subtitle = 'step 1 - select an action'
output = createactionselect(id_action=id_action,
action="deleteroleaction",
step=1,
actions=acca.acc_get_all_actions(),
reverse=reverse)
if id_action != "0":
actroles = acca.acc_get_action_roles(id_action=id_action)
subtitle = 'step 2 - select the role'
output += createroleselect(id_role=id_role,
action="deleteroleaction",
step=2,
roles=actroles,
button="remove connection and all authorizations",
id_action=id_action,
reverse=reverse)
if id_action != "0" and id_role != "0":
subtitle = 'step 3 - confirm to remove authorizations'
# ask for confirmation
res = acca.acc_find_possible_actions(id_role, id_action)
if res:
output += 'authorizations that will be deleted:
'
output += tupletotable(header=res[0], tuple=res[1:])
output += createhiddenform(action="deleteroleaction",
text='remove %s from %s' % (headerstrong(action=id_action), headerstrong(role=id_role)),
confirm=1,
id_role=id_role,
id_action=id_action,
reverse=reverse)
else:
output += 'no authorizations'
# confirmation is given
if confirm:
subtitle = 'step 4 - confirm authorizations removed '
res = acca.acc_delete_role_action(id_role=id_role, id_action=id_action)
if res:
output += 'confirm: removed %s from %s
' % (headerstrong(action=id_action), headerstrong(role=id_role))
output += '%s entries were removed.
' % (res, )
else:
output += 'sorry, no entries could be removed.
'
return index(req=req,
title=title,
subtitle=subtitle,
body=[output],
adminarea=adminarea)
def perform_modifyauthorizations(req, id_role="0", id_action="0", reverse=0, confirm=0, errortext='', sel='', authids=[]):
"""given ids of a role and an action, show all possible action combinations
with checkboxes and allow user to access other functions.
id_role - id of the role
id_action - id of the action
reverse - 0: ask for role first
1: ask for action first
sel - which button and modification that is selected
errortext - text to print when no connection exist between role and action
authids - ids of checked checkboxes """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0: return mustloginpage(req, auth_message)
name_role = acca.acc_get_role_name(id_role)
name_action = acca.acc_get_action_name(id_action)
output = ''
try: id_role, id_action, reverse = int(id_role), int(id_action), int(reverse)
except ValueError: pass
extra = """
- Create new role
- go here to add a new role.
"""
if id_role or id_action:
extra += '\n\n'
if id_role and id_action:
extra += """
- Add authorizations
- add an authorization to the existing ones.
""" % (id_role, id_action, reverse)
if id_role:
extra += """
- Add authorizations
- add to role %s.
""" % (id_role, name_role)
if id_action:
extra += """
- Add authorizations
- add to action %s.
""" % (id_action, name_action)
extra += '\n
\n'
if not reverse:
# role -> action
adminarea = 3
subtitle = 'step 1 - select the role'
output += createroleselect(id_role=str(id_role),
action="modifyauthorizations",
step=1,
roles=acca.acc_get_all_roles(),
reverse=reverse)
if id_role:
rolacts = acca.acc_get_role_actions(id_role=id_role)
subtitle = 'step 2 - select the action'
output += createactionselect(id_action=str(id_action),
action="modifyauthorizations",
step=2,
actions=rolacts,
id_role=id_role,
reverse=reverse)
else:
adminarea = 4
# action -> role
subtitle = 'step 1 - select the action'
output += createactionselect(id_action=str(id_action),
action="modifyauthorizations",
step=1,
actions=acca.acc_get_all_actions(),
reverse=reverse)
if id_action:
actroles = acca.acc_get_action_roles(id_action=id_action)
subtitle = 'step 2 - select the role'
output += createroleselect(id_role=str(id_role),
action="modifyauthorizations",
step=2,
roles=actroles,
id_action=id_action,
reverse=reverse)
if errortext: output += '%s
' % (errortext, )
if id_role and id_action:
# adding to main area
if type(authids) is not list: authids = [authids]
subtitle = 'step 3 - select groups and modification'
# get info
res = acca.acc_find_possible_actions(id_role, id_action)
# clean the authids
hiddenids = []
if sel in ['delete selected']:
hiddenids = authids[:]
elif sel in ['split groups', 'merge groups']:
for authid in authids:
arghlp = res[int(authid)][0]
if authid not in hiddenids and arghlp not in [-1, '-1', 0, '0']: hiddenids.append(authid)
authids = hiddenids[:]
if confirm:
# do selected modification and output with new authorizations
if sel == 'split groups':
res = splitgroups(id_role, id_action, authids)
elif sel == 'merge groups':
res = mergegroups(id_role, id_action, authids)
elif sel == 'delete selected':
res = deleteselected(id_role, id_action, authids)
authids = []
res = acca.acc_find_possible_actions(id_role, id_action)
output += 'authorizations after %s.
\n' % (sel, )
elif sel and authids:
output += 'confirm choice of authorizations and modification.
\n'
else:
output += 'select authorizations and perform modification.
\n'
if not res:
errortext = 'all connections deleted, try different '
if reverse in ["0", 0]:
return perform_modifyauthorizations(req=req, id_role=id_role, errortext=errortext + 'action.')
else:
return perform_modifyauthorizations(req=req, id_action=id_action, reverse=reverse, errortext=errortext + 'role.')
# display
output += modifyauthorizationsmenu(id_role, id_action, header=res[0], tuple=res[1:], checked=authids, reverse=reverse)
if sel and authids:
subtitle = 'step 4 - confirm to perform modification'
# form with hidden authids
output += ''
# tried to perform modification without something selected
elif sel and not authids and not confirm:
output += 'no valid groups selected
'
# trying to put extra link on the right side
try:
body = [output, extra]
except NameError:
body = [output]
# Display the page
return index(req=req,
title='Modify Authorizations',
subtitle=subtitle,
body=body,
adminarea=adminarea)
def modifyauthorizationsmenu(id_role, id_action, tuple=[], header=[],
checked=[], reverse=0):
"""create table with header and checkboxes, used for multiple choice.
makes use of tupletotable to add the actual table
id_role - selected role, hidden value in the form
id_action - selected action, hidden value in the form
tuple - all rows to be put in the table (with checkboxes)
header - column headers, empty strings added at start and end
checked - ids of rows to be checked """
if not tuple:
return 'no authorisations...'
argnum = len(acca.acc_get_action_keywords(id_action=id_action))
tuple2 = []
for t in tuple:
tuple2.append(t[:])
tuple2 = addcheckboxes(datalist=tuple2, name='authids', startindex=1,
checked=checked)
hidden = ' \n' \
% (id_role, )
hidden += ' \n' \
% (id_action, )
hidden += ' \n' \
% (reverse, )
button = '\n'
if argnum > 1:
button += '\n'
button += '\n'
hdrstr = ''
for h in [''] + header + ['']:
hdrstr += ' %s \n' % (h, )
if hdrstr:
hdrstr = ' \n%s\n \n' % (hdrstr, )
output = '\n'
return output
def splitgroups(id_role=0, id_action=0, authids=[]):
"""get all the old ones, gather up the arglistids find a list of
arglistidgroups to be split, unique get all actions in groups outside
of the old ones, (old arglistid is allowed).
show them like in showselect. """
if not id_role or not id_action or not authids:
return 0
# find all the actions
datalist = acca.acc_find_possible_actions(id_role, id_action)
if type(authids) is str:
authids = [authids]
for i in range(len(authids)):
authids[i] = int(authids[i])
# argumentlistids of groups to be split
splitgrps = []
for authid in authids:
hlp = datalist[authid][0]
if hlp not in splitgrps and authid in range(1, len(datalist)):
splitgrps.append(hlp)
# split groups and return success or failure
result = 1
for splitgroup in splitgrps:
result = 1 and acca.acc_split_argument_group(id_role, id_action,
splitgroup)
return result
def mergegroups(id_role=0, id_action=0, authids=[]):
"""get all the old ones, gather up the argauthids find a list
of arglistidgroups to be split, unique get all actions in groups
outside of the old ones, (old arglistid is allowed).
show them like in showselect."""
if not id_role or not id_action or not authids:
return 0
datalist = acca.acc_find_possible_actions(id_role, id_action)
if type(authids) is str:
authids = [authids]
for i in range(len(authids)):
authids[i] = int(authids[i])
# argumentlistids of groups to be merged
mergegroups = []
for authid in authids:
hlp = datalist[authid][0]
if hlp not in mergegroups and authid in range(1, len(datalist)):
mergegroups.append(hlp)
# merge groups and return success or failure
if acca.acc_merge_argument_groups(id_role, id_action, mergegroups):
return 1
else:
return 0
def deleteselected(id_role=0, id_action=0, authids=[]):
"""delete checked authorizations/possible actions, ids in authids.
id_role - role to delete from
id_action - action to delete from
authids - listids for which possible actions to delete."""
if not id_role or not id_action or not authids:
return 0
if type(authids) in [str, int]:
authids = [authids]
for i in range(len(authids)):
authids[i] = int(authids[i])
result = acca.acc_delete_possible_actions(id_role=id_role,
id_action=id_action,
authids=authids)
return result
def headeritalic(**ids):
"""transform keyword=value pairs to string with value in italics.
**ids - a dictionary of pairs to create string from """
output = ''
value = ''
table = ''
for key in ids.keys():
if key in ['User', 'user']:
value, table = 'email', 'user'
elif key in ['Role', 'role']:
value, table = 'name', 'accROLE'
elif key in ['Action', 'action']:
value, table = 'name', 'accACTION'
else:
if output:
output += ' and '
output += ' %s %s' % (key, ids[key])
continue
res = run_sql("""SELECT %%s FROM %s WHERE id = %%s""" % wash_table_column_name(table), (value, ids[key])) # kwalitee: disable=sql
if res:
if output:
output += ' and '
output += ' %s %s' % (key, res[0][0])
return output
def headerstrong(query=1, **ids):
"""transform keyword=value pairs to string with value in strong text.
**ids - a dictionary of pairs to create string from
query - 1 -> try to find names to ids of role, user and action.
0 -> do not try to find names, use the value passed on """
output = ''
value = ''
table = ''
for key in ids.keys():
if key in ['User', 'user']:
value, table = 'email', 'user'
elif key in ['Role', 'role']:
value, table = 'name', 'accROLE'
elif key in ['Action', 'action']:
value, table = 'name', 'accACTION'
else:
if output:
output += ' and '
output += ' %s %s' % (key, ids[key])
continue
if query:
res = run_sql("""SELECT %%s FROM %s WHERE id = %%s""" % wash_table_column_name(table), (value, ids[key])) # kwalitee: disable=sql
if res:
if output:
output += ' and '
output += ' %s %s' % (key, res[0][0])
else:
if output:
output += ' and '
output += ' %s %s' % (key, ids[key])
return output
def startpage():
"""create the menu for the startpage"""
body = """
selection for WebAccess Admin
- Role Area
- main area to configure administration rights and authorization rules.
- Action Area
- configure administration rights with the actions as starting point.
- User Area
- configure administration rights with the users as starting point.
- Reset Area
- reset roles, actions and authorizations.
- Manage accounts Area
- manage user accounts.
- Delegate Rights - With Restrictions
- delegate your rights for some roles.
- Manage Robot Login
- Manage robot login keys and test URLs
"""
return body
def rankarea():
return "Rankmethod area"
def perform_simpleauthorization(req, id_role=0, id_action=0):
"""show a page with simple overview of authorizations between a
connected role and action. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0:
return mustloginpage(req, auth_message)
res = acca.acc_find_possible_actions(id_role, id_action)
if res:
extra = createhiddenform(action='modifyauthorizations',
button='modify authorizations',
id_role=id_role,
id_action=id_action)
output = 'authorizations for %s:
' \
% (headerstrong(action=id_action, role=id_role), )
output += tupletotable(header=res[0], tuple=res[1:], extracolumn=extra)
else:
output = 'no details to show'
return index(req=req,
title='Simple authorization details',
subtitle='simple authorization details',
body=[output],
adminarea=3)
def perform_showroleusers(req, id_role=0):
"""show a page with simple overview of a role and connected users. """
(auth_code, auth_message) = is_adminuser(req)
if auth_code != 0:
return mustloginpage(req, auth_message)
res = acca.acc_get_role_users(id_role=id_role)
name_role = acca.acc_get_role_name(id_role=id_role)
if res:
users = []
for (role_id, name, dummy) in res:
users.append([role_id, name, 'show user details' % (role_id, )])
output = 'users connected to %s:
' \
% (headerstrong(role=id_role), )
output += tupletotable(header=['id', 'name', ''], tuple=users)
else:
output = 'no users connected to role %s' \
% (name_role, )
extra = """
- Connect user
- connect users to the role.
""" % (id_role, )
return index(req=req,
title='Users connected to role %s' % (name_role, ),
subtitle='simple details',
body=[output, extra],
adminarea=3)
def createselect(id_input="0", label="", step=0, name="",
action="", list=[], extralist=[], extrastamp='',
button="", **hidden):
"""create form with select and hidden values
id - the one to choose as selected if exists
label - label shown to the left of the select
name - the name of the select on which to reference it
list - primary list to select from
extralist - list of options to be put in paranthesis
extrastamp - stamp extralist entries with this if not ''
usually paranthesis around the entry
button - the value/text to be put on the button
**hidden - name=value pairs to be put as hidden in the form. """
step = step and '%s. ' % step or ''
output = '\n'
return output
def createactionselect(id_action="0", label="select action", step=0,
name="id_action", action="", actions=[], extraactions=[],
extrastamp='', button="select action", **hidden):
"""create a select for roles in a form. see createselect."""
return createselect(id_input=id_action, label=label, step=step, name=name,
action=action, list=actions, extralist=extraactions,
extrastamp=extrastamp, button=button, **hidden)
def createroleselect(id_role="0", label="select role", step=0, name="id_role",
action="", roles=[], extraroles=[], extrastamp='',
button="select role", **hidden):
"""create a select for roles in a form. see createselect."""
return createselect(id_input=id_role, label=label, step=step, name=name,
action=action, list=roles, extralist=extraroles, extrastamp=extrastamp,
button=button, **hidden)
def createuserselect(id_user="0", label="select user", step=0, name="id_user",
action="", users=[], extrausers=[], extrastamp='(connected)',
button="select user", **hidden):
"""create a select for users in a form.see createselect."""
return createselect(id_input=id_user, label=label, step=step, name=name,
action=action, list=users, extralist=extrausers, extrastamp=extrastamp,
button=button, **hidden)
def cleanstring(txt='', comma=0):
"""clean all the strings before submitting to access control admin.
remove characters not letter, number or underscore, also remove leading
underscores and numbers. return cleaned string.
str - string to be cleaned
comma - 1 -> allow the comma to divide multiple arguments
0 -> wash commas as well """
# remove not allowed characters
txt = re.sub(r'[^a-zA-Z0-9_,]', '', txt)
# split string on commas
items = txt.split(',')
txt = ''
for item in items:
if not item:
continue
if comma and txt:
txt += ','
# create valid variable names
txt += re.sub(r'^([0-9_])*', '', item)
return txt
def cleanstring_argumentvalue(txt=''):
"""clean the value of an argument before submitting it.
allowed characters: a-z A-Z 0-9 _ * and space
txt - string to be cleaned """
# remove not allowed characters
txt = re.sub(r'[^a-zA-Z0-9_ *.]', '', txt)
# trim leading and ending spaces
txt = re.sub(r'^ *| *$', '', txt)
return txt
def cleanstring_email(txt=''):
"""clean the string and return a valid email address.
txt - string to be cleaned """
# remove not allowed characters
txt = re.sub(r'[^a-zA-Z0-9_.@-]', '', txt)
return txt
def check_email(txt=''):
"""control that submitted emails are correct.
this little check is not very good, but better than nothing. """
r = re.compile(r'(.)+\@(.)+\.(.)+')
return r.match(txt) and 1 or 0
def send_account_activated_message(account_email, send_to, password, ln=CFG_SITE_LANG):
"""Send an email to the address given by send_to about the new activated
account."""
_ = gettext_set_language(ln)
sub = _("Your account on '%s' has been activated") % CFG_SITE_NAME
body = _("Your account earlier created on '%s' has been activated:") \
% CFG_SITE_NAME + '\n\n'
body += ' ' + _("Username/Email:") + " %s\n" % account_email
body += ' ' + _("Password:") + " %s\n" % ("*" * len(str(password)))
body += "\n---------------------------------"
body += "\n%s" % CFG_SITE_NAME
return send_email(CFG_SITE_SUPPORT_EMAIL, send_to, sub, body, header='')
def send_new_user_account_warning(new_account_email, send_to, password, ln=CFG_SITE_LANG):
"""Send an email to the address given by send_to about the new account
new_account_email."""
_ = gettext_set_language(ln)
sub = _("Account created on '%s'") % CFG_SITE_NAME
body = _("An account has been created for you on '%s':") % CFG_SITE_NAME + '\n\n'
body += ' ' + _("Username/Email:") + " %s\n" % new_account_email
body += ' ' + _("Password:") + " %s\n" % ("*" * len(str(password)))
body += "\n---------------------------------"
body += "\n%s" % CFG_SITE_NAME
return send_email(CFG_SITE_SUPPORT_EMAIL, send_to, sub, body, header='')
def send_account_rejected_message(new_account_email, send_to, ln=CFG_SITE_LANG):
"""Send an email to the address given by send_to about the new account
new_account_email."""
_ = gettext_set_language(ln)
sub = _("Account rejected on '%s'") % CFG_SITE_NAME
body = _("Your request for an account has been rejected on '%s':") \
% CFG_SITE_NAME + '\n\n'
body += ' ' + _("Username/Email: %s") % new_account_email + "\n"
body += "\n---------------------------------"
body += "\n%s" % CFG_SITE_NAME
return send_email(CFG_SITE_SUPPORT_EMAIL, send_to, sub, body, header='')
def send_account_deleted_message(new_account_email, send_to, ln=CFG_SITE_LANG):
"""Send an email to the address given by send_to about the new account
new_account_email."""
_ = gettext_set_language(ln)
sub = _("Account deleted on '%s'") % CFG_SITE_NAME
body = _("Your account on '%s' has been deleted:") % CFG_SITE_NAME + '\n\n'
body += ' ' + _("Username/Email:") + " %s\n" % new_account_email
body += "\n---------------------------------"
body += "\n%s" % CFG_SITE_NAME
return send_email(CFG_SITE_SUPPORT_EMAIL, send_to, sub, body, header='')
def usage(exitcode=1, msg=""):
"""Prints usage info."""
if msg:
print >> sys.stderr, "Error: %s." % msg
print >> sys.stderr
print >> sys.stderr, """Usage: %s [options]
General options:
-h, --help\t\tprint this help
-V, --version\t\tprint version number
Authentication options:
-u, --user=USER\tUser name needed to perform the administrative task
Option to administrate authorizations:
-a, --add\t\tadd default authorization settings
-c, --compile\t\tcompile firewall like role definitions (FireRole)
-r, --reset\t\treset to default settings
-D, --demo\t\tto be used with -a or -r in order to consider demo site authorizationss
""" % sys.argv[0]
sys.exit(exitcode)
def main():
"""Main function that analyzes command line input and calls whatever
is appropriate. """
## parse command line:
# set user-defined options:
options = {'user' : '', 'reset' : 0, 'compile' : 0, 'add' : 0, 'demo' : 0}
try:
opts, args = getopt.getopt(sys.argv[1:], "hVu:racD",
["help", "version", "user=",
"reset", "add", "compile", "demo"])
except getopt.GetoptError, err:
usage(1, err)
try:
for opt in opts:
if opt[0] in ("-h", "--help"):
usage(0)
elif opt[0] in ("-V", "--version"):
print __revision__
sys.exit(0)
elif opt[0] in ("-u", "--user"):
options["user"] = opt[1]
elif opt[0] in ("-r", "--reset"):
options["reset"] = 1
elif opt[0] in ("-a", "--add"):
options["add"] = 1
elif opt[0] in ("-c", "--compile"):
options["compile"] = 1
elif opt[0] in ("-D", "--demo"):
options["demo"] = 1
else:
usage(1)
if options['add'] or options['reset'] or options['compile']:
if acca.acc_get_action_id('cfgwebaccess'):
# Action exists hence authentication works :-)
options['user'] = authenticate(options['user'],
authorization_msg="WebAccess Administration",
authorization_action="cfgwebaccess")
if options['reset'] and options['demo']:
acca.acc_reset_default_settings([CFG_SITE_ADMIN_EMAIL], DEF_DEMO_USER_ROLES, DEF_DEMO_ROLES, DEF_DEMO_AUTHS)
print "Reset default demo site settings."
elif options['reset']:
acca.acc_reset_default_settings([CFG_SITE_ADMIN_EMAIL])
print "Reset default settings."
elif options['add'] and options['demo']:
acca.acc_add_default_settings([CFG_SITE_ADMIN_EMAIL], DEF_DEMO_USER_ROLES, DEF_DEMO_ROLES, DEF_DEMO_AUTHS)
print "Added default demo site settings."
elif options['add']:
acca.acc_add_default_settings([CFG_SITE_ADMIN_EMAIL])
print "Added default settings."
if options['compile']:
repair_role_definitions()
print "Compiled firewall like role definitions."
else:
usage(1, "You must specify at least one command")
except StandardError, e:
register_exception()
usage(e)
return
### okay, here we go:
if __name__ == '__main__':
main()
diff --git a/modules/webaccess/web/admin/webaccessadmin.py b/modules/webaccess/web/admin/webaccessadmin.py
index d6703fbd7..f2f206cec 100644
--- a/modules/webaccess/web/admin/webaccessadmin.py
+++ b/modules/webaccess/web/admin/webaccessadmin.py
@@ -1,348 +1,353 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Invenio WebAccess Administrator Interface."""
__revision__ = "$Id$"
from invenio.config import CFG_SITE_LANG
import invenio.webaccessadmin_lib as wal
from invenio.access_control_config import CFG_ACC_EMPTY_ROLE_DEFINITION_SRC
# reload(wal)
# from invenio.webaccessadmin_lib import index
index = wal.index
def rolearea(req, grep='', ln=CFG_SITE_LANG):
"""create the role area menu page."""
return wal.perform_rolearea(req=req, grep=grep)
def actionarea(req, grep='', ln=CFG_SITE_LANG):
"""create the role area menu page."""
return wal.perform_actionarea(req=req, grep=grep)
def managerobotlogin(req, robot_name='', new_pwd1='', new_pwd2='', login_method='', timeout='', referer='', ip='', action='', confirm=0, email='', groups='', nickname='', json_assertion='', url_only=0, ln=CFG_SITE_LANG):
"""Manage robot login keys and test URL."""
return wal.perform_managerobotlogin(req=req, robot_name=robot_name, new_pwd1=new_pwd1, new_pwd2=new_pwd2, login_method=login_method, timeout=timeout, referer=referer, ip=ip, action=action, confirm=confirm, email=email, groups=groups, nickname=nickname, json_assertion=json_assertion, url_only=url_only)
def userarea(req, email_user_pattern='', ln=CFG_SITE_LANG):
"""create the user area menu page. """
return wal.perform_userarea(req=req,
email_user_pattern=email_user_pattern)
def listgroups(req, ln=CFG_SITE_LANG):
return wal.perform_listgroups(req=req)
def resetarea(req, ln=CFG_SITE_LANG):
"""create the role area menu page."""
return wal.perform_resetarea(req=req)
def resetdefaultsettings(req, superusers=[], confirm=0, ln=CFG_SITE_LANG):
"""create the reset default settings page. """
return wal.perform_resetdefaultsettings(req=req,
superusers=superusers,
confirm=confirm)
def adddefaultsettings(req, superusers=[], confirm=0, ln=CFG_SITE_LANG):
"""create the add default settings page. """
return wal.perform_adddefaultsettings(req=req,
superusers=superusers,
confirm=confirm)
def manageaccounts(req, mtype='', content='', confirm=0, ln=CFG_SITE_LANG):
"""enable, disable and edit accounts"""
return wal.perform_manageaccounts(req=req, mtype=mtype, content=content, confirm=confirm)
def modifyaccountstatus(req, userID, email_user_pattern='', limit_to=-1, maxpage=25, page=1, callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""enable or disable account"""
return wal.perform_modifyaccountstatus(req=req, userID=userID, email_user_pattern=email_user_pattern, limit_to=limit_to, maxpage=maxpage, page=page, callback=callback, confirm=confirm)
def modifypreferences(req, userID, login_method='', callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""modify the preferences of an account"""
return wal.perform_modifypreferences(req=req, userID=userID, login_method=login_method, callback=callback, confirm=confirm)
+def modifyapikeydata(req, userID, keyID, status, callback='yes', confirm=0, ln=CFG_SITE_LANG):
+ """modify the status of a REST API key"""
+
+ return wal.perform_modifyapikeydata(req=req, userID=userID, keyID=keyID, status=status, callback=callback, confirm=confirm)
+
def modifylogindata(req, userID, nickname='', email='', password='', callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""modify the email/password of an account"""
return wal.perform_modifylogindata(req=req, userID=userID, nickname=nickname, email=email, password=password, callback=callback, confirm=confirm)
def rejectaccount(req, userID, email_user_pattern='', limit_to=-1, maxpage=25, page=1, callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""Set account inactive, delete it and send email to the owner."""
return wal.perform_rejectaccount(req=req, userID=userID, email_user_pattern=email_user_pattern, limit_to=limit_to, maxpage=maxpage, page=page, callback=callback, confirm=confirm)
def deleteaccount(req, userID, callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""delete account"""
return wal.perform_deleteaccount(req=req, userID=userID, callback=callback, confirm=confirm)
def createaccount(req, email='', password='', callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""create account"""
return wal.perform_createaccount(req=req, email=email, password=password, callback=callback, confirm=confirm)
def editaccount(req, userID, mtype='', content='', callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""edit account. """
return wal.perform_editaccount(req=req, userID=userID, mtype=mtype, content=content, callback=callback, confirm=confirm)
def becomeuser(req, userID='', callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""edit account. """
return wal.perform_becomeuser(req=req, userID=userID, callback=callback, confirm=confirm)
def modifyaccounts(req, email_user_pattern='', limit_to=-1, maxpage=25, page=1, callback='yes', confirm=0, ln=CFG_SITE_LANG):
"""Modify accounts. """
return wal.perform_modifyaccounts(req=req, email_user_pattern=email_user_pattern, limit_to=limit_to, maxpage=maxpage, page=page, callback=callback,confirm=confirm)
def delegate_startarea(req, ln=CFG_SITE_LANG):
"""add info here"""
return wal.perform_delegate_startarea(req=req)
def delegate_adminsetup(req, id_role_admin=0, id_role_delegate=0, confirm=0, ln=CFG_SITE_LANG):
"""add info here"""
return wal.perform_delegate_adminsetup(req=req,
id_role_admin=id_role_admin,
id_role_delegate=id_role_delegate,
confirm=confirm)
def delegate_adduserrole(req, id_role=0, email_user_pattern='', id_user=0, confirm=0, ln=CFG_SITE_LANG):
"""add info here"""
return wal.perform_delegate_adduserrole(req=req,
id_role=id_role,
email_user_pattern=email_user_pattern,
id_user=id_user,
confirm=confirm)
def delegate_deleteuserrole(req, id_role=0, id_user=0, confirm=0, ln=CFG_SITE_LANG):
"""add info here"""
return wal.perform_delegate_deleteuserrole(req=req,
id_role=id_role,
id_user=id_user,
confirm=confirm)
def addrole(req, name_role='', description='put description here.', firerole_def_src=CFG_ACC_EMPTY_ROLE_DEFINITION_SRC, confirm=0, ln=CFG_SITE_LANG):
"""form to add a new role with these values:
name_role - name of the new role
description - optional description of the role
firerole_def_src - optional firerole like definition
"""
return wal.perform_addrole(req=req,
name_role=name_role,
description=description,
firerole_def_src=firerole_def_src,
confirm=confirm)
def modifyrole(req, id_role='0', name_role='', description='put description here.', firerole_def_src='', modified='0', confirm=0, ln=CFG_SITE_LANG):
"""form to add a new role with these values:
name_role - name of the new role
description - optional description of the role
firerole_def_src - optional firerole like definition
"""
return wal.perform_modifyrole(req=req,
id_role=id_role,
name_role=name_role,
description=description,
firerole_def_src=firerole_def_src,
modified=modified,
confirm=confirm)
def deleterole(req, id_role="0", confirm=0, ln=CFG_SITE_LANG):
"""select a role and show all connected information,
users - users that can access the role.
actions - actions with possible authorizations."""
return wal.perform_deleterole(req=req,
id_role=id_role,
confirm=confirm)
def showroledetails(req, id_role='0', ln=CFG_SITE_LANG):
"""show the details of a role."""
return wal.perform_showroledetails(req=req,
id_role=id_role)
def showactiondetails(req, id_action="0", ln=CFG_SITE_LANG):
"""show the details of an action. """
return wal.perform_showactiondetails(req=req,
id_action=id_action)
def showuserdetails(req, id_user="0", ln=CFG_SITE_LANG):
"""show the details of an action. """
return wal.perform_showuserdetails(req=req,
id_user=id_user)
def adduserrole(req, id_role='0', email_user_pattern='', id_user='0', confirm=0, ln=CFG_SITE_LANG):
"""create connection between user and role.
id_role - id of the role to add user to
email_user_pattern - search for users using this pattern
id_user - id of user to add to the role. """
return wal.perform_adduserrole(req=req,
id_role=id_role,
email_user_pattern=email_user_pattern,
id_user=id_user,
confirm=confirm)
def addroleuser(req, email_user_pattern='', id_user='0', id_role='0', confirm=0, ln=CFG_SITE_LANG):
"""create connection between user and role.
email_user_pattern - search for users using this pattern
id_user - id of user to add to the role.
id_role - id of the role to add user to. """
return wal.perform_addroleuser(req=req,
email_user_pattern=email_user_pattern,
id_user=id_user,
id_role=id_role,
confirm=confirm)
def deleteuserrole(req, id_role='0', id_user='0', reverse=0, confirm=0, ln=CFG_SITE_LANG):
"""delete connection between role and user.
id_role - id of role to disconnect
id_user - id of user to disconnect. """
return wal.perform_deleteuserrole(req=req,
id_role=id_role,
id_user=id_user,
reverse=reverse,
confirm=confirm)
def addauthorization(req, id_role="0", id_action="0", reverse="0", confirm=0, **keywords):
""" form to add new connection between user and role:
id_role - role to connect
id_action - action to connect
reverse - role or action first? """
return wal.perform_addauthorization(req=req,
id_role=id_role,
id_action=id_action,
reverse=reverse,
confirm=confirm,
**keywords)
def deleteroleaction(req, id_role="0", id_action="0", reverse=0, confirm=0, ln=CFG_SITE_LANG):
"""delete all connections between a role and an action.
id_role - id of the role
id_action - id of the action
reverse - 0: ask for role first
1: ask for action first"""
return wal.perform_deleteroleaction(req=req,
id_role=id_role,
id_action=id_action,
reverse=reverse,
confirm=confirm)
def modifyauthorizations(req, id_role="0", id_action="0", reverse=0, confirm=0, sel='', errortext='', authids=[], ln=CFG_SITE_LANG):
"""given ids of a role and an action, show all possible action combinations
with checkboxes and allow user to access other functions.
id_role - id of the role
id_action - id of the action
reverse - 0: ask for role first
1: ask for action first
sel - which button and modification that is selected
errortext - text to print when no connection exist between role and action
authids - ids of checked checkboxes """
return wal.perform_modifyauthorizations(req=req,
id_role=id_role,
id_action=id_action,
reverse=reverse,
confirm=confirm,
sel=sel,
authids=authids)
def simpleauthorization(req, id_role=0, id_action=0, ln=CFG_SITE_LANG):
"""show a page with simple overview of authorizations between a
connected role and action. """
return wal.perform_simpleauthorization(req=req,
id_role=id_role,
id_action=id_action)
def showroleusers(req, id_role=0, ln=CFG_SITE_LANG):
"""show a page with simple overview of a role and connected users. """
return wal.perform_showroleusers(req=req,
id_role=id_role)
diff --git a/modules/webjournal/lib/webjournal_utils.py b/modules/webjournal/lib/webjournal_utils.py
index 5fbd2840f..6b16455d3 100644
--- a/modules/webjournal/lib/webjournal_utils.py
+++ b/modules/webjournal/lib/webjournal_utils.py
@@ -1,1771 +1,1774 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Various utilities for WebJournal, e.g. config parser, etc.
"""
import time
import datetime
import calendar
import re
import os
import cPickle
import math
import urllib
from MySQLdb import OperationalError
from xml.dom import minidom
from urlparse import urlparse
from invenio.config import \
CFG_ETCDIR, \
CFG_SITE_URL, \
CFG_CACHEDIR, \
CFG_SITE_LANG, \
CFG_ACCESS_CONTROL_LEVEL_SITE, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_DEVEL_SITE
from invenio.dbquery import run_sql
from invenio.bibformat_engine import BibFormatObject
from invenio.search_engine import search_pattern, record_exists
from invenio.messages import gettext_set_language
from invenio.errorlib import register_exception
+from invenio.urlutils import make_invenio_opener
+
+WEBJOURNAL_OPENER = make_invenio_opener('WebJournal')
########################### REGULAR EXPRESSIONS ######################
header_pattern = re.compile('\s*(?P.*?) \s*')
header_pattern2 = re.compile('(?P.*?)')
para_pattern = re.compile('(?P.+?)', re.DOTALL)
img_pattern = re.compile('\S+?)("|\'|\s).*?/>', re.DOTALL)
image_pattern = re.compile(r'''
(\S*)["']?>)?# get the link location for the image
\s*# after each tag we can have arbitrary whitespaces
# the image is always centered
\s*
\S*)\s*border=1\s*(/)?># getting the image itself
\s*
\s*
()?
(
|
|
)*# the caption can be separated by any nr of line breaks
(
\s*
\s*
(?P.*?) # getting the caption
\s*
\s*
)?''', re.DOTALL | re.VERBOSE | re.IGNORECASE )
#'
############################## FEATURED RECORDS ######################
def get_featured_records(journal_name):
"""
Returns the 'featured' records i.e. records chosen to be displayed
with an image on the main page, in the widgets section, for the
given journal.
parameter:
journal_name - (str) the name of the journal for which we want
to get the featured records
returns:
list of tuples (recid, img_url)
"""
try:
feature_file = open('%s/webjournal/%s/featured_record' % \
(CFG_ETCDIR, journal_name))
except:
return []
records = feature_file.readlines()
return [(record.split('---', 1)[0], record.split('---', 1)[1]) \
for record in records if "---" in record]
def add_featured_record(journal_name, recid, img_url):
"""
Adds the given record to the list of featured records of the given
journal.
parameters:
journal_name - (str) the name of the journal to which the record
should be added.
recid - (int) the record id of the record to be featured.
img_url - (str) a url to an image icon displayed along the
featured record.
returns:
0 if everything went ok
1 if record is already in the list
2 if other problems
"""
# Check that record is not already there
featured_records = get_featured_records(journal_name)
for featured_recid, featured_img in featured_records:
if featured_recid == str(recid):
return 1
try:
fptr = open('%s/webjournal/%s/featured_record'
% (CFG_ETCDIR, journal_name), "a")
fptr.write(str(recid) + '---' + img_url + '\n')
fptr.close()
except:
return 2
return 0
def remove_featured_record(journal_name, recid):
"""
Removes the given record from the list of featured records of the
given journal.
parameters:
journal_name - (str) the name of the journal to which the record
should be added.
recid - (int) the record id of the record to be featured.
"""
featured_records = get_featured_records(journal_name)
try:
fptr = open('%s/webjournal/%s/featured_record'
% (CFG_ETCDIR, journal_name), "w")
for featured_recid, featured_img in featured_records:
if str(featured_recid) != str(recid):
fptr.write(str(featured_recid) + '---' + featured_img + \
'\n')
fptr.close()
except:
return 1
return 0
############################ ARTICLES RELATED ########################
def get_order_dict_from_recid_list(recids, journal_name, issue_number,
newest_first=False,
newest_only=False):
"""
Returns the ordered list of input recids, for given
'issue_number'.
Since there might be several articles at the same position, the
returned structure is a dictionary with keys being order number
indicated in record metadata, and values being list of recids for
this order number (recids for one position are ordered from
highest to lowest recid).
Eg: {'1': [2390, 2386, 2385],
'3': [2388],
'2': [2389],
'4': [2387]}
Parameters:
recids - a list of all recid's that should be brought
into order
journal_name - the name of the journal
issue_number - *str* the issue_number for which we are
deriving the order
newest_first - *bool* if True, new articles should be placed
at beginning of the list. If so, their
position/order will be negative integers
newest_only - *bool* if only new articles should be returned
Returns:
ordered_records: a dictionary with the recids ordered by
keys
"""
ordered_records = {}
ordered_new_records = {}
records_without_defined_order = []
new_records_without_defined_order = []
for record in recids:
temp_rec = BibFormatObject(record)
articles_info = temp_rec.fields('773__')
for article_info in articles_info:
if article_info.get('n', '') == issue_number or \
'0' + article_info.get('n', '') == issue_number:
if article_info.has_key('c') and \
article_info['c'].isdigit():
order_number = int(article_info.get('c', ''))
if (newest_first or newest_only) and \
is_new_article(journal_name, issue_number, record):
if ordered_new_records.has_key(order_number):
ordered_new_records[order_number].append(record)
else:
ordered_new_records[order_number] = [record]
elif not newest_only:
if ordered_records.has_key(order_number):
ordered_records[order_number].append(record)
else:
ordered_records[order_number] = [record]
else:
# No order? No problem! Append it at the end.
if newest_first and is_new_article(journal_name, issue_number, record):
new_records_without_defined_order.append(record)
elif not newest_only:
records_without_defined_order.append(record)
# Append records without order at the end of the list
if records_without_defined_order:
if ordered_records:
ordered_records[max(ordered_records.keys()) + 1] = records_without_defined_order
else:
ordered_records[1] = records_without_defined_order
# Append new records without order at the end of the list of new
# records
if new_records_without_defined_order:
if ordered_new_records:
ordered_new_records[max(ordered_new_records.keys()) + 1] = new_records_without_defined_order
else:
ordered_new_records[1] = new_records_without_defined_order
# Append new records at the beginning of the list of 'old'
# records. To do so, use negative integers
if ordered_new_records:
highest_new_record_order = max(ordered_new_records.keys())
for order, new_records in ordered_new_records.iteritems():
ordered_records[- highest_new_record_order + order - 1] = new_records
for (order, records) in ordered_records.iteritems():
# Reverse so that if there are several articles at same
# positon, newest appear first
records.reverse()
return ordered_records
def get_journal_articles(journal_name, issue, category,
newest_first=False, newest_only=False):
"""
Returns the recids in given category and journal, for given issue
number. The returned recids are grouped according to their 773__c
field.
Example of returned value:
{'1': [2390, 2386, 2385],
'3': [2388],
'2': [2389],
'4': [2387]}
Parameters:
journal_name - *str* the name of the journal (as used in URLs)
issue - *str* the issue. Eg: "08/2007"
category - *str* the name of the category
newest_first - *bool* if True, new articles should be placed
at beginning of the list. If so, their
position/order will be negative integers
newest_only - *bool* if only new articles should be returned
"""
use_cache = True
current_issue = get_current_issue(CFG_SITE_LANG, journal_name)
if issue_is_later_than(issue, current_issue):
# If we are working on unreleased issue, do not use caching
# mechanism
use_cache = False
if use_cache:
cached_articles = _get_cached_journal_articles(journal_name, issue, category)
if cached_articles is not None:
ordered_articles = get_order_dict_from_recid_list(cached_articles,
journal_name,
issue,
newest_first,
newest_only)
return ordered_articles
# Retrieve the list of rules that map Category -> Search Pattern.
# Keep only the rule matching our category
config_strings = get_xml_from_config(["record/rule"], journal_name)
category_to_search_pattern_rules = config_strings["record/rule"]
try:
matching_rule = [rule.split(',', 1) for rule in \
category_to_search_pattern_rules \
if rule.split(',')[0] == category]
except:
return []
recids_issue = search_pattern(p='773__n:%s' % issue)
recids_rule = search_pattern(p=matching_rule[0][1])
if issue[0] == '0':
# search for 09/ and 9/
recids_issue.union_update(search_pattern(p='773__n:%s' % issue.lstrip('0')))
recids_rule.intersection_update(recids_issue)
recids = [recid for recid in recids_rule if record_exists(recid) == 1]
if use_cache:
_cache_journal_articles(journal_name, issue, category, recids)
ordered_articles = get_order_dict_from_recid_list(recids,
journal_name,
issue,
newest_first,
newest_only)
return ordered_articles
def _cache_journal_articles(journal_name, issue, category, articles):
"""
Caches given articles IDs.
"""
journal_cache_path = get_journal_article_cache_path(journal_name,
issue)
try:
journal_cache_file = open(journal_cache_path, 'r')
journal_info = cPickle.load(journal_cache_file)
journal_cache_file.close()
except cPickle.PickleError, e:
journal_info = {}
except IOError:
journal_info = {}
except EOFError:
journal_info = {}
except ValueError:
journal_info = {}
if not journal_info.has_key('journal_articles'):
journal_info['journal_articles'] = {}
journal_info['journal_articles'][category] = articles
# Create cache directory if it does not exist
journal_cache_dir = os.path.dirname(journal_cache_path)
if not os.path.exists(journal_cache_dir):
try:
os.makedirs(journal_cache_dir)
except:
return False
journal_cache_file = open(journal_cache_path, 'w')
cPickle.dump(journal_info, journal_cache_file)
journal_cache_file.close()
return True
def _get_cached_journal_articles(journal_name, issue, category):
"""
Retrieve the articles IDs cached for this journal.
Returns None if cache does not exist or more than 5 minutes old
"""
# Check if our cache is more or less up-to-date (not more than 5
# minutes old)
try:
journal_cache_path = get_journal_article_cache_path(journal_name,
issue)
last_update = os.path.getctime(journal_cache_path)
except Exception, e :
return None
now = time.time()
if (last_update + 5*60) < now:
return None
# Get from cache
try:
journal_cache_file = open(journal_cache_path, 'r')
journal_info = cPickle.load(journal_cache_file)
journal_articles = journal_info.get('journal_articles', {}).get(category, None)
journal_cache_file.close()
except cPickle.PickleError, e:
journal_articles = None
except IOError:
journal_articles = None
except EOFError:
journal_articles = None
except ValueError:
journal_articles = None
return journal_articles
def is_new_article(journal_name, issue, recid):
"""
Check if given article should be considered as new or not.
New articles are articles that have never appeared in older issues
than given one.
"""
article_found_in_older_issue = False
temp_rec = BibFormatObject(recid)
publication_blocks = temp_rec.fields('773__')
for publication_block in publication_blocks:
this_issue_number, this_issue_year = issue.split('/')
issue_number, issue_year = publication_block.get('n', '/').split('/', 1)
if int(issue_year) < int(this_issue_year):
# Found an older issue
article_found_in_older_issue = True
break
elif int(issue_year) == int(this_issue_year) and \
int(issue_number) < int(this_issue_number):
# Found an older issue
article_found_in_older_issue = True
break
return not article_found_in_older_issue
############################ CATEGORIES RELATED ######################
def get_journal_categories(journal_name, issue=None):
"""
List the categories for the given journal and issue.
Returns categories in same order as in config file.
Parameters:
journal_name - *str* the name of the journal (as used in URLs)
issue - *str* the issue. Eg:'08/2007'. If None, consider
all categories defined in journal config
"""
categories = []
current_issue = get_current_issue(CFG_SITE_LANG, journal_name)
config_strings = get_xml_from_config(["record/rule"], journal_name)
all_categories = [rule.split(',')[0] for rule in \
config_strings["record/rule"]]
if issue is None:
return all_categories
for category in all_categories:
recids = get_journal_articles(journal_name,
issue,
category)
if len(recids.keys()) > 0:
categories.append(category)
return categories
def get_category_query(journal_name, category):
"""
Returns the category definition for the given category and journal name
Parameters:
journal_name - *str* the name of the journal (as used in URLs)
categoy - *str* a category name, as found in the XML config
"""
config_strings = get_xml_from_config(["record/rule"], journal_name)
category_to_search_pattern_rules = config_strings["record/rule"]
try:
matching_rule = [rule.split(',', 1)[1].strip() for rule in \
category_to_search_pattern_rules \
if rule.split(',')[0] == category]
except:
return None
return matching_rule[0]
######################### JOURNAL CONFIG VARS ######################
cached_parsed_xml_config = {}
def get_xml_from_config(nodes, journal_name):
"""
Returns values from the journal configuration file.
The needed values can be specified by node name, or by a hierarchy
of nodes names using '/' as character to mean 'descendant of'.
Eg. 'record/rule' to get all the values of 'rule' tags inside the
'record' node
Returns a dictionary with a key for each query and a list of
strings (innerXml) results for each key.
Has a special field "config_fetching_error" that returns an error when
something has gone wrong.
"""
# Get and open the config file
results = {}
if cached_parsed_xml_config.has_key(journal_name):
config_file = cached_parsed_xml_config[journal_name]
else:
config_path = '%s/webjournal/%s/%s-config.xml' % \
(CFG_ETCDIR, journal_name, journal_name)
config_file = minidom.Document
try:
config_file = minidom.parse("%s" % config_path)
except:
# todo: raise exception "error: no config file found"
results["config_fetching_error"] = "could not find config file"
return results
else:
cached_parsed_xml_config[journal_name] = config_file
for node_path in nodes:
node = config_file
for node_path_component in node_path.split('/'):
# pylint: disable=E1103
# The node variable can be rewritten in the loop and therefore
# its type can change.
if node != config_file and node.length > 0:
# We have a NodeList object: consider only first child
node = node.item(0)
# pylint: enable=E1103
try:
node = node.getElementsByTagName(node_path_component)
except:
# WARNING, config did not have such value
node = []
break
results[node_path] = []
for result in node:
try:
result_string = result.firstChild.toxml(encoding="utf-8")
except:
# WARNING, config did not have such value
continue
results[node_path].append(result_string)
return results
def get_journal_issue_field(journal_name):
"""
Returns the MARC field in which this journal expects to find
the issue number. Read this from the journal config file
Parameters:
journal_name - *str* the name of the journal (as used in URLs)
"""
config_strings = get_xml_from_config(["issue_number"], journal_name)
try:
issue_field = config_strings["issue_number"][0]
except:
issue_field = '773__n'
return issue_field
def get_journal_css_url(journal_name, type='screen'):
"""
Returns URL to this journal's CSS.
Parameters:
journal_name - *str* the name of the journal (as used in URLs)
type - *str* 'screen' or 'print', depending on the kind
of CSS
"""
config_strings = get_xml_from_config([type], journal_name)
css_path = ''
try:
css_path = config_strings["screen"][0]
except Exception:
register_exception(req=None,
suffix="No css file for journal %s. Is this right?" % \
journal_name)
return CFG_SITE_URL + '/' + css_path
def get_journal_submission_params(journal_name):
"""
Returns the (doctype, identifier element, identifier field) for
the submission of articles in this journal, so that it is possible
to build direct submission links.
Parameter:
journal_name - *str* the name of the journal (as used in URLs)
"""
doctype = ''
identifier_field = ''
identifier_element = ''
config_strings = get_xml_from_config(["submission/doctype"], journal_name)
if config_strings.get('submission/doctype', ''):
doctype = config_strings['submission/doctype'][0]
config_strings = get_xml_from_config(["submission/identifier_element"], journal_name)
if config_strings.get('submission/identifier_element', ''):
identifier_element = config_strings['submission/identifier_element'][0]
config_strings = get_xml_from_config(["submission/identifier_field"], journal_name)
if config_strings.get('submission/identifier_field', ''):
identifier_field = config_strings['submission/identifier_field'][0]
else:
identifier_field = '037__a'
return (doctype, identifier_element, identifier_field)
def get_journal_draft_keyword_to_remove(journal_name):
"""
Returns the keyword that should be removed from the article
metadata in order to move the article from Draft to Ready
"""
config_strings = get_xml_from_config(["draft_keyword"], journal_name)
if config_strings.get('draft_keyword', ''):
return config_strings['draft_keyword'][0]
return ''
def get_journal_alert_sender_email(journal_name):
"""
Returns the email address that should be used as send of the alert
email.
If not specified, use CFG_SITE_SUPPORT_EMAIL
"""
config_strings = get_xml_from_config(["alert_sender"], journal_name)
if config_strings.get('alert_sender', ''):
return config_strings['alert_sender'][0]
return CFG_SITE_SUPPORT_EMAIL
def get_journal_alert_recipient_email(journal_name):
"""
Returns the default email address of the recipients of the email
Return a string of comma-separated emails.
"""
if CFG_DEVEL_SITE:
# To be on the safe side, do not return the default alert recipients.
return ''
config_strings = get_xml_from_config(["alert_recipients"], journal_name)
if config_strings.get('alert_recipients', ''):
return config_strings['alert_recipients'][0]
return ''
def get_journal_collection_to_refresh_on_release(journal_name):
"""
Returns the list of collection to update (WebColl) upon release of
an issue.
"""
from invenio.search_engine import collection_reclist_cache
config_strings = get_xml_from_config(["update_on_release/collection"], journal_name)
return [coll for coll in config_strings.get('update_on_release/collection', []) if \
collection_reclist_cache.cache.has_key(coll)]
def get_journal_index_to_refresh_on_release(journal_name):
"""
Returns the list of indexed to update (BibIndex) upon release of
an issue.
"""
from invenio.bibindex_engine import get_index_id_from_index_name
config_strings = get_xml_from_config(["update_on_release/index"], journal_name)
return [index for index in config_strings.get('update_on_release/index', []) if \
get_index_id_from_index_name(index) != '']
def get_journal_template(template, journal_name, ln=CFG_SITE_LANG):
"""
Returns the journal templates name for the given template type
Raise an exception if template cannot be found.
"""
from invenio.webjournal_config import \
InvenioWebJournalTemplateNotFoundError
config_strings = get_xml_from_config([template], journal_name)
try:
index_page_template = 'webjournal' + os.sep + \
config_strings[template][0]
except:
raise InvenioWebJournalTemplateNotFoundError(ln,
journal_name,
template)
return index_page_template
def get_journal_name_intl(journal_name, ln=CFG_SITE_LANG):
"""
Returns the nice name of the journal, translated if possible
"""
_ = gettext_set_language(ln)
config_strings = get_xml_from_config(["niceName"], journal_name)
if config_strings.get('niceName', ''):
return _(config_strings['niceName'][0])
return ''
def get_journal_languages(journal_name):
"""
Returns the list of languages defined for this journal
"""
config_strings = get_xml_from_config(["languages"], journal_name)
if config_strings.get('languages', ''):
return [ln.strip() for ln in \
config_strings['languages'][0].split(',')]
return []
def get_journal_issue_grouping(journal_name):
"""
Returns the number of issue that are typically released at the
same time.
This is used if every two weeks you release an issue that should
contains issue of next 2 weeks (eg. at week 16, you relase an
issue named '16-17/2009')
This number should help in the admin interface to guess how to
release the next issue (can be overidden by user).
"""
config_strings = get_xml_from_config(["issue_grouping"], journal_name)
if config_strings.get('issue_grouping', ''):
issue_grouping = config_strings['issue_grouping'][0]
if issue_grouping.isdigit() and int(issue_grouping) > 0:
return int(issue_grouping)
return 1
def get_journal_nb_issues_per_year(journal_name):
"""
Returns the default number of issues per year for this journal.
This number should help in the admin interface to guess the next
issue number (can be overidden by user).
"""
config_strings = get_xml_from_config(["issues_per_year"], journal_name)
if config_strings.get('issues_per_year', ''):
issues_per_year = config_strings['issues_per_year'][0]
if issues_per_year.isdigit() and int(issues_per_year) > 0:
return int(issues_per_year)
return 52
def get_journal_preferred_language(journal_name, ln):
"""
Returns the most adequate language to display the journal, given a
language.
"""
languages = get_journal_languages(journal_name)
if ln in languages:
return ln
elif CFG_SITE_LANG in languages:
return CFG_SITE_LANG
elif languages:
return languages
else:
return CFG_SITE_LANG
def get_unreleased_issue_hiding_mode(journal_name):
"""
Returns how unreleased issue should be treated. Can be one of the
following string values:
'future' - only future unreleased issues are hidden. Past
unreleased one can be viewed
'all' - any unreleased issue (past and future) have to be
hidden
- 'none' - no unreleased issue is hidden
"""
config_strings = get_xml_from_config(["hide_unreleased_issues"], journal_name)
if config_strings.get('hide_unreleased_issues', ''):
hide_unreleased_issues = config_strings['hide_unreleased_issues'][0]
if hide_unreleased_issues in ['future', 'all', 'none']:
return hide_unreleased_issues
return 'all'
def get_first_issue_from_config(journal_name):
"""
Returns the first issue as defined from config. This should only
be useful when no issue have been released.
If not specified, returns the issue made of current week number
and year.
"""
config_strings = get_xml_from_config(["first_issue"], journal_name)
if config_strings.has_key('first_issue'):
return config_strings['first_issue'][0]
return time.strftime("%W/%Y", time.localtime())
######################## TIME / ISSUE FUNCTIONS ######################
def get_current_issue(ln, journal_name):
"""
Returns the current issue of a journal as a string.
Current issue is the latest released issue.
"""
journal_id = get_journal_id(journal_name, ln)
try:
current_issue = run_sql("""SELECT issue_number
FROM jrnISSUE
WHERE date_released <= NOW()
AND id_jrnJOURNAL=%s
ORDER BY date_released DESC
LIMIT 1""",
(journal_id,))[0][0]
except:
# start the first journal ever
current_issue = get_first_issue_from_config(journal_name)
run_sql("""INSERT INTO jrnISSUE (id_jrnJOURNAL, issue_number, issue_display)
VALUES(%s, %s, %s)""",
(journal_id,
current_issue,
current_issue))
return current_issue
def get_all_released_issues(journal_name):
"""
Returns the list of released issue, ordered by release date
Note that it only includes the issues that are considered as
released in the DB: it will not for example include articles that
have been imported in the system but not been released
"""
journal_id = get_journal_id(journal_name)
res = run_sql("""SELECT issue_number
FROM jrnISSUE
WHERE id_jrnJOURNAL = %s
AND UNIX_TIMESTAMP(date_released) != 0
ORDER BY date_released DESC""",
(journal_id,))
if res:
return [row[0] for row in res]
else:
return []
def get_next_journal_issues(current_issue_number, journal_name, n=2):
"""
This function suggests the 'n' next issue numbers
"""
number, year = current_issue_number.split('/', 1)
number = int(number)
year = int(year)
number_issues_per_year = get_journal_nb_issues_per_year(journal_name)
next_issues = [make_issue_number(journal_name,
((number - 1 + i) % (number_issues_per_year)) + 1,
year + ((number - 1 + i) / number_issues_per_year)) \
for i in range(1, n + 1)]
return next_issues
def get_grouped_issues(journal_name, issue_number):
"""
Returns all the issues grouped with a given one.
Issues are sorted from the oldest to newest one.
"""
grouped_issues = []
journal_id = get_journal_id(journal_name, CFG_SITE_LANG)
issue_display = get_issue_number_display(issue_number, journal_name)
res = run_sql("""SELECT issue_number
FROM jrnISSUE
WHERE id_jrnJOURNAL=%s AND issue_display=%s""",
(journal_id,
issue_display))
if res:
grouped_issues = [row[0] for row in res]
grouped_issues.sort(compare_issues)
return grouped_issues
def compare_issues(issue1, issue2):
"""
Comparison function for issues.
Returns:
-1 if issue1 is older than issue2
0 if issues are equal
1 if issue1 is newer than issue2
"""
issue1_number, issue1_year = issue1.split('/', 1)
issue2_number, issue2_year = issue2.split('/', 1)
if int(issue1_year) == int(issue2_year):
return cmp(int(issue1_number), int(issue2_number))
else:
return cmp(int(issue1_year), int(issue2_year))
def issue_is_later_than(issue1, issue2):
"""
Returns true if issue1 is later than issue2
"""
issue_number1, issue_year1 = issue1.split('/', 1)
issue_number2, issue_year2 = issue2.split('/', 1)
if int(issue_year1) > int(issue_year2):
return True
elif int(issue_year1) == int(issue_year2):
return int(issue_number1) > int(issue_number2)
else:
return False
def get_issue_number_display(issue_number, journal_name,
ln=CFG_SITE_LANG):
"""
Returns the display string for a given issue number.
"""
journal_id = get_journal_id(journal_name, ln)
issue_display = run_sql("""SELECT issue_display
FROM jrnISSUE
WHERE issue_number=%s
AND id_jrnJOURNAL=%s""",
(issue_number, journal_id))
if issue_display:
return issue_display[0][0]
else:
# Not yet released...
return issue_number
def make_issue_number(journal_name, number, year, for_url_p=False):
"""
Creates a normalized issue number representation with given issue
number (as int or str) and year (as int or str).
Reverse the year and number if for_url_p is True
"""
number_issues_per_year = get_journal_nb_issues_per_year(journal_name)
precision = len(str(number_issues_per_year))
number = int(str(number))
year = int(str(year))
if for_url_p:
return ("%i/%0" + str(precision) + "i") % \
(year, number)
else:
return ("%0" + str(precision) + "i/%i") % \
(number, year)
def get_release_datetime(issue, journal_name, ln=CFG_SITE_LANG):
"""
Gets the date at which an issue was released from the DB.
Returns None if issue has not yet been released.
See issue_to_datetime() to get the *theoretical* release time of an
issue.
"""
journal_id = get_journal_id(journal_name, ln)
try:
release_date = run_sql("""SELECT date_released
FROM jrnISSUE
WHERE issue_number=%s
AND id_jrnJOURNAL=%s""",
(issue, journal_id))[0][0]
except:
return None
if release_date:
return release_date
else:
return None
def get_announcement_datetime(issue, journal_name, ln=CFG_SITE_LANG):
"""
Get the date at which an issue was announced through the alert system.
Return None if not announced
"""
journal_id = get_journal_id(journal_name, ln)
try:
announce_date = run_sql("""SELECT date_announced
FROM jrnISSUE
WHERE issue_number=%s
AND id_jrnJOURNAL=%s""",
(issue, journal_id))[0][0]
except:
return None
if announce_date:
return announce_date
else:
return None
def datetime_to_issue(issue_datetime, journal_name):
"""
Returns the issue corresponding to the given datetime object.
If issue_datetime is too far in the future or in the past, gives
the best possible matching issue, or None, if it does not seem to
exist.
#If issue_datetime is too far in the future, return the latest
#released issue.
#If issue_datetime is too far in the past, return None
Parameters:
issue_datetime - *datetime* date of the issue to be retrieved
journal_name - *str* the name of the journal (as used in URLs)
"""
issue_number = None
journal_id = get_journal_id(journal_name)
# Try to discover how much days an issue is valid
nb_issues_per_year = get_journal_nb_issues_per_year(journal_name)
this_year_number_of_days = 365
if calendar.isleap(issue_datetime.year):
this_year_number_of_days = 366
issue_day_lifetime = math.ceil(float(this_year_number_of_days)/nb_issues_per_year)
res = run_sql("""SELECT issue_number, date_released
FROM jrnISSUE
WHERE date_released < %s
AND id_jrnJOURNAL = %s
ORDER BY date_released DESC LIMIT 1""",
(issue_datetime, journal_id))
if res and res[0][1]:
issue_number = res[0][0]
issue_release_date = res[0][1]
# Check that the result is not too far in the future:
if issue_release_date + datetime.timedelta(issue_day_lifetime) < issue_datetime:
# In principle, the latest issue will no longer be valid
# at that time
return None
else:
# Mmh, are we too far in the past? This can happen in the case
# of articles that have been imported in the system but never
# considered as 'released' in the database. So we should still
# try to approximate/match an issue:
if round(issue_day_lifetime) in [6, 7, 8]:
# Weekly issues. We can use this information to better
# match the issue number
issue_nb = int(issue_datetime.strftime('%W')) # = week number
else:
# Compute the number of days since beginning of year, and
# divide by the lifetime of an issue: we get the
# approximate issue_number
issue_nb = math.ceil((int(issue_datetime.strftime('%j')) / issue_day_lifetime))
issue_number = ("%0" + str(len(str(nb_issues_per_year)))+ "i/%i") % (issue_nb, issue_datetime.year)
# Now check if this issue exists in the system for this
# journal
if not get_journal_categories(journal_name, issue_number):
# This issue did not exist
return None
return issue_number
DAILY = 1
WEEKLY = 2
MONTHLY = 3
def issue_to_datetime(issue_number, journal_name, granularity=None):
"""
Returns the *theoretical* date of release for given issue: useful
if you release on Friday, but the issue date of the journal
should correspond to the next Monday.
This will correspond to the next day/week/month, depending on the
number of issues per year (or the 'granularity' if specified) and
the release time (if close to the end of a period defined by the
granularity, consider next period since release is made a bit in
advance).
See get_release_datetime() for the *real* release time of an issue
THIS FUNCTION SHOULD ONLY BE USED FOR INFORMATIVE DISPLAY PURPOSE,
AS IT GIVES APPROXIMATIVE RESULTS. Do not use it to make decisions.
Parameters:
issue_number - *str* issue number to consider
journal_name - *str* the name of the journal (as used in URLs)
granularity - *int* the granularity to consider
"""
# If we have released, we can use this information. Otherwise we
# have to approximate.
issue_date = get_release_datetime(issue_number, journal_name)
if not issue_date:
# Approximate release date
number, year = issue_number.split('/')
number = int(number)
year = int(year)
nb_issues_per_year = get_journal_nb_issues_per_year(journal_name)
this_year_number_of_days = 365
if calendar.isleap(year):
this_year_number_of_days = 366
issue_day_lifetime = float(this_year_number_of_days)/nb_issues_per_year
# Compute from beginning of the year
issue_date = datetime.datetime(year, 1, 1) + \
datetime.timedelta(days=int(round((number - 1) * issue_day_lifetime)))
# Okay, but if last release is not too far in the past, better
# compute from the release.
current_issue = get_current_issue(CFG_SITE_LANG, journal_name)
current_issue_time = get_release_datetime(current_issue, journal_name)
if current_issue_time.year == issue_date.year:
current_issue_number, current_issue_year = current_issue.split('/')
current_issue_number = int(current_issue_number)
# Compute from last release
issue_date = current_issue_time + \
datetime.timedelta(days=int((number - current_issue_number) * issue_day_lifetime))
# If granularity is not specifed, deduce from config
if granularity is None:
nb_issues_per_year = get_journal_nb_issues_per_year(journal_name)
if nb_issues_per_year > 250:
granularity = DAILY
elif nb_issues_per_year > 40:
granularity = WEEKLY
else:
granularity = MONTHLY
# Now we can adapt the date to match the granularity
if granularity == DAILY:
if issue_date.hour >= 15:
# If released after 3pm, consider it is the issue of the next
# day
issue_date = issue_date + datetime.timedelta(days=1)
elif granularity == WEEKLY:
(year, week_nb, day_nb) = issue_date.isocalendar()
if day_nb > 4:
# If released on Fri, Sat or Sun, consider that it is next
# week's issue.
issue_date = issue_date + datetime.timedelta(weeks=1)
# Get first day of the week
issue_date = issue_date - datetime.timedelta(days=issue_date.weekday())
else:
if issue_date.day > 22:
# If released last week of the month, consider release for
# next month
issue_date = issue_date.replace(month=issue_date.month+1)
date_string = issue_date.strftime("%Y %m 1")
issue_date = datetime.datetime(*(time.strptime(date_string, "%Y %m %d")[0:6]))
return issue_date
def get_number_of_articles_for_issue(issue, journal_name, ln=CFG_SITE_LANG):
"""
Function that returns a dictionary with all categories and number of
articles in each category.
"""
all_articles = {}
categories = get_journal_categories(journal_name, issue)
for category in categories:
all_articles[category] = len(get_journal_articles(journal_name, issue, category))
return all_articles
########################## JOURNAL RELATED ###########################
def get_journal_info_path(journal_name):
"""
Returns the path to the info file of the given journal. The info
file should be used to get information about a journal when database
is not available.
Returns None if path cannot be determined
"""
# We must make sure we don't try to read outside of webjournal
# cache dir
info_path = os.path.abspath("%s/webjournal/%s/info.dat" % \
(CFG_CACHEDIR, journal_name))
if info_path.startswith(CFG_CACHEDIR + '/webjournal/'):
return info_path
else:
return None
def get_journal_article_cache_path(journal_name, issue):
"""
Returns the path to cache file of the articles of a given issue
Returns None if path cannot be determined
"""
# We must make sure we don't try to read outside of webjournal
# cache dir
cache_path = os.path.abspath("%s/webjournal/%s/%s_articles_cache.dat" % \
(CFG_CACHEDIR, journal_name,
issue.replace('/', '_')))
if cache_path.startswith(CFG_CACHEDIR + '/webjournal/'):
return cache_path
else:
return None
def get_journal_id(journal_name, ln=CFG_SITE_LANG):
"""
Get the id for this journal from the DB. If DB is down, try to get
from cache.
"""
journal_id = None
from invenio.webjournal_config import InvenioWebJournalJournalIdNotFoundDBError
if CFG_ACCESS_CONTROL_LEVEL_SITE == 2:
# do not connect to the database as the site is closed for
# maintenance:
journal_info_path = get_journal_info_path(journal_name)
try:
journal_info_file = open(journal_info_path, 'r')
journal_info = cPickle.load(journal_info_file)
journal_id = journal_info.get('journal_id', None)
except cPickle.PickleError, e:
journal_id = None
except IOError:
journal_id = None
except ValueError:
journal_id = None
else:
try:
res = run_sql("SELECT id FROM jrnJOURNAL WHERE name=%s",
(journal_name,))
if len(res) > 0:
journal_id = res[0][0]
except OperationalError, e:
# Cannot connect to database. Try to read from cache
journal_info_path = get_journal_info_path(journal_name)
try:
journal_info_file = open(journal_info_path, 'r')
journal_info = cPickle.load(journal_info_file)
journal_id = journal_info['journal_id']
except cPickle.PickleError, e:
journal_id = None
except IOError:
journal_id = None
except ValueError:
journal_id = None
if journal_id is None:
raise InvenioWebJournalJournalIdNotFoundDBError(ln, journal_name)
return journal_id
def guess_journal_name(ln, journal_name=None):
"""
Tries to take a guess what a user was looking for on the server if
not providing a name for the journal, or if given journal name
does not match case of original journal.
"""
from invenio.webjournal_config import InvenioWebJournalNoJournalOnServerError
from invenio.webjournal_config import InvenioWebJournalNoNameError
journals_id_and_names = get_journals_ids_and_names()
if len(journals_id_and_names) == 0:
raise InvenioWebJournalNoJournalOnServerError(ln)
elif not journal_name and \
journals_id_and_names[0].has_key('journal_name'):
return journals_id_and_names[0]['journal_name']
elif len(journals_id_and_names) > 0:
possible_journal_names = [journal_id_and_name['journal_name'] for journal_id_and_name \
in journals_id_and_names \
if journal_id_and_name.get('journal_name', '').lower() == journal_name.lower()]
if possible_journal_names:
return possible_journal_names[0]
else:
raise InvenioWebJournalNoNameError(ln)
else:
raise InvenioWebJournalNoNameError(ln)
def get_journals_ids_and_names():
"""
Returns the list of existing journals IDs and names. Try to read
from the DB, or from cache if DB is not accessible.
"""
journals = []
if CFG_ACCESS_CONTROL_LEVEL_SITE == 2:
# do not connect to the database as the site is closed for
# maintenance:
files = os.listdir("%s/webjournal" % CFG_CACHEDIR)
info_files = [path + os.sep + 'info.dat' for path in files if \
os.path.isdir(path) and \
os.path.exists(path + os.sep + 'info.dat')]
for info_file in info_files:
try:
journal_info_file = open(info_file, 'r')
journal_info = cPickle.load(journal_info_file)
journal_id = journal_info.get('journal_id', None)
journal_name = journal_info.get('journal_name', None)
current_issue = journal_info.get('current_issue', None)
if journal_id is not None and \
journal_name is not None:
journals.append({'journal_id': journal_id,
'journal_name': journal_name,
'current_issue': current_issue})
except cPickle.PickleError, e:
# Well, can't do anything...
continue
except IOError:
# Well, can't do anything...
continue
except ValueError:
continue
else:
try:
res = run_sql("SELECT id, name FROM jrnJOURNAL ORDER BY id")
for journal_id, journal_name in res:
journals.append({'journal_id': journal_id,
'journal_name': journal_name})
except OperationalError, e:
# Cannot connect to database. Try to read from cache
files = os.listdir("%s/webjournal" % CFG_CACHEDIR)
info_files = [path + os.sep + 'info.dat' for path in files if \
os.path.isdir(path) and \
os.path.exists(path + os.sep + 'info.dat')]
for info_file in info_files:
try:
journal_info_file = open(info_file, 'r')
journal_info = cPickle.load(journal_info_file)
journal_id = journal_info.get('journal_id', None)
journal_name = journal_info.get('journal_name', None)
current_issue = journal_info.get('current_issue', None)
if journal_id is not None and \
journal_name is not None:
journals.append({'journal_id': journal_id,
'journal_name': journal_name,
'current_issue': current_issue})
except cPickle.PickleError, e:
# Well, can't do anything...
continue
except IOError:
# Well, can't do anything...
continue
except ValueError:
continue
return journals
def parse_url_string(uri):
"""
Centralized function to parse any url string given in
webjournal. Useful to retrieve current category, journal,
etc. from within format elements
The webjournal interface handler should already have cleaned the
URI beforehand, so that journal name exist, issue number is
correct, etc. The only remaining problem might be due to the
capitalization of journal name in contact, search and popup pages,
so clean the journal name. Note that language is also as returned
from the URL, which might need to be filtered to match available
languages (WebJournal elements can rely in bfo.lang to retrieve
washed language)
returns:
args: all arguments in dict form
"""
args = {'journal_name' : '',
'issue_year' : '',
'issue_number' : None,
'issue' : None,
'category' : '',
'recid' : -1,
'verbose' : 0,
'ln' : CFG_SITE_LANG,
'archive_year' : None,
'archive_search': ''}
if not uri.startswith('/journal'):
# Mmh, incorrect context. Still, keep language if available
url_params = urlparse(uri)[4]
args['ln'] = dict([part.split('=') for part in url_params.split('&') \
if len(part.split('=')) == 2]).get('ln', CFG_SITE_LANG)
return args
# Take everything after journal and before first question mark
splitted_uri = uri.split('journal', 1)
second_part = splitted_uri[1]
splitted_uri = second_part.split('?')
uri_middle_part = splitted_uri[0]
uri_arguments = ''
if len(splitted_uri) > 1:
uri_arguments = splitted_uri[1]
arg_list = uri_arguments.split("&")
args['ln'] = CFG_SITE_LANG
args['verbose'] = 0
for arg_pair in arg_list:
arg_and_value = arg_pair.split('=')
if len(arg_and_value) == 2:
if arg_and_value[0] == 'ln':
args['ln'] = arg_and_value[1]
elif arg_and_value[0] == 'verbose' and \
arg_and_value[1].isdigit():
args['verbose'] = int(arg_and_value[1])
elif arg_and_value[0] == 'archive_year' and \
arg_and_value[1].isdigit():
args['archive_year'] = int(arg_and_value[1])
elif arg_and_value[0] == 'archive_search':
args['archive_search'] = arg_and_value[1]
elif arg_and_value[0] == 'name':
args['journal_name'] = guess_journal_name(args['ln'],
arg_and_value[1])
arg_list = uri_middle_part.split("/")
if len(arg_list) > 1 and arg_list[1] not in ['search', 'contact', 'popup']:
args['journal_name'] = urllib.unquote(arg_list[1])
elif arg_list[1] not in ['search', 'contact', 'popup']:
args['journal_name'] = guess_journal_name(args['ln'],
args['journal_name'])
cur_issue = get_current_issue(args['ln'], args['journal_name'])
if len(arg_list) > 2:
try:
args['issue_year'] = int(urllib.unquote(arg_list[2]))
except:
args['issue_year'] = int(cur_issue.split('/')[1])
else:
args['issue'] = cur_issue
args['issue_year'] = int(cur_issue.split('/')[1])
args['issue_number'] = int(cur_issue.split('/')[0])
if len(arg_list) > 3:
try:
args['issue_number'] = int(urllib.unquote(arg_list[3]))
except:
args['issue_number'] = int(cur_issue.split('/')[0])
args['issue'] = make_issue_number(args['journal_name'],
args['issue_number'],
args['issue_year'])
if len(arg_list) > 4:
args['category'] = urllib.unquote(arg_list[4])
if len(arg_list) > 5:
try:
args['recid'] = int(urllib.unquote(arg_list[5]))
except:
pass
args['ln'] = get_journal_preferred_language(args['journal_name'],
args['ln'])
# FIXME : wash arguments?
return args
def make_journal_url(current_uri, custom_parameters=None):
"""
Create a URL, using the current URI and overriding values
with the given custom_parameters
Parameters:
current_uri - *str* the current full URI
custom_parameters - *dict* a dictionary of parameters that
should override those of curent_uri
"""
if not custom_parameters:
custom_parameters = {}
default_params = parse_url_string(current_uri)
for key, value in custom_parameters.iteritems():
# Override default params with custom params
default_params[key] = str(value)
uri = CFG_SITE_URL + '/journal/'
if default_params['journal_name']:
uri += urllib.quote(default_params['journal_name']) + '/'
if default_params['issue_year'] and default_params['issue_number']:
uri += make_issue_number(default_params['journal_name'],
default_params['issue_number'],
default_params['issue_year'],
for_url_p=True) + '/'
if default_params['category']:
uri += urllib.quote(default_params['category'])
if default_params['recid'] and \
default_params['recid'] != -1:
uri += '/' + str(default_params['recid'])
printed_question_mark = False
if default_params['ln']:
uri += '?ln=' + default_params['ln']
printed_question_mark = True
if default_params['verbose'] != 0:
if printed_question_mark:
uri += '&verbose=' + str(default_params['verbose'])
else:
uri += '?verbose=' + str(default_params['verbose'])
return uri
############################ HTML CACHING FUNCTIONS ############################
def cache_index_page(html, journal_name, category, issue, ln):
"""
Caches the index page main area of a Bulletin
(right hand menu cannot be cached)
"""
issue = issue.replace("/", "_")
category = category.replace(" ", "")
cache_path = os.path.abspath('%s/webjournal/%s/%s_index_%s_%s.html' % \
(CFG_CACHEDIR, journal_name,
issue, category,
ln))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop caching
return False
cache_path_dir = '%s/webjournal/%s' % (CFG_CACHEDIR, journal_name)
if not os.path.isdir(cache_path_dir):
os.makedirs(cache_path_dir)
cached_file = open(cache_path, "w")
cached_file.write(html)
cached_file.close()
def get_index_page_from_cache(journal_name, category, issue, ln):
"""
Function to get an index page from the cache.
False if not in cache.
"""
issue = issue.replace("/", "_")
category = category.replace(" ", "")
cache_path = os.path.abspath('%s/webjournal/%s/%s_index_%s_%s.html' % \
(CFG_CACHEDIR, journal_name,
issue, category, ln))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop reading cache
return False
try:
cached_file = open(cache_path).read()
except:
return False
return cached_file
def cache_article_page(html, journal_name, category, recid, issue, ln):
"""
Caches an article view of a journal.
"""
issue = issue.replace("/", "_")
category = category.replace(" ", "")
cache_path = os.path.abspath('%s/webjournal/%s/%s_article_%s_%s_%s.html' % \
(CFG_CACHEDIR, journal_name,
issue, category, recid, ln))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop caching
return
cache_path_dir = '%s/webjournal/%s' % (CFG_CACHEDIR, journal_name)
if not os.path.isdir(cache_path_dir):
os.makedirs(cache_path_dir)
cached_file = open(cache_path, "w")
cached_file.write(html)
cached_file.close()
def get_article_page_from_cache(journal_name, category, recid, issue, ln):
"""
Gets an article view of a journal from cache.
False if not in cache.
"""
issue = issue.replace("/", "_")
category = category.replace(" ", "")
cache_path = os.path.abspath('%s/webjournal/%s/%s_article_%s_%s_%s.html' % \
(CFG_CACHEDIR, journal_name,
issue, category, recid, ln))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop reading cache
return False
try:
cached_file = open(cache_path).read()
except:
return False
return cached_file
def clear_cache_for_article(journal_name, category, recid, issue):
"""
Resets the cache for an article (e.g. after an article has been
modified)
"""
issue = issue.replace("/", "_")
category = category.replace(" ", "")
cache_path = os.path.abspath('%s/webjournal/%s/' %
(CFG_CACHEDIR, journal_name))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop deleting cache
return False
# try to delete the article cached file
try:
os.remove('%s/webjournal/%s/%s_article_%s_%s_en.html' %
(CFG_CACHEDIR, journal_name, issue, category, recid))
except:
pass
try:
os.remove('%s/webjournal/%s/%s_article_%s_%s_fr.html' %
(CFG_CACHEDIR, journal_name, issue, category, recid))
except:
pass
# delete the index page for the category
try:
os.remove('%s/webjournal/%s/%s_index_%s_en.html'
% (CFG_CACHEDIR, journal_name, issue, category))
except:
pass
try:
os.remove('%s/webjournal/%s/%s_index_%s_fr.html'
% (CFG_CACHEDIR, journal_name, issue, category))
except:
pass
try:
path = get_journal_article_cache_path(journal_name, issue)
os.remove(path)
except:
pass
return True
def clear_cache_for_issue(journal_name, issue):
"""
clears the cache of a whole issue.
"""
issue = issue.replace("/", "_")
cache_path_dir = os.path.abspath('%s/webjournal/%s' % \
(CFG_CACHEDIR, journal_name))
if not cache_path_dir.startswith(CFG_CACHEDIR + '/webjournal'):
# Mmh, not accessing correct path. Stop deleting cache
return False
all_cached_files = os.listdir(cache_path_dir)
non_deleted = []
for cached_file in all_cached_files:
if cached_file.startswith(issue.replace('/', '_')):
try:
os.remove(cache_path_dir + '/' + cached_file)
except:
return False
else:
non_deleted.append(cached_file)
return True
######################### CERN SPECIFIC FUNCTIONS #################
def get_recid_from_legacy_number(issue_number, category, number):
"""
Returns the recid based on the issue number, category and
'number'.
This is used to support URLs using the now deprecated 'number'
argument. The function tries to reproduce the behaviour of the
old way of doing, even keeping some of its 'problems' (so that we
reach the same article as before with a given number)..
Returns the recid as int, or -1 if not found
"""
recids = []
if issue_number[0] == "0":
alternative_issue_number = issue_number[1:]
recids = list(search_pattern(p='65017a:"%s" and 773__n:%s' %
(category, issue_number)))
recids.extend(list(search_pattern(p='65017a:"%s" and 773__n:%s' %
(category, alternative_issue_number))))
else:
recids = list(search_pattern(p='65017:"%s" and 773__n:%s' %
(category, issue_number)))
# Now must order the records and pick the one at index 'number'.
# But we have to take into account that there can be multiple
# records at position 1, and that these additional records should
# be numbered with negative numbers:
# 1, 1, 1, 2, 3 -> 1, -1, -2, 2, 3...
negative_index_records = {}
positive_index_records = {}
# Fill in 'negative_index_records' and 'positive_index_records'
# lists with the following loop
for recid in recids:
bfo = BibFormatObject(recid)
order = [subfield['c'] for subfield in bfo.fields('773__') if \
issue_number in subfield.get('n', '')]
if len(order) > 0:
# If several orders are defined for the same article and
# the same issue, keep the first one
order = order[0]
if order.isdigit():
# Order must be an int. Otherwise skip
order = int(order)
if order == 1 and positive_index_records.has_key(1):
# This is then a negative number for this record
index = (len(negative_index_records.keys()) > 0 and \
min(negative_index_records.keys()) -1) or 0
negative_index_records[index] = recid
else:
# Positive number for this record
if not positive_index_records.has_key(order):
positive_index_records[order] = recid
else:
# We make the assumption that we cannot have
# twice the same position for two
# articles. Previous WebJournal module was not
# clear about that. Just drop this record
# (better than crashing or looping forever..)
pass
recid_to_return = -1
# Ok, we can finally pick the recid corresponding to 'number'
if number <= 0:
negative_indexes = negative_index_records.keys()
negative_indexes.sort()
negative_indexes.reverse()
if len(negative_indexes) > abs(number):
recid_to_return = negative_index_records[negative_indexes[abs(number)]]
else:
if positive_index_records.has_key(number):
recid_to_return = positive_index_records[number]
return recid_to_return
def is_recid_in_released_issue(recid):
"""
Returns True if recid is part of the latest issue of the given
journal.
WARNING: the function does not check that the article does not
belong to the draft collection of the record. This is wanted, in
order to workaround the time needed for a record to go from the
draft collection to the final collection
"""
bfo = BibFormatObject(recid)
journal_name = ''
journal_names = [journal_name for journal_name in bfo.fields('773__t') if journal_name]
if journal_names:
journal_name = journal_names[0]
else:
return False
existing_journal_names = [o['journal_name'] for o in get_journals_ids_and_names()]
if not journal_name in existing_journal_names:
# Try to remove whitespace
journal_name = journal_name.replace(' ', '')
if not journal_name in existing_journal_names:
# Journal name unknown from WebJournal
return False
config_strings = get_xml_from_config(["draft_image_access_policy"], journal_name)
if config_strings['draft_image_access_policy'] and \
config_strings['draft_image_access_policy'][0] != 'allow':
# The journal does not want to optimize access to images
return False
article_issues = bfo.fields('773__n')
current_issue = get_current_issue(CFG_SITE_LANG, journal_name)
for article_issue in article_issues:
# Check each issue until a released one is found
if get_release_datetime(article_issue, journal_name):
# Release date exists, issue has been released
return True
else:
# Unreleased issue. Do we still allow based on journal config?
unreleased_issues_mode = get_unreleased_issue_hiding_mode(journal_name)
if (unreleased_issues_mode == 'none' or \
(unreleased_issues_mode == 'future' and \
not issue_is_later_than(article_issue, current_issue))):
return True
return False
diff --git a/modules/webjournal/lib/webjournaladminlib.py b/modules/webjournal/lib/webjournaladminlib.py
index 477d2540f..b06670702 100644
--- a/modules/webjournal/lib/webjournaladminlib.py
+++ b/modules/webjournal/lib/webjournaladminlib.py
@@ -1,957 +1,957 @@
## This file is part of Invenio.
## Copyright (C) 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable=C0301
"""Invenio WebJournal Administration Interface."""
__revision__ = "$Id$"
import sys
import cPickle
import re
import os
-from urllib2 import urlopen
if sys.hexversion < 0x2040000:
# pylint: disable=W0622
from sets import Set as set
# pylint: enable=W0622
from invenio.errorlib import register_exception
from invenio.config import \
CFG_SITE_URL, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_ETCDIR, \
CFG_CACHEDIR, \
CFG_TMPSHAREDDIR, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_SITE_RECORD
from invenio.messages import gettext_set_language
from invenio.mailutils import send_email
from invenio.access_control_engine import acc_authorize_action
from invenio.webjournal_config import \
InvenioWebJournalJournalIdNotFoundDBError, \
InvenioWebJournalReleaseUpdateError, \
InvenioWebJournalNoJournalOnServerError
from invenio.webjournal_utils import \
get_journals_ids_and_names, \
guess_journal_name, \
get_current_issue, \
get_issue_number_display, \
get_featured_records, \
add_featured_record, \
remove_featured_record, \
clear_cache_for_issue, \
get_next_journal_issues, \
get_release_datetime, \
get_journal_id, \
compare_issues, \
get_journal_info_path, \
get_journal_css_url, \
get_journal_alert_sender_email, \
get_journal_alert_recipient_email, \
get_journal_draft_keyword_to_remove, \
get_journal_categories, \
get_journal_articles, \
get_grouped_issues, \
get_journal_issue_grouping, \
get_journal_languages, \
get_journal_collection_to_refresh_on_release, \
get_journal_index_to_refresh_on_release, \
- issue_is_later_than
+ issue_is_later_than, \
+ WEBJOURNAL_OPENER
from invenio.dbquery import run_sql
from invenio.bibrecord import \
create_record, \
print_rec
from invenio.bibformat import format_record
from invenio.bibtask import task_low_level_submission, bibtask_allocate_sequenceid
from invenio.search_engine import get_all_collections_of_a_record
import invenio.template
wjt = invenio.template.load('webjournal')
def getnavtrail(previous = ''):
"""Get the navtrail"""
navtrail = """Admin Area """ % (CFG_SITE_URL,)
navtrail = navtrail + previous
return navtrail
def perform_index(ln=CFG_SITE_LANG, journal_name=None, action=None, uid=None):
"""
Index page
Lists the journals, and offers options to edit them, delete them
or add new journal.
Parameters:
journal_name - the journal affected by action, if any
action - one of ['', 'askDelete', _('Delete'), _('Cancel')]
ln - language
uid - user id
"""
_ = gettext_set_language(ln)
msg = None
if action == 'askDelete' and journal_name is not None:
msg = '''''' % {'CFG_SITE_URL': CFG_SITE_URL,
'journal_name': journal_name,
'delete': _("Delete"),
'cancel': _("Cancel")}
if action == _("Delete") and journal_name is not None:
# User confirmed and clicked on "Delete" button
remove_journal(journal_name)
journals = get_journals_ids_and_names()
# Only keep journal that user can view or edit
journals = [(journal_info, acc_authorize_action(uid,
'cfgwebjournal',
name=journal_info['journal_name'],
with_editor_rights='yes')[0] == 0) \
for journal_info in journals \
if acc_authorize_action(uid,
'cfgwebjournal',
name=journal_info['journal_name'])[0] == 0]
return wjt.tmpl_admin_index(ln=ln,
journals=journals,
msg=msg)
def perform_administrate(ln=CFG_SITE_LANG, journal_name=None,
as_editor=True):
"""
Administration of a journal
Show the current and next issues/publications, and display links
to more specific administrative pages.
Parameters:
journal_name - the journal to be administrated
ln - language
with_editor_rights - True if can edit configuration. Read-only mode otherwise
"""
if journal_name is None:
try:
journal_name = guess_journal_name(ln)
except InvenioWebJournalNoJournalOnServerError, e:
return e.user_box()
if not can_read_xml_config(journal_name):
return 'Configuration could not be read. Please check that %s/webjournal/%s/%s-config.xml exists and can be read by the server.
' % (CFG_ETCDIR, journal_name, journal_name)
current_issue = get_current_issue(ln, journal_name)
current_publication = get_issue_number_display(current_issue,
journal_name,
ln)
issue_list = get_grouped_issues(journal_name, current_issue)
next_issue_number = get_next_journal_issues(issue_list[-1], journal_name, 1)
return wjt.tmpl_admin_administrate(journal_name,
current_issue,
current_publication,
issue_list,
next_issue_number[0],
ln,
as_editor=as_editor)
def perform_feature_record(journal_name,
recid,
img_url='',
action='',
ln=CFG_SITE_LANG):
"""
Interface to feature a record
Used to list, add and remove featured records of the journal.
Parameters:
journal_name - the journal for which the article is featured
recid - the record affected by 'action'
img_url - the URL to image displayed with given record
(only when action == 'add')
action - One of ['', 'add', 'askremove', _('Remove'), _('Cancel')]
ln - language
"""
_ = gettext_set_language(ln)
if action == 'add':
result = add_featured_record(journal_name, recid, img_url)
if result == 0:
msg ='''Successfully featured
record %(recid)s.
Go to the %(name)s journal to
see the result.''' % {'CFG_SITE_URL': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'name': journal_name,
'recid': recid}
elif result == 1:
msg = '''record %(recid)s is already featured. Choose another one or remove it first.''' % \
{'CFG_SITE_URL': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'recid': recid}
else:
msg = '''Record could not be featured. Check file permission.'''
featured_records = get_featured_records(journal_name)
return wjt.tmpl_admin_feature_record(ln=ln,
journal_name=journal_name,
featured_records=featured_records,
msg=msg)
elif action == 'askremove':
msg = '''''' % \
{'CFG_SITE_URL': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'name': journal_name,
'recid': recid,
'cancel': _("Cancel"),
'remove': _("Remove")}
featured_records = get_featured_records(journal_name)
return wjt.tmpl_admin_feature_record(ln=ln,
journal_name=journal_name,
featured_records=featured_records,
msg=msg)
elif action == _("Remove"):
result = remove_featured_record(journal_name, recid)
msg = '''Record %(recid)s
has been removed.''' % \
{'CFG_SITE_URL': CFG_SITE_URL,
'CFG_SITE_RECORD': CFG_SITE_RECORD,
'recid': recid}
featured_records = get_featured_records(journal_name)
return wjt.tmpl_admin_feature_record(ln=ln,
journal_name=journal_name,
featured_records=featured_records,
msg=msg)
else:
msg = '''Here you can choose which records from the %s should
be featured on the journal webpage.''' % CFG_SITE_NAME
featured_records = get_featured_records(journal_name)
return wjt.tmpl_admin_feature_record(ln=ln,
journal_name=journal_name,
featured_records=featured_records,
msg=msg)
def perform_regenerate_issue(issue,
journal_name,
ln=CFG_SITE_LANG,
confirmed_p=False,
publish_draft_articles_p=False):
"""
Clears the cache for the given issue.
Parameters:
journal_name - the journal for which the cache should
be deleted
issue - the issue for which the cache should be
deleted
ln - language
confirmed_p - if True, regenerate. Else ask confirmation
publish_draft_articles_p - should the remaining draft articles in
the issue be made public?
"""
if not confirmed_p:
# Ask user confirmation about the regeneration
current_issue = get_current_issue(ln, journal_name)
issue_released_p = not issue_is_later_than(issue, current_issue)
return wjt.tmpl_admin_regenerate_confirm(ln,
journal_name,
issue,
issue_released_p)
else:
# Regenerate the issue (clear the cache)
success = clear_cache_for_issue(journal_name,
issue)
if publish_draft_articles_p:
current_issue = get_current_issue(ln, journal_name)
if not issue_is_later_than(issue, current_issue):
# This issue is already released: we can safely publish
# the articles. Otherwise we'll refuse to publish the drafts
move_drafts_articles_to_ready(journal_name, issue)
if success:
return wjt.tmpl_admin_regenerate_success(ln,
journal_name,
issue)
else:
return wjt.tmpl_admin_regenerate_error(ln,
journal_name,
issue)
def perform_request_issue_control(journal_name, issues,
action, ln=CFG_SITE_LANG):
"""
Central logic for issue control.
Regenerates the flat files 'current_issue' and 'issue_group' of
the journal that control which issue is currently active for the
journal.
Parameters:
journal_name - the journal affected by 'action'
issues - list of issues affected by 'action' TODO: check
action - One of ['cfg', _('Add'), _('Refresh'),
_('Publish'), _('Update')]
ln - language
"""
_ = gettext_set_language(ln)
out = ''
if action == "cfg" or action == _("Refresh") or action == _("Add"):
# find out if we are in update or release
current_issue = get_current_issue(ln, journal_name)
grouped_issues = get_grouped_issues(journal_name, current_issue)
if current_issue != grouped_issues[-1]:
# The current issue has "pending updates", i.e. is grouped
# with unreleased issues. Propose to update these issues
next_issue = grouped_issues[grouped_issues.index(current_issue) + 1]
out = wjt.tmpl_admin_update_issue(ln,
journal_name,
next_issue,
current_issue)
else:
# Propose a release
next_issues = get_next_journal_issues(current_issue,
journal_name,
n=get_journal_issue_grouping(journal_name))
if action == _("Refresh"):
next_issues += issues
next_issues = list(set(next_issues))# avoid double entries
elif action == _("Add"):
next_issues += issues
next_issues = list(set(next_issues))# avoid double entries
next_issues.sort(compare_issues)
highest_issue_so_far = next_issues[-1]
one_more_issue = get_next_journal_issues(highest_issue_so_far,
journal_name,
1)
next_issues += one_more_issue
next_issues = list(set(next_issues)) # avoid double entries
else:
# get the next issue numbers to publish
next_issues = get_next_journal_issues(current_issue,
journal_name,
n=get_journal_issue_grouping(journal_name))
next_issues.sort(compare_issues)
out = wjt.tmpl_admin_control_issue(ln,
journal_name,
next_issues)
elif action == _("Publish"):
# Publish the given issues (mark them as current issues)
publish_issues = issues
publish_issues = list(set(publish_issues)) # avoid double entries
publish_issues.sort(compare_issues)
if len(publish_issues) == 0:
# User did not select an issue
current_issue = get_current_issue(ln, journal_name)
next_issues = get_next_journal_issues(current_issue,
journal_name,
n=get_journal_issue_grouping(journal_name))
out = '' + \
_('Please select an issue') + '
'
out += wjt.tmpl_admin_control_issue(ln,
journal_name,
next_issues)
return out
try:
release_journal_issue(publish_issues, journal_name, ln)
except InvenioWebJournalJournalIdNotFoundDBError, e:
register_exception(req=None)
return e.user_box()
out = wjt.tmpl_admin_control_issue_success_msg(ln,
publish_issues,
journal_name)
elif action == _("Update"):
try:
try:
update_issue = issues[0]
except:
raise InvenioWebJournalReleaseUpdateError(ln, journal_name)
except InvenioWebJournalReleaseUpdateError, e:
register_exception(req=None)
return e.user_box()
try:
release_journal_update(update_issue, journal_name, ln)
except InvenioWebJournalJournalIdNotFoundDBError, e:
register_exception(req=None)
return e.user_box()
out = wjt.tmpl_admin_updated_issue_msg(ln,
update_issue,
journal_name)
return out
def perform_request_alert(journal_name, issue,
sent, plain_text, subject, recipients,
html_mail, force, ln=CFG_SITE_LANG):
"""
All the logic for alert emails.
Display a form to edit email/recipients and options to send the
email. Sent in HTML/PlainText or only PlainText if wished so.
Also prevent mistake of sending the alert more than one for a
particular issue.
Parameters:
journal_name - the journal for which the alert is sent
issue - the issue for which the alert is sent
sent - Display interface to edit email if "False"
(string). Else send the email.
plain_text - the text of the mail
subject - the subject of the mail
recipients - the recipients of the mail (string with
comma-separated emails)
html_mail - if 'html', also send email as HTML (copying
from the current issue on the web)
force - if different than "False", the email is sent
even if it has already been sent.
ln - language
"""
# FIXME: more flexible options to choose the language of the alert
languages = get_journal_languages(journal_name)
if languages:
alert_ln = languages[0]
else:
alert_ln = CFG_SITE_LANG
if not get_release_datetime(issue, journal_name, ln):
# Trying to send an alert for an unreleased issue
return wjt.tmpl_admin_alert_unreleased_issue(ln,
journal_name)
if sent == "False":
# Retrieve default message, subject and recipients, and
# display email editor
subject = wjt.tmpl_admin_alert_subject(journal_name,
alert_ln,
issue)
plain_text = wjt.tmpl_admin_alert_plain_text(journal_name,
alert_ln,
issue)
plain_text = plain_text.encode('utf-8')
recipients = get_journal_alert_recipient_email(journal_name)
return wjt.tmpl_admin_alert_interface(ln,
journal_name,
subject,
plain_text,
recipients,
alert_ln)
else:
# User asked to send the mail
if was_alert_sent_for_issue(issue,
journal_name,
ln) != False and force == "False":
# Mmh, email already sent before for this issue. Ask
# confirmation
return wjt.tmpl_admin_alert_was_already_sent(ln,
journal_name,
subject,
plain_text,
recipients,
html_mail,
issue)
html_string = None
if html_mail == "html":
# Also send as HTML: retrieve from current issue
- html_file = urlopen('%s/journal/%s?ln=%s'
+ html_file = WEBJOURNAL_OPENER.open('%s/journal/%s?ln=%s'
% (CFG_SITE_URL, journal_name, alert_ln))
html_string = html_file.read()
html_file.close()
html_string = put_css_in_file(html_string, journal_name)
html_string = insert_journal_link(html_string, journal_name, issue, ln)
sender_email = get_journal_alert_sender_email(journal_name)
send_email(sender_email, recipients, subject, plain_text,
html_string, header='', footer='', html_header='',
html_footer='', charset='utf-8')
update_DB_for_alert(issue, journal_name, ln)
return wjt.tmpl_admin_alert_success_msg(ln,
journal_name)
def perform_request_configure(journal_name, xml_config, action, ln=CFG_SITE_LANG):
"""
Add a new journal or configure the settings of an existing journal.
Parameters:
journal_name - the journal to configure, or name of the new journal
xml_config - the xml configuration of the journal (string)
action - One of ['edit', 'editDone', 'add', 'addDone']
ln - language
"""
msg = None
if action == 'edit':
# Read existing config
if journal_name is not None:
if not can_read_xml_config(journal_name):
return 'Configuration could not be read. Please check that %s/webjournal/%s/%s-config.xml exists and can be read by the server.
' % (CFG_ETCDIR, journal_name, journal_name)
config_path = '%s/webjournal/%s/%s-config.xml' % (CFG_ETCDIR, journal_name, journal_name)
xml_config = file(config_path).read()
else:
# cannot edit unknown journal...
return 'You must specify a journal name'
if action in ['editDone', 'addDone']:
# Save config
if action == 'addDone':
res = add_journal(journal_name, xml_config)
if res == -1:
msg = 'A journal with that name already exists. Please choose another name.'
action = 'add'
elif res == -2:
msg = 'Configuration could not be written (no permission). Please manually copy your config to %s/webjournal/%s/%s-config.xml
' % (CFG_ETCDIR, journal_name, journal_name)
action = 'edit'
elif res == -4:
msg = 'Cache file could not be written (no permission). Please manually create directory %s/webjournal/%s/ and make it writable for your Apache user
' % (CFG_CACHEDIR, journal_name)
action = 'edit'
elif res > 0:
msg = 'Journal successfully added.'
action = 'edit'
else:
msg = 'An error occurred. The journal could not be added'
action = 'edit'
if action == 'add':
# Display a sample config.
xml_config = '''
Atlantis Times
%(CFG_SITE_URL)s
/img/AtlantisTimes.css
/img/AtlantisTimes.css
AtlantisTimes_Index.bft
AtlantisTimes_Detailed.bft
AtlantisTimes_Search.bft
AtlantisTimes_Popup.bft
AtlantisTimes_Contact.bft
News, 980__a:ATLANTISTIMESNEWS or 980__a:ATLANTISTIMESNEWSDRAFT
Science, 980__a:ATLANTISTIMESSCIENCE or 980__a:ATLANTISTIMESSCIENCEDRAFT
Arts, 980__a:ATLANTISTIMESARTS or 980__a:ATLANTISTIMESARTSDRAFT
2
52
all
773__n
773__c
%(CFG_SITE_SUPPORT_EMAIL)s
recipients@atlantis.atl
en,fr
DEMOJRN
DEMOJRN_RN
02/2009
DRAFT
''' % {'CFG_SITE_URL': CFG_SITE_URL,
'CFG_SITE_SUPPORT_EMAIL': CFG_SITE_SUPPORT_EMAIL}
out = wjt.tmpl_admin_configure_journal(ln=ln,
journal_name=journal_name,
xml_config=xml_config,
action=action,
msg=msg)
return out
######################## ADDING/REMOVING JOURNALS ###############################
def add_journal(journal_name, xml_config):
"""
Add a new journal to the DB. Also create the configuration file
Parameters:
journal_name - the name (used in URLs) of the new journal
xml_config - the xml configuration of the journal (string)
Returns:
the id of the journal if successfully added
-1 if could not be added because journal name already exists
-2 if config could not be saved
-3 if could not be added for other reasons
-4 if database cache could not be added
"""
try:
get_journal_id(journal_name)
except InvenioWebJournalJournalIdNotFoundDBError:
# Perfect, journal does not exist
res = run_sql("INSERT INTO jrnJOURNAL (name) VALUES(%s)", (journal_name,))
# Also save xml_config
config_dir = '%s/webjournal/%s/' % (CFG_ETCDIR, journal_name)
try:
if not os.path.exists(config_dir):
os.makedirs(config_dir)
xml_config_file = file(config_dir + journal_name + '-config.xml', 'w')
xml_config_file.write(xml_config)
xml_config_file.close()
except Exception:
res = -2
# And save some info in file in case database is down
journal_info_path = get_journal_info_path(journal_name)
journal_info_dir = os.path.dirname(journal_info_path)
if not os.path.exists(journal_info_dir):
try:
os.makedirs(journal_info_dir)
except Exception:
if res <= 0:
res = -4
journal_info_file = open(journal_info_path, 'w')
cPickle.dump({'journal_id': res,
'journal_name': journal_name,
'current_issue':'01/2000'}, journal_info_file)
return res
return -1
def remove_journal(journal_name):
"""
Remove a journal from the DB. Does not completely remove
everything, in case it was an error from the editor..
Parameters:
journal_name - the journal to remove
Returns:
the id of the journal if successfully removed or
-1 if could not be removed because journal name does not exist or
-2 if could not be removed for other reasons
"""
run_sql("DELETE FROM jrnJOURNAL WHERE name=%s", (journal_name,))
######################## TIME / ISSUE FUNCTIONS ###############################
def release_journal_issue(publish_issues, journal_name, ln=CFG_SITE_LANG):
"""
Releases a new issue.
This sets the current issue in the database to 'publish_issues' for
given 'journal_name'
Parameters:
journal_name - the journal for which we release a new issue
publish_issues - the list of issues that will be considered as
current (there can be several)
ln - language
"""
journal_id = get_journal_id(journal_name, ln)
if len(publish_issues) > 1:
publish_issues.sort(compare_issues)
low_bound = publish_issues[0]
high_bound = publish_issues[-1]
issue_display = '%s-%s/%s' % (low_bound.split("/")[0],
high_bound.split("/")[0],
high_bound.split("/")[1])
# remember convention: if we are going over a new year, take the higher
else:
issue_display = publish_issues[0]
# produce the DB lines
for publish_issue in publish_issues:
move_drafts_articles_to_ready(journal_name, publish_issue)
run_sql("INSERT INTO jrnISSUE (id_jrnJOURNAL, issue_number, issue_display) \
VALUES(%s, %s, %s)", (journal_id,
publish_issue,
issue_display))
# set first issue to published
release_journal_update(publish_issues[0], journal_name, ln)
# update information in file (in case DB is down)
journal_info_path = get_journal_info_path(journal_name)
journal_info_file = open(journal_info_path, 'w')
cPickle.dump({'journal_id': journal_id,
'journal_name': journal_name,
'current_issue': get_current_issue(ln, journal_name)},
journal_info_file)
def delete_journal_issue(issue, journal_name, ln=CFG_SITE_LANG):
"""
Deletes an issue from the DB.
(Not currently used)
"""
journal_id = get_journal_id(journal_name, ln)
run_sql("DELETE FROM jrnISSUE WHERE issue_number=%s \
AND id_jrnJOURNAL=%s",(issue, journal_id))
# update information in file (in case DB is down)
journal_info_path = get_journal_info_path(journal_name)
journal_info_file = open(journal_info_path, 'w')
cPickle.dump({'journal_id': journal_id,
'journal_name': journal_name,
'current_issue': get_current_issue(ln, journal_name)},
journal_info_file)
def was_alert_sent_for_issue(issue, journal_name, ln):
"""
Returns False if alert has not already been sent for given journal and
issue, else returns time of last alert, as time tuple
Parameters:
journal_name - the journal for which we want to check last alert
issue - the issue for which we want to check last alert
ln - language
Returns:
time tuple or False. Eg: (2008, 4, 25, 7, 58, 37, 4, 116, -1)
"""
journal_id = get_journal_id(journal_name, ln)
date_announced = run_sql("SELECT date_announced FROM jrnISSUE \
WHERE issue_number=%s \
AND id_jrnJOURNAL=%s", (issue, journal_id))[0][0]
if date_announced == None:
return False
else:
return date_announced.timetuple()
def update_DB_for_alert(issue, journal_name, ln):
"""
Update the 'last sent alert' timestamp for the given journal and
issue.
Parameters:
journal_name - the journal for which we want to update the time
of last alert
issue - the issue for which we want to update the time
of last alert
ln - language
"""
journal_id = get_journal_id(journal_name, ln)
run_sql("UPDATE jrnISSUE set date_announced=NOW() \
WHERE issue_number=%s \
AND id_jrnJOURNAL=%s", (issue,
journal_id))
def release_journal_update(update_issue, journal_name, ln=CFG_SITE_LANG):
"""
Releases an update to a journal.
"""
move_drafts_articles_to_ready(journal_name, update_issue)
journal_id = get_journal_id(journal_name, ln)
run_sql("UPDATE jrnISSUE set date_released=NOW() \
WHERE issue_number=%s \
AND id_jrnJOURNAL=%s", (update_issue,
journal_id))
def move_drafts_articles_to_ready(journal_name, issue):
"""
Move draft articles to their final "collection".
To do so we rely on the convention that an admin-chosen keyword
must be removed from the metadata
"""
protected_datafields = ['100', '245', '246', '520', '590', '700']
keyword_to_remove = get_journal_draft_keyword_to_remove(journal_name)
collections_to_refresh = {}
indexes_to_refresh = get_journal_index_to_refresh_on_release(journal_name)
bibindex_indexes_params = []
if indexes_to_refresh:
bibindex_indexes_params = ['-w', ','.join(indexes_to_refresh)]
categories = get_journal_categories(journal_name, issue)
task_sequence_id = str(bibtask_allocate_sequenceid())
for category in categories:
articles = get_journal_articles(journal_name, issue, category)
for order, recids in articles.iteritems():
for recid in recids:
record_xml = format_record(recid, of='xm')
if not record_xml:
continue
new_record_xml_path = os.path.join(CFG_TMPSHAREDDIR,
'webjournal_publish_' + \
str(recid) + '.xml')
if os.path.exists(new_record_xml_path):
# Do not modify twice
continue
record_struc = create_record(record_xml)
record = record_struc[0]
new_record = update_draft_record_metadata(record,
protected_datafields,
keyword_to_remove)
new_record_xml = print_rec(new_record)
if new_record_xml.find(keyword_to_remove) >= 0:
new_record_xml = new_record_xml.replace(keyword_to_remove, '')
# Write to file
new_record_xml_file = file(new_record_xml_path, 'w')
new_record_xml_file.write(new_record_xml)
new_record_xml_file.close()
# Submit
task_low_level_submission('bibupload',
'WebJournal',
'-c', new_record_xml_path,
'-I', task_sequence_id)
task_low_level_submission('bibindex',
'WebJournal',
'-i', str(recid),
'-I', task_sequence_id,
*bibindex_indexes_params)
for collection in get_all_collections_of_a_record(recid):
collections_to_refresh[collection] = ''
# Refresh collections
collections_to_refresh.update([(c, '') for c in get_journal_collection_to_refresh_on_release(journal_name)])
for collection in collections_to_refresh.keys():
task_low_level_submission('webcoll',
'WebJournal',
'-f', '-P', '2', '-p', '1', '-c', collection,
'-I', task_sequence_id)
def update_draft_record_metadata(record, protected_datafields, keyword_to_remove):
"""
Returns a new record with fields that should be modified in order
for this draft record to be considered as 'ready': keep only
controlfield 001 and non-protected fields that contains the
'keyword_to_remove'
Parameters:
record - a single recored (as BibRecord structure)
protected_datafields - *list* tags that should not be part of the
returned record
keyword_to_remove - *str* keyword that should be considered
when checking if a field should be part of
the returned record.
"""
new_record = {}
for tag, field in record.iteritems():
if tag in protected_datafields:
continue
elif not keyword_to_remove in str(field) and \
not tag == '001':
continue
else:
# Keep
new_record[tag] = field
return new_record
######################## XML CONFIG ###############################
def can_read_xml_config(journal_name):
"""
Check that configuration xml for given journal name is exists and
can be read.
"""
config_path = '%s/webjournal/%s/%s-config.xml' % \
(CFG_ETCDIR, journal_name, journal_name)
try:
file(config_path).read()
except IOError:
return False
return True
######################## EMAIL HELPER FUNCTIONS ###############################
def insert_journal_link(html_string, journal_name, issue, ln):
"""
Insert a warning regarding HTML formatting inside mail client and
link to journal page just after the body of the page.
@param html_string: the HTML newsletter
@param journal_name: the journal name
@param issue: journal issue for which the alert is sent (in the form number/year)
@param ln: language
"""
def replace_body(match_obj):
"Replace body with itself + header message"
header = wjt.tmpl_admin_alert_header_html(journal_name, ln, issue)
return match_obj.group() + header
return re.sub('', replace_body, html_string, 1)
def put_css_in_file(html_message, journal_name):
"""
Retrieve the CSS of the journal and insert/inline it in the
section of the given html_message. (Used for HTML alert emails)
Parameters:
journal_name - the journal name
html_message - the html message (string) in which the CSS
should be inserted
Returns:
the HTML message with its CSS inlined
"""
css_path = get_journal_css_url(journal_name)
if not css_path:
return
- css_file = urlopen(css_path)
+ css_file = WEBJOURNAL_OPENER.open(css_path)
css = css_file.read()
css = make_full_paths_in_css(css, journal_name)
html_parted = html_message.split("")
if len(html_parted) > 1:
html = '%s%s' % (html_parted[0],
css,
html_parted[1])
else:
html_parted = html_message.split("")
if len(html_parted) > 1:
html = '%s%s' % (html_parted[0],
css,
html_parted[1])
else:
return
return html
def make_full_paths_in_css(css, journal_name):
"""
Update the URLs in a CSS from relative to absolute URLs, so that the
URLs are accessible from anywhere (Used for HTML alert emails)
Parameters:
journal_name - the journal name
css - a cascading stylesheet (string)
Returns:
(str) the given css with relative paths converted to absolute paths
"""
url_pattern = re.compile('''url\(["']?\s*(?P\S*)\s*["']?\)''',
re.DOTALL)
url_iter = url_pattern.finditer(css)
rel_to_full_path = {}
for url in url_iter:
url_string = url.group("url")
url_string = url_string.replace('"', "")
url_string = url_string.replace("'", "")
if url_string[:6] != "http://":
rel_to_full_path[url_string] = '"%s/img/webjournal_%s/%s"' % \
(CFG_SITE_URL,
journal_name,
url_string)
for url in rel_to_full_path.keys():
css = css.replace(url, rel_to_full_path[url])
return css
diff --git a/modules/webjournal/lib/widgets/bfe_webjournal_widget_seminars.py b/modules/webjournal/lib/widgets/bfe_webjournal_widget_seminars.py
index e7c4c95d4..7a2a4d040 100644
--- a/modules/webjournal/lib/widgets/bfe_webjournal_widget_seminars.py
+++ b/modules/webjournal/lib/widgets/bfe_webjournal_widget_seminars.py
@@ -1,249 +1,248 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
WebJournal widget - Display Indico seminars
"""
from invenio.config import CFG_CACHEDIR, CFG_SITE_LANG
-from urllib2 import urlopen
from xml.dom import minidom
-from invenio.urlutils import create_Indico_request_url
+from invenio.urlutils import create_Indico_request_url, make_invenio_opener
import time
import base64
import socket
from invenio.webjournal_utils import \
- parse_url_string
+ parse_url_string, WEBJOURNAL_OPENER
from invenio.messages import gettext_set_language
update_frequency = 3600 # in seconds
def format_element(bfo, indico_baseurl="https://indico.cern.ch", indico_what='categ', indico_loc="", indico_id="1l7", indico_key="", indico_sig="", indico_onlypublic='yes', indico_from="today", indico_to='today', indico_credential_path=""):
"""
Display the list of seminar from the given Indico instance
See Indico HTTP Export APIs:
http://indico.cern.ch/ihelp/html/ExportAPI/index.html
@param indico_baseurl: Indico base URL from which to retrieve information
@param indico_what: element to export
@type indico_what: one of the strings: C{categ}, C{event}, C{room}, C{reservation}
@param indico_loc: location of the element(s) specified by ID (only used for some elements)
@param indico_id: ID of the element to be exported
@type indico_id: a string or a list/tuple of strings
@param indico_type: output format
@type indico_type: one of the strings: C{json}, C{jsonp}, C{xml}, C{html}, C{ics}, C{atom}
@param indico_params: parameters of the query. See U{http://indico.cern.ch/ihelp/html/ExportAPI/common.html}
@param indico_key: API key provided for the given Indico instance
@param indico_sig: API secret key (signature) provided for the given Indico instance
@param indico_credential_path: if provided, load 'indico_key' and 'indico_sig' from this path
"""
args = parse_url_string(bfo.user_info['uri'])
journal_name = args["journal_name"]
cached_filename = "webjournal_widget_seminars_%s.xml" % journal_name
out = get_widget_html(bfo, indico_baseurl, indico_what, indico_loc, indico_id,
indico_onlypublic, indico_from, indico_to,
indico_key, indico_sig, indico_credential_path,
cached_filename, bfo.lang)
return out
def escape_values(bfo):
"""
Called by BibFormat in order to check if output of this element
should be escaped.
"""
return 0
def get_widget_html(bfo, indico_baseurl, indico_what, indico_loc, indico_id,
indico_onlypublic, indico_from, indico_to, indico_key,
indico_sig, indico_credential_path,
cached_filename, ln=CFG_SITE_LANG):
"""
Indico seminars of the day service
Gets seminars of the day from CERN Indico every 60 minutes and displays
them in a widget.
"""
_ = gettext_set_language(ln)
try:
seminar_xml = minidom.parse('%s/%s' % (CFG_CACHEDIR, cached_filename))
except:
try:
_update_seminars(indico_baseurl, indico_what, indico_loc, indico_id,
indico_onlypublic, indico_from, indico_to, indico_key,
indico_sig, indico_credential_path, cached_filename)
seminar_xml = minidom.parse('%s/%s' % (CFG_CACHEDIR, cached_filename))
except:
return "- " + _("No information available") + "
"
try:
timestamp = seminar_xml.firstChild.getAttribute("time")
except:
timestamp = time.struct_time()
last_update = time.mktime(time.strptime(timestamp,
"%a, %d %b %Y %H:%M:%S %Z"))
now = time.mktime(time.gmtime())
if last_update + update_frequency < now:
try:
_update_seminars(indico_baseurl, indico_what, indico_loc, indico_id,
indico_onlypublic, indico_from, indico_to, indico_key,
indico_sig, indico_credential_path, cached_filename)
seminar_xml = minidom.parse('%s/%s' % (CFG_CACHEDIR, cached_filename))
except:
return "- " + _("No information available") + "
"
html = ""
seminars = seminar_xml.getElementsByTagName("seminar")
if len(seminars) == 0:
return "- " + _("No seminars today") + "
"
for seminar in seminars:
html += ""
try:
seminar_time = seminar.getElementsByTagName("start_time")[0].firstChild.toxml(encoding="utf-8")
except:
seminar_time = ""
try:
category = seminar.getElementsByTagName("category")[0].firstChild.toxml(encoding="utf-8")
except:
category = "Seminar"
html += '%s %s
' % (seminar_time, category)
try:
title = seminar.getElementsByTagName("title")[0].firstChild.toxml(encoding="utf-8")
except:
title = ""
try:
url = seminar.getElementsByTagName("url")[0].firstChild.toxml(encoding="utf-8")
except:
url = "#"
try:
speaker = seminar.getElementsByTagName("speaker")[0].firstChild.toxml(encoding="utf-8")
except:
speaker = ""
if (title != ""):
html += '%s, %s
' % (url, title, speaker)
try:
location = seminar.getElementsByTagName("location")[0].firstChild.toxml(encoding="utf-8") + ' '
except:
location = ""
html += location
try:
room = seminar.getElementsByTagName("room")[0].firstChild.toxml(encoding="utf-8")
except:
room = ""
html += room
html += " "
html = '' + html + '
'
return html
def _update_seminars(indico_baseurl, indico_what, indico_loc, indico_id,
indico_onlypublic, indico_from, indico_to,
indico_key, indico_sig, indico_credential_path,
cached_filename):
"""
helper function that gets the xml data source from CERN Indico and creates
a dedicated xml file in the cache for easy use in the widget.
"""
if indico_credential_path:
indico_key, indico_sig = get_indico_credentials(indico_credential_path)
url = create_Indico_request_url(indico_baseurl,
indico_what,
indico_loc,
indico_id,
'xml',
{'onlypublic': indico_onlypublic,
'from': indico_from,
'to': indico_to},
indico_key, indico_sig)
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(2.0)
try:
try:
- indico_xml = urlopen(url)
+ indico_xml = WEBJOURNAL_OPENER.open(url)
except:
return
finally:
socket.setdefaulttimeout(default_timeout)
xml_file_handler = minidom.parseString(indico_xml.read())
seminar_xml = ['' % time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()), ]
agenda_items = xml_file_handler.getElementsByTagName("conference")
for item in agenda_items:
seminar_xml.extend(["", ])
try:
start_date = item.getElementsByTagName("startDate")[0].firstChild.toxml(encoding="utf-8")
start_time = start_date[11:16]
except:
start_time = ""
seminar_xml.extend(["%s " % start_time, ])
try:
category = item.getElementsByTagName("category")[0].firstChild.toxml(encoding="utf-8")
category = category.split("/")[-1]
category = category.replace("&", "")
category = category.replace("nbsp;", "")
category = category.replace(" ", "")
except:
category = ""
seminar_xml.extend(["%s " % category, ])
try:
title = item.getElementsByTagName("title")[0].firstChild.toxml(encoding="utf-8")
except:
title = ""
seminar_xml.extend(["%s " % title, ])
try:
url = item.getElementsByTagName("url")[0].firstChild.toxml(encoding="utf-8")
except:
url = "#"
seminar_xml.extend(["%s " % url, ])
try:
speaker = item.getElementsByTagName("fullName")[0].firstChild.toxml(encoding="utf-8")
except:
speaker = ""
seminar_xml.extend(["%s " % speaker, ])
try:
room = item.getElementsByTagName("room")[0].firstChild.toxml(encoding="utf-8")
except:
room = ""
seminar_xml.extend(["%s " % room, ])
try:
location = item.getElementsByTagName("location")[0].firstChild.toxml(encoding="utf-8")
except:
location = ""
seminar_xml.extend(["%s " % location, ])
seminar_xml.extend([" ", ])
seminar_xml.extend([" ", ])
# write the created file to cache
fptr = open("%s/%s" % (CFG_CACHEDIR, cached_filename), "w")
fptr.write("\n".join(seminar_xml))
fptr.close()
def get_indico_credentials(path):
"""
Returns the Indico API key and (secret) signature as a tuple
(public_key, private_key).
"""
try:
fd = open(path, "r")
_indico_credentials = fd.read()
fd.close()
except IOError, e:
return ('', '')
return base64.decodestring(_indico_credentials).split('\n', 1)
_ = gettext_set_language('en')
dummy = _("What's on today")
dummy = _("Seminars of the week")
diff --git a/modules/webjournal/lib/widgets/bfe_webjournal_widget_weather.py b/modules/webjournal/lib/widgets/bfe_webjournal_widget_weather.py
index 801d0758d..a62e90225 100644
--- a/modules/webjournal/lib/widgets/bfe_webjournal_widget_weather.py
+++ b/modules/webjournal/lib/widgets/bfe_webjournal_widget_weather.py
@@ -1,225 +1,224 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
WebJournal widget - Display weather forecast
"""
import os
import time
import re
import socket
-from urllib2 import urlopen
try:
# Try to load feedparser. Remember for later if it was installed
# or not. Note that feedparser is slow to load: if we don't load
# it in a 'global' way, it will be loaded for every call to this
# element.
global feedparser
import feedparser
feedparser_available = 1
except ImportError:
feedparser_available = 0
from invenio.config import \
CFG_CACHEDIR, \
CFG_ACCESS_CONTROL_LEVEL_SITE
from invenio.errorlib import register_exception
from invenio.webjournal_utils import \
- parse_url_string
+ parse_url_string, WEBJOURNAL_OPENER
from invenio.messages import gettext_set_language
re_image_pattern = re.compile(r'\S*)"\s*/>',
re.DOTALL | re.IGNORECASE | re.VERBOSE)
yahoo_weather_rss_base_url = 'http://weather.yahooapis.com/forecastrss?w=%(location)s&u=%(degree_unit)s'
def format_element(bfo, location='782041', degree_unit='c' ,
display_weather_icon='false', weather_icon_only='false'):
"""
Display the latest weather forecast from Yahoo Weather
(See http://developer.yahoo.com/weather/)
@param location: Yahoo location code for the forecast
@param degree_unit: Degree unit ('f'=Fahrenheit or 'c'=Celsius)
@param display_weather_icon: if 'true', display weather icon inside the forecasts
@param weather_icon_only: it 'true' display only the wheater icon (without text)
"""
if not feedparser_available:
return ""
args = parse_url_string(bfo.user_info['uri'])
journal_name = args["journal_name"]
cached_filename = "webjournal_widget_weather_%s.rss" % journal_name
expire_time_filename = "webjournal_widget_weather_%s_RSS_expires" % \
journal_name
out = get_widget_html(yahoo_weather_rss_base_url % \
{'location': location, 'degree_unit': degree_unit},
cached_filename,
expire_time_filename,
journal_name)
if weather_icon_only == 'true':
try:
out = '
' % \
re_image_pattern.findall(out)[0][1]
except:
register_exception(req=bfo.req)
out = ''
elif display_weather_icon == 'false':
try:
out = re.sub(re_image_pattern, "", out)
except:
register_exception(req=bfo.req)
out = ''
return out
def escape_values(bfo):
"""
Called by BibFormat in order to check if output of this element
should be escaped.
"""
return 0
def get_widget_html(yahoo_weather_rss, cached_filename, expire_time_filename, journal_name):
"""
weather forecast using Yahoo! Weather service
we check and store the "expires" data from the rss feed to decide when
an update is needed.
there always resides a cached version in cds CFG_CACHEDIR along with a flat
file that indicates the time when the feed expires.
"""
cached_weather_box = _get_weather_from_cache(journal_name)
if cached_weather_box:
return cached_weather_box
# No HTML cache? Then read locally saved feed data, and even
# refresh it from Yahoo if it has expired.
try:
cached_rss_path = os.path.join(CFG_CACHEDIR, cached_filename)
assert(os.path.exists(cached_rss_path))
weather_feed = feedparser.parse(cached_rss_path)
assert(not weather_feed.bozo_exception)
except:
try:
_update_feed(yahoo_weather_rss, cached_filename, expire_time_filename)
weather_feed = feedparser.parse('%s/%s' % \
(CFG_CACHEDIR, cached_filename))
except:
return "- " + _("No information available") + "
"
now_in_gmt = time.gmtime()
try:
expire_time = time.strptime(open(expire_time_filename).read(),
"%a, %d %b %Y %H:%M:%S %Z")
diff = time.mktime(expire_time) - time.mktime(now_in_gmt)
except:
diff = -1
if diff < 0:
try:
_update_feed(yahoo_weather_rss, cached_filename, expire_time_filename)
weather_feed = feedparser.parse('%s/%s' % \
(CFG_CACHEDIR, cached_filename))
except:
return "- " + _("No information available") + "
"
# Construct the HTML. Well, simply take the one provided by
# Yahoo..
html = weather_feed.entries[0]['summary']
cache_weather(html, journal_name)
return html
def _get_weather_from_cache(journal_name):
"""
Try to get the weather information from cache. Return False if
cache does not exist
"""
cache_path = os.path.abspath('%s/webjournal/%s/weather.html' % \
(CFG_CACHEDIR,
journal_name))
if not cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Make sure we are reading from correct directory (you
# know, in case there are '../../' inside journal name..)
return False
try:
last_update = os.path.getctime(cache_path)
except:
return False
now = time.time()
if (last_update + 15*60) < now:
# invalidate after 15 minutes
return False
try:
cached_file = open(cache_path).read()
except:
return False
return cached_file
def cache_weather(html, journal_name):
"""
Caches the weather box for 30 minutes.
"""
if not CFG_ACCESS_CONTROL_LEVEL_SITE == 2:
cache_path = os.path.abspath('%s/webjournal/%s/weather.html' % \
(CFG_CACHEDIR,
journal_name))
if cache_path.startswith(CFG_CACHEDIR + '/webjournal'):
# Do not try to cache if the journal name led us to some
# other directory ('../../' inside journal name for
# example)
cache_dir = CFG_CACHEDIR + '/webjournal/' + journal_name
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
cache_file = file(cache_path, "w")
cache_file.write(html)
cache_file.close()
def _update_feed(yahoo_weather_rss, cached_filename, expire_time_filename):
"""
Retrieve the latest weather information from Yahoo and write it to
'cached_filename'. Also write the supposed expiration date
provided by Yahoo to 'expire_time_filename'.
"""
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(2.0)
try:
try:
- feed = urlopen(yahoo_weather_rss)
+ feed = WEBJOURNAL_OPENER.open(yahoo_weather_rss)
except:
return
finally:
socket.setdefaulttimeout(default_timeout)
cached_file = open('%s/%s' % (CFG_CACHEDIR, cached_filename), 'w')
cached_file.write(feed.read())
cached_file.close()
feed_data = feedparser.parse(yahoo_weather_rss)
expire_time = feed_data.headers['expires']
expire_file = open('%s/%s' % (CFG_CACHEDIR, expire_time_filename), 'w')
expire_file.write(expire_time)
expire_file.close()
_ = gettext_set_language('en')
dummy = _("Under the CERN sky")
diff --git a/modules/websearch/lib/websearch_external_collections_getter.py b/modules/websearch/lib/websearch_external_collections_getter.py
index 38303f328..b158788fb 100644
--- a/modules/websearch/lib/websearch_external_collections_getter.py
+++ b/modules/websearch/lib/websearch_external_collections_getter.py
@@ -1,246 +1,246 @@
# -*- coding: utf-8 -*-
## This file is part of Invenio.
## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""Module to download web pages using asyncore.
Example 1, downloading a set of webpages :
from websearch_external_collections_getter import *
urls = ['http://www.google.fr', 'http://linuxfr.org']
pagegetters = [HTTPAsyncPageGetter(url) for url in urls]
async_download(pagegetters)
for pagegetter in pagegetters:
if pagegetter.done:
print pagegetter.data
else:
print "Error downloading : " + pagegetter.uri
Example 2, downloading a set of webpages but with callback function.
def func(pagegetter, data, current_time):
print "OK (%f): " % current_time + pagegetter.uri + " - " + data
from websearch_external_collections_getter import *
urls = ['http://www.google.fr', 'http://linuxfr.org']
pagegetters = [HTTPAsyncPageGetter(url) for url in urls]
async_download(pagegetters, func, ['info1', 'info2'], 10)
"""
__revision__ = "$Id$"
import asyncore
import mimetools
import socket
import sys
import StringIO
import time
import urlparse
#from invenio.websearch_external_collections_config import CFG_EXTERNAL_COLLECTION_TIMEOUT
from invenio.config import CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_TIMEOUT
CFG_EXTERNAL_COLLECTION_TIMEOUT = CFG_WEBSEARCH_EXTERNAL_COLLECTION_SEARCH_TIMEOUT
def async_download(pagegetter_list, finish_function=None, datastructure_list=None, timeout=15, print_search_info=True, print_body=True):
"""Download web pages asynchronously with timeout.
pagegetter_list : list of HTTPAsyncPageGetter objects
finish_function : function called when a web page is downloaded;
prototype def funct(pagetter, datastructure, current_time, print_search_info(optional))
datastructure_list : list (same size as pagegetter_list) with information to pass as datastructure
to the finish function.
timeout : float, timeout in seconds.
print_search_info: boolean, whether to print the search info or not in the finish function"""
time_start = time.time()
finished_list = [False] * len(pagegetter_list)
nb_remaining = 0
check_redirected(pagegetter_list)
for pagegetter in pagegetter_list:
if pagegetter and not pagegetter.done:
nb_remaining += 1
while (time.time() - time_start < timeout) and nb_remaining > 0:
if sys.hexversion < 0x2040000:
asyncore.poll(0.01)
else:
asyncore.loop(0.01, True, None, 1)
check_redirected(pagegetter_list)
for i in range(len(pagegetter_list)):
if pagegetter_list[i] and not finished_list[i] and pagegetter_list[i].done:
nb_remaining -= 1
if finish_function:
if datastructure_list:
datastructure = datastructure_list[i]
else:
datastructure = None
current_time = time.time() - time_start
try:
finish_function(pagegetter_list[i], datastructure, current_time, print_search_info, print_body)
except TypeError:
finish_function(pagegetter_list[i], datastructure, current_time)
finished_list[i] = True
return finished_list
class HTTPAsyncPageGetter(asyncore.dispatcher_with_send):
"""Class to download a web page using asyncore."""
def __init__(self, uri):
asyncore.dispatcher_with_send.__init__(self)
self.uri = uri
self.redirected = None
self.status = None
self.header = None
self.done = False
self.data = ""
self.header_data = ""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
- self.request, self.host, self.port = build_request(self.uri)
+ self.request, self.host, self.port = build_rest_request(self.uri)
try:
self.connect((self.host, self.port))
except:
self.done = True
def handle_connect(self):
"""Handle the connection event. By sending the request to the server."""
try:
self.send(self.request)
except socket.error:
# do nothing because self.done is false by default
pass
def handle_expt(self):
"""Handle an exception. Close the socket and put done at True."""
self.close()
self.done = True
def handle_read(self):
"""Handle a read event."""
data = self.recv(1024)
if not self.header:
self.header_data += data
(self.status, self.header, data) = decode_header(self.header_data)
if self.status is not None:
if self.status[1] in ("301", "302"):
self.redirected = self.header["location"]
self.data += data
def handle_close(self):
"""Handle a close event."""
self.done = True
self.close()
def log_info(self, message, type='info'):
"""
Workaround broken asyncore log_info method that tries to print
to stdout.
"""
print >> sys.stderr, "%s: %s" % (type, message)
-def build_request(uri):
+def build_rest_request(uri):
"""Build an http request for a specific url."""
scheme, host, path, params, query, dummy = urlparse.urlparse(uri)
assert scheme == "http", "only supports HTTP requests (uri = " + uri + ")"
host, port = decode_host_port(host)
path = encode_path(path, params, query)
request = "GET %s HTTP/1.0\r\n" % (path) + \
"User-Agent: Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/48 (like Gecko) Safari/48\r\n" + \
"Accept: text/html, image/jpeg, image/png, text/*, image/*, */*\r\n" + \
"Accept-Charset: utf-8, utf-8;q=0.5, *;q=0.5\r\n" + \
"Host: %s\r\n" % (host) + \
"Connection: close\r\n\r\n"
return (request, host, port)
def decode_host_port(host):
"""Decode the host string in an (host, port) pair."""
try:
host, port = host.split(":", 1)
port = int(port)
except (TypeError, ValueError):
port = 80
return (host, port)
def encode_path(path, params, query):
"""Bind the path, the params and the query in a unique http path."""
if not path:
path = "/"
if params:
path = path + ";" + params
if query:
path = path + "?" + query
return path
def decode_header(data):
"""Try to decode an html header.
If the header can be decoded, will return (status, header, remaining_data)
If it cannot, (None, None, data)
"""
i = data.find("\r\n\r\n")
size = 4
if i == -1:
i = data.find("\n\n")
size = 2
if i == -1:
return (None, None, data)
# parse header
header_fp = StringIO.StringIO(data[:i+size])
# status line is "HTTP/version status message"
status = header_fp.readline()
status = status.split(" ", 2)
# followed by a rfc822-style message header
header = mimetools.Message(header_fp)
# followed by a newline, and the payload (if any)
data = data[i+size:]
return (status, header, data)
def check_redirected(pagegetter_list):
"""Check if a redirection occured in the engines_list."""
for i in range(len(pagegetter_list)):
getter = pagegetter_list[i]
if getter and getter.redirected is not None:
if getter.redirected.startswith('http://'):
getter = HTTPAsyncPageGetter(getter.redirected)
else:
getter.done = True
pagegetter_list[i] = getter
def fetch_url_content(urls, timeout=CFG_EXTERNAL_COLLECTION_TIMEOUT):
"""Given a list of urls this function returns a list of their contents
using a optional custom timeout."""
urls_content = []
try:
pagegetters_list = [HTTPAsyncPageGetter(url) for url in urls]
except AssertionError:
return [None] * len(urls)
async_download(pagegetters_list, None, None, timeout)
for i in range(len(pagegetters_list)):
if pagegetters_list[i].done: urls_content.append(pagegetters_list[i].data)
else: urls_content.append(None)
return urls_content
diff --git a/modules/websession/lib/Makefile.am b/modules/websession/lib/Makefile.am
index 990dc792b..4684028a0 100644
--- a/modules/websession/lib/Makefile.am
+++ b/modules/websession/lib/Makefile.am
@@ -1,32 +1,32 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
pylibdir = $(libdir)/python/invenio
pylib_DATA = session.py webuser.py webuser_tests.py \
websession_templates.py websession_webinterface.py \
webgroup.py webgroup_dblayer.py websession_config.py \
webaccount.py websession_regression_tests.py \
webgroup_regression_tests.py webuser_regression_tests.py \
- webgroup_tests.py inveniogc.py \
+ webgroup_tests.py inveniogc.py webuser_config.py \
websession_web_tests.py
noinst_DATA = password_migration_kit.py
EXTRA_DIST = $(pylib_DATA) $(noinst_DATA)
CLEANFILES = *~ *.tmp *.pyc
diff --git a/modules/websession/lib/webaccount.py b/modules/websession/lib/webaccount.py
index ca5b94342..98df64c83 100644
--- a/modules/websession/lib/webaccount.py
+++ b/modules/websession/lib/webaccount.py
@@ -1,426 +1,435 @@
## This file is part of Invenio.
## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__revision__ = "$Id$"
import re
import MySQLdb
import urllib
from invenio.config import \
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS, \
CFG_CERN_SITE, \
CFG_SITE_LANG, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_SITE_ADMIN_EMAIL, \
CFG_SITE_SECURE_URL, \
CFG_VERSION, \
CFG_DATABASE_HOST, \
CFG_DATABASE_NAME, \
CFG_SITE_RECORD
from invenio.access_control_engine import acc_authorize_action
from invenio.access_control_config import CFG_EXTERNAL_AUTHENTICATION, \
SUPERADMINROLE, CFG_EXTERNAL_AUTH_DEFAULT
from invenio.dbquery import run_sql
from invenio.webuser import getUid, get_user_preferences, \
collect_user_info
from invenio.access_control_admin import acc_find_user_role_actions
from invenio.messages import gettext_set_language
from invenio.external_authentication import InvenioWebAccessExternalAuthError
import invenio.template
websession_templates = invenio.template.load('websession')
+from invenio import web_api_key
+
def perform_info(req, ln):
"""Display the main features of CDS personalize"""
uid = getUid(req)
user_info = collect_user_info(req)
return websession_templates.tmpl_account_info(
ln = ln,
uid = uid,
guest = int(user_info['guest']),
CFG_CERN_SITE = CFG_CERN_SITE,
)
def perform_display_external_user_settings(settings, ln):
"""show external user settings which is a dictionary."""
_ = gettext_set_language(ln)
html_settings = ""
print_settings = False
settings_keys = settings.keys()
settings_keys.sort()
for key in settings_keys:
value = settings[key]
if key.startswith("EXTERNAL_") and not "HIDDEN_" in key:
print_settings = True
key = key[9:].capitalize()
html_settings += websession_templates.tmpl_external_setting(ln, key, value)
return print_settings and websession_templates.tmpl_external_user_settings(ln, html_settings) or ""
def perform_youradminactivities(user_info, ln):
"""Return text for the `Your Admin Activities' box. Analyze
whether user UID has some admin roles, and if yes, then print
suitable links for the actions he can do. If he's not admin,
print a simple non-authorized message."""
your_role_actions = acc_find_user_role_actions(user_info)
your_roles = []
your_admin_activities = []
guest = int(user_info['guest'])
for (role, action) in your_role_actions:
if role not in your_roles:
your_roles.append(role)
if action not in your_admin_activities:
your_admin_activities.append(action)
if SUPERADMINROLE in your_roles:
for action in ("runbibedit", "cfgbibformat", "cfgbibharvest", "cfgoairepository", "cfgbibrank", "cfgbibindex", "cfgwebaccess", "cfgwebcomment", "cfgwebsearch", "cfgwebsubmiit", "cfgbibknowledge", "runbatchuploader"):
if action not in your_admin_activities:
your_admin_activities.append(action)
return websession_templates.tmpl_account_adminactivities(
ln = ln,
uid = user_info['uid'],
guest = guest,
roles = your_roles,
activities = your_admin_activities,
)
def perform_display_account(req, username, bask, aler, sear, msgs, loan, grps, sbms, appr, admn, ln):
"""Display a dynamic page that shows the user's account."""
# load the right message language
_ = gettext_set_language(ln)
uid = getUid(req)
user_info = collect_user_info(req)
#your account
if int(user_info['guest']):
user = "guest"
login = "%s/youraccount/login?ln=%s" % (CFG_SITE_SECURE_URL, ln)
accBody = _("You are logged in as guest. You may want to %(x_url_open)slogin%(x_url_close)s as a regular user.") %\
{'x_url_open': '',
'x_url_close': ''}
accBody += "
"
bask=aler=msgs= _("The %(x_fmt_open)sguest%(x_fmt_close)s users need to %(x_url_open)sregister%(x_url_close)s first") %\
{'x_fmt_open': '',
'x_fmt_close': '',
'x_url_open': '',
'x_url_close': ''}
sear= _("No queries found")
else:
user = username
accBody = websession_templates.tmpl_account_body(
ln = ln,
user = user,
)
#Display warnings if user is superuser
roles = acc_find_user_role_actions(user_info)
warnings = "0"
for role in roles:
if "superadmin" in role:
warnings = "1"
break
warning_list = superuser_account_warnings()
#check if tickets ok
tickets = (acc_authorize_action(user_info, 'runbibedit')[0] == 0)
return websession_templates.tmpl_account_page(
ln = ln,
warnings = warnings,
warning_list = warning_list,
accBody = accBody,
baskets = bask,
alerts = aler,
searches = sear,
messages = msgs,
loans = loan,
groups = grps,
submissions = sbms,
approvals = appr,
tickets = tickets,
administrative = admn
)
def superuser_account_warnings():
"""Check to see whether admin accounts have default / blank password etc. Returns a list"""
warning_array = []
#Try and connect to the mysql database with the default invenio password
try:
conn = MySQLdb.connect (host = CFG_DATABASE_HOST,
user = "root",
passwd = "my123p$ss",
db = "mysql")
conn.close()
warning_array.append("warning_mysql_password_equal_to_invenio_password")
except:
pass
#Try and connect to the invenio database with the default invenio password
try:
conn = MySQLdb.connect (host = CFG_DATABASE_HOST,
user = "invenio",
passwd = "my123p$ss",
db = CFG_DATABASE_NAME)
conn.close ()
warning_array.append("warning_invenio_password_equal_to_default")
except:
pass
#Check if the admin password is empty
res = run_sql("SELECT password, email from user where nickname = 'admin'")
if res:
res1 = run_sql("SELECT email from user where nickname = 'admin' and password = AES_ENCRYPT(%s,'')", (res[0][1], ))
else:
# no account nick-named `admin' exists; keep on going
res1 = []
for user in res1:
warning_array.append("warning_empty_admin_password")
#Check if the admin email has been changed from the default
if (CFG_SITE_ADMIN_EMAIL == "info@invenio-software.org" or CFG_SITE_SUPPORT_EMAIL == "info@invenio-software.org") and CFG_CERN_SITE == 0:
warning_array.append("warning_site_support_email_equal_to_default")
#Check for a new release of Invenio
try:
find = re.compile('Invenio v[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])? is released')
webFile = urllib.urlopen("http://invenio-software.org/repo/invenio/tree/RELEASE-NOTES")
temp = ""
version = ""
version1 = ""
while 1:
temp = webFile.readline()
match1 = find.match(temp)
try:
version = match1.group()
break
except:
pass
if not temp:
break
webFile.close()
submatch = re.compile('[0-9]+.[0-9]+.[0-9]+(\-rc[0-9])?')
version1 = submatch.search(version)
web_version = version1.group().split(".")
local_version = CFG_VERSION.split(".")
if web_version[0] > local_version[0]:
warning_array.append("note_new_release_available")
elif web_version[0] == local_version[0] and web_version[1] > local_version[1]:
warning_array.append("note_new_release_available")
elif web_version[0] == local_version[0] and web_version[1] == local_version[1] and web_version[2] > local_version[2]:
warning_array.append("note_new_release_available")
except:
warning_array.append("error_cannot_download_release_notes")
return warning_array
def template_account(title, body, ln):
"""It is a template for print each of the options from the user's account."""
return websession_templates.tmpl_account_template(
ln = ln,
title = title,
body = body
)
def warning_guest_user(type, ln=CFG_SITE_LANG):
"""It returns an alert message,showing that the user is a guest user and should log into the system."""
# load the right message language
_ = gettext_set_language(ln)
return websession_templates.tmpl_warning_guest_user(
ln = ln,
type = type,
)
def perform_delete(ln):
"""Delete the account of the user, not implement yet."""
# TODO
return websession_templates.tmpl_account_delete(ln = ln)
def perform_set(email, ln, can_config_bibcatalog = False, verbose = 0):
"""Perform_set(email,password): edit your account parameters, email and
password.
If can_config_bibcatalog is True, show the bibcatalog dialog (if configured).
"""
try:
res = run_sql("SELECT id, nickname FROM user WHERE email=%s", (email,))
uid = res[0][0]
nickname = res[0][1]
except:
uid = 0
nickname = ""
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS_LOCAL = CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS
prefs = get_user_preferences(uid)
if CFG_EXTERNAL_AUTHENTICATION.has_key(prefs['login_method']) and CFG_EXTERNAL_AUTHENTICATION[prefs['login_method']] is not None:
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS_LOCAL = 3
out = websession_templates.tmpl_user_preferences(
ln = ln,
email = email,
email_disabled = (CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS_LOCAL >= 2),
password_disabled = (CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS_LOCAL >= 3),
nickname = nickname,
)
if len(CFG_EXTERNAL_AUTHENTICATION) > 1:
try:
uid = run_sql("SELECT id FROM user where email=%s", (email,))
uid = uid[0][0]
except:
uid = 0
current_login_method = prefs['login_method']
methods = CFG_EXTERNAL_AUTHENTICATION.keys()
# Filtering out methods that don't provide user_exists to check if
# a user exists in the external auth method before letting him/her
# to switch.
for method in methods:
if CFG_EXTERNAL_AUTHENTICATION[method] is not None:
try:
if not CFG_EXTERNAL_AUTHENTICATION[method].user_exists(email):
methods.remove(method)
except (AttributeError, InvenioWebAccessExternalAuthError, NotImplementedError):
methods.remove(method)
methods.sort()
if len(methods) > 1:
out += websession_templates.tmpl_user_external_auth(
ln = ln,
methods = methods,
current = current_login_method,
method_disabled = (CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS >= 4)
)
current_group_records = prefs.get('websearch_group_records', 10)
show_latestbox = prefs.get('websearch_latestbox', True)
show_helpbox = prefs.get('websearch_helpbox', True)
out += websession_templates.tmpl_user_websearch_edit(
ln = ln,
current = current_group_records,
show_latestbox = show_latestbox,
show_helpbox = show_helpbox,
)
preferred_lang = prefs.get('language', ln)
out += websession_templates.tmpl_user_lang_edit(
ln = ln,
preferred_lang = preferred_lang
)
+
+ keys_info = web_api_key.show_web_api_keys(uid=uid)
+ out+=websession_templates.tmpl_user_api_key(
+ ln = ln,
+ keys_info = keys_info
+ )
+
#show this dialog only if the system has been configured to use a ticket system
from invenio.config import CFG_BIBCATALOG_SYSTEM
if CFG_BIBCATALOG_SYSTEM and can_config_bibcatalog:
bibcatalog_username = prefs.get('bibcatalog_username', "")
bibcatalog_password = prefs.get('bibcatalog_password', "")
out += websession_templates.tmpl_user_bibcatalog_auth(bibcatalog_username, \
bibcatalog_password, ln=ln)
if verbose >= 9:
for key, value in prefs.items():
out += "%s:%s
" % (key, value)
out += perform_display_external_user_settings(prefs, ln)
return out
def create_register_page_box(referer='', ln=CFG_SITE_LANG):
"""Register a new account."""
return websession_templates.tmpl_register_page(
referer = referer,
ln = ln,
level = CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS,
)
## create_login_page_box(): ask for the user's email and password, for login into the system
def create_login_page_box(referer='', ln=CFG_SITE_LANG):
# List of referer regexep and message to print
_ = gettext_set_language(ln)
login_referrer2msg = (
(re.compile(r"/search"), "" + _("This collection is restricted. If you think you have right to access it, please authenticate yourself.") + "
"),
(re.compile(r"/%s/\d+/files/.+" % CFG_SITE_RECORD), "" + _("This file is restricted. If you think you have right to access it, please authenticate yourself.") + "
"),
)
msg = ""
for regexp, txt in login_referrer2msg:
if regexp.search(referer):
msg = txt
break
internal = None
for system in CFG_EXTERNAL_AUTHENTICATION.keys():
if CFG_EXTERNAL_AUTHENTICATION[system] is None:
internal = system
break
register_available = CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS <= 1 and internal
## Let's retrieve all the login method that are not dedicated to robots
methods = [method[0] for method in CFG_EXTERNAL_AUTHENTICATION.iteritems() if not method[1] or not method[1].robot_login_method_p()]
methods.sort()
return websession_templates.tmpl_login_form(
ln = ln,
referer = referer,
internal = internal,
register_available = register_available,
methods = methods,
selected_method = CFG_EXTERNAL_AUTH_DEFAULT,
msg = msg,
)
# perform_logout: display the message of not longer authorized,
def perform_logout(req, ln):
return websession_templates.tmpl_account_logout(ln = ln)
#def perform_lost: ask the user for his email, in order to send him the lost password
def perform_lost(ln):
return websession_templates.tmpl_lost_password_form(ln)
#def perform_reset_password: ask the user for a new password to reset the lost one
def perform_reset_password(ln, email, reset_key, msg=''):
return websession_templates.tmpl_reset_password_form(ln, email, reset_key, msg)
# perform_emailSent(email): confirm that the password has been emailed to 'email' address
def perform_emailSent(email, ln):
return websession_templates.tmpl_account_emailSent(ln = ln, email = email)
# peform_emailMessage : display a error message when the email introduced is not correct, and sugest to try again
def perform_emailMessage(eMsg, ln):
return websession_templates.tmpl_account_emailMessage( ln = ln,
msg = eMsg
)
# perform_back(): template for return to a previous page, used for login,register and setting
def perform_back(mess, url, linkname, ln='en'):
return websession_templates.tmpl_back_form(
ln = ln,
message = mess,
url = url,
link = linkname,
)
diff --git a/modules/websession/lib/websession_templates.py b/modules/websession/lib/websession_templates.py
index a9941f46c..19789fa1c 100644
--- a/modules/websession/lib/websession_templates.py
+++ b/modules/websession/lib/websession_templates.py
@@ -1,2474 +1,2568 @@
## This file is part of Invenio.
## Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
__revision__ = "$Id$"
import urllib
import cgi
from invenio.config import \
CFG_CERN_SITE, \
CFG_SITE_LANG, \
CFG_SITE_NAME, \
CFG_SITE_NAME_INTL, \
CFG_SITE_SUPPORT_EMAIL, \
CFG_SITE_SECURE_URL, \
CFG_SITE_URL, \
CFG_WEBSESSION_RESET_PASSWORD_EXPIRE_IN_DAYS, \
CFG_WEBSESSION_ADDRESS_ACTIVATION_EXPIRE_IN_DAYS, \
CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS, \
CFG_WEBSEARCH_MAX_RECORDS_IN_GROUPS, \
CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS, \
CFG_SITE_RECORD
from invenio.access_control_config import CFG_EXTERNAL_AUTH_USING_SSO, \
- CFG_EXTERNAL_AUTH_LOGOUT_SSO
+ CFG_EXTERNAL_AUTH_LOGOUT_SSO, CFG_WEB_API_KEY_STATUS
from invenio.urlutils import make_canonical_urlargd, create_url, create_html_link
from invenio.htmlutils import escape_html, nmtoken_from_string
from invenio.messages import gettext_set_language, language_list_long
from invenio.websession_config import CFG_WEBSESSION_GROUP_JOIN_POLICY
class Template:
def tmpl_back_form(self, ln, message, url, link):
"""
A standard one-message-go-back-link page.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'message' *string* - The message to display
- 'url' *string* - The url to go back to
- 'link' *string* - The link text
"""
out = """
%(message)s
%(link)s
"""% {
'message' : message,
'url' : url,
'link' : link,
'ln' : ln
}
return out
def tmpl_external_setting(self, ln, key, value):
_ = gettext_set_language(ln)
out = """
%s:
%s
""" % (key, value)
return out
def tmpl_external_user_settings(self, ln, html_settings):
_ = gettext_set_language(ln)
out = """
%(external_user_settings)s
%(html_settings)s
%(external_user_groups)s
%(consult_external_groups)s
""" % {
'external_user_settings' : _('External account settings'),
'html_settings' : html_settings,
'consult_external_groups' : _('You can consult the list of your external groups directly in the %(x_url_open)sgroups page%(x_url_close)s.') % {
'x_url_open' : '' % ln,
'x_url_close' : ''
},
'external_user_groups' : _('External user groups'),
}
return out
+ def tmpl_user_api_key(self, ln=CFG_SITE_LANG, keys_info=None):
+ """
+ Displays all the API key that the user owns the user
+
+ Parameters:
+
+ - 'ln' *string* - The language to display the interface in
+ - 'key_info' *tuples* - Contains the tuples with the key data (id, desciption, status)
+
+ """
+
+ # load the right message language
+ _ = gettext_set_language(ln)
+
+ out = """
+
+ %(user_api_key)s
+ """ % {
+ 'user_api_key' : _("API keys")
+ }
+
+ if keys_info and len(keys_info) != 0:
+ out += "%(user_keys)s
" % {'user_keys': _("These are your current API keys")}
+ out += """
+
+ """
+ for key_info in keys_info:
+ out += """
+ %(key_description)s
+ %(key_status)s
+
+
+ %(key_label)s
+
+
+
+
+
+
+ """ % {
+ 'key_description': _("Description: " + cgi.escape(key_info[1])),
+ 'key_status': _("Status: " + key_info[2]),
+ 'key_id': _(key_info[0]),
+ 'index': keys_info.index(key_info),
+ 'key_label': _("API key"),
+ 'remove_key' : _("Delete key"),
+ 'sitesecureurl': CFG_SITE_SECURE_URL,
+ 'input_type': ("submit", "hidden")[key_info[2] == CFG_WEB_API_KEY_STATUS['REVOKED']]
+ }
+ out += "
"
+
+ out += """
+
+ """ % {
+ 'create_new_key' : _("If you want to create a new API key, please enter a description for it"),
+ 'new_key_description_label' : _("Description for the new API key"),
+ 'mandatory' : _("mandatory"),
+ 'note' : _("Note"),
+ 'new_key_description_note': _("The description should be something meaningful for you to recognize the API key"),
+ 'create_new_key_button' : _("Create new key"),
+ 'sitesecureurl': CFG_SITE_SECURE_URL
+ }
+
+ return out
+
def tmpl_user_preferences(self, ln, email, email_disabled, password_disabled, nickname):
"""
Displays a form for the user to change his email/password.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'email' *string* - The email of the user
- 'email_disabled' *boolean* - If the user has the right to edit his email
- 'password_disabled' *boolean* - If the user has the right to edit his password
- 'nickname' *string* - The nickname of the user (empty string if user does not have it)
"""
# load the right message language
_ = gettext_set_language(ln)
out = """
%(edit_params)s
""" % {
'change_user' : _("If you want to change your email or set for the first time your nickname, please set new values in the form below."),
'edit_params' : _("Edit login credentials"),
'nickname_label' : _("Nickname"),
'nickname' : nickname,
'nickname_prefix' : nickname=='' and '
'+_("Example")+':johnd' or '',
'new_email' : _("New email address"),
'mandatory' : _("mandatory"),
'example' : _("Example"),
'note' : _("Note"),
'set_values' : _("Set new values"),
'email' : email,
'email_disabled' : email_disabled and "readonly" or "",
'sitesecureurl': CFG_SITE_SECURE_URL,
'fixed_nickname_note' : _('Since this is considered as a signature for comments and reviews, once set it can not be changed.')
}
if not password_disabled and not CFG_EXTERNAL_AUTH_USING_SSO:
out += """
""" % {
'change_pass' : _("If you want to change your password, please enter the old one and set the new value in the form below."),
'mandatory' : _("mandatory"),
'old_password' : _("Old password"),
'new_password' : _("New password"),
'optional' : _("optional"),
'note' : _("Note"),
'password_note' : _("The password phrase may contain punctuation, spaces, etc."),
'old_password_note' : _("You must fill the old password in order to set a new one."),
'retype_password' : _("Retype password"),
'set_values' : _("Set new password"),
'password_disabled' : password_disabled and "disabled" or "",
'sitesecureurl': CFG_SITE_SECURE_URL,
}
elif not CFG_EXTERNAL_AUTH_USING_SSO and CFG_CERN_SITE:
out += "" + _("""If you are using a lightweight CERN account you can
%(x_url_open)sreset the password%(x_url_close)s.""") % \
{'x_url_open' : \
'' \
% (make_canonical_urlargd({'email': email, 'returnurl' : CFG_SITE_SECURE_URL + '/youraccount/edit' + make_canonical_urlargd({'lang' : ln}, {})}, {})), 'x_url_close' : ''} + "
"
elif CFG_EXTERNAL_AUTH_USING_SSO and CFG_CERN_SITE:
out += "" + _("""You can change or reset your CERN account password by means of the %(x_url_open)sCERN account system%(x_url_close)s.""") % \
{'x_url_open' : '', 'x_url_close' : ''} + "
"
return out
def tmpl_user_bibcatalog_auth(self, bibcatalog_username="", bibcatalog_password="", ln=CFG_SITE_LANG):
"""template for setting username and pw for bibcatalog backend"""
_ = gettext_set_language(ln)
out = """
""" % {
'select_lang' : _('Select desired language of the web interface.'),
'update_settings' : _('Update settings')
}
return out
def tmpl_user_websearch_edit(self, ln, current = 10, show_latestbox = True, show_helpbox = True):
_ = gettext_set_language(ln)
out = """
""" % {
'update_settings' : _("Update settings"),
'select_group_records' : _("Number of search results per page"),
}
return out
def tmpl_user_external_auth(self, ln, methods, current, method_disabled):
"""
Displays a form for the user to change his authentication method.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'methods' *array* - The methods of authentication
- 'method_disabled' *boolean* - If the user has the right to change this
- 'current' *string* - The currently selected method
"""
# load the right message language
_ = gettext_set_language(ln)
out = """
""" % {
'select_method' : _("Select method"),
}
return out
def tmpl_lost_password_form(self, ln):
"""
Displays a form for the user to ask for his password sent by email.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'msg' *string* - Explicative message on top of the form.
"""
# load the right message language
_ = gettext_set_language(ln)
out = "" + _("If you have lost the password for your %(sitename)s %(x_fmt_open)sinternal account%(x_fmt_close)s, then please enter your email address in the following form in order to have a password reset link emailed to you.") % {'x_fmt_open' : '', 'x_fmt_close' : '', 'sitename' : CFG_SITE_NAME_INTL[ln]} + "
"
out += """
""" % {
'ln': ln,
'email' : _("Email address"),
'send' : _("Send password reset link"),
}
if CFG_CERN_SITE:
out += "" + _("If you have been using the %(x_fmt_open)sCERN login system%(x_fmt_close)s, then you can recover your password through the %(x_url_open)sCERN authentication system%(x_url_close)s.") % {'x_fmt_open' : '', 'x_fmt_close' : '', 'x_url_open' : '' \
% make_canonical_urlargd({'lf': 'auth', 'returnURL' : CFG_SITE_SECURE_URL + '/youraccount/login?ln='+ln}, {}), 'x_url_close' : ''} + " "
else:
out += "
" + _("Note that if you have been using an external login system, then we cannot do anything and you have to ask there.") + " "
out += _("Alternatively, you can ask %s to change your login system from external to internal.") % ("""%(email)s""" % { 'email' : CFG_SITE_SUPPORT_EMAIL }) + "
"
return out
def tmpl_account_info(self, ln, uid, guest, CFG_CERN_SITE):
"""
Displays the account information
Parameters:
- 'ln' *string* - The language to display the interface in
- 'uid' *string* - The user id
- 'guest' *boolean* - If the user is guest
- 'CFG_CERN_SITE' *boolean* - If the site is a CERN site
"""
# load the right message language
_ = gettext_set_language(ln)
out = """%(account_offer)s
""" % {
'account_offer' : _("%s offers you the possibility to personalize the interface, to set up your own personal library of documents, or to set up an automatic alert query that would run periodically and would notify you of search results by email.") % CFG_SITE_NAME_INTL[ln],
}
if not guest:
out += """
-
%(your_settings)s
- %(change_account)s
""" % {
'ln' : ln,
'your_settings' : _("Your Settings"),
'change_account' : _("Set or change your account email address or password. Specify your preferences about the look and feel of the interface.")
}
out += """
- %(your_searches)s
- %(search_explain)s
""" % {
'ln' : ln,
'your_searches' : _("Your Searches"),
'search_explain' : _("View all the searches you performed during the last 30 days."),
}
out += """
- %(your_baskets)s
- %(basket_explain)s""" % {
'ln' : ln,
'your_baskets' : _("Your Baskets"),
'basket_explain' : _("With baskets you can define specific collections of items, store interesting records you want to access later or share with others."),
}
if guest and CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS:
out += self.tmpl_warning_guest_user(ln = ln, type = "baskets")
out += """
- %(your_alerts)s
- %(explain_alerts)s""" % {
'ln' : ln,
'your_alerts' : _("Your Alerts"),
'explain_alerts' : _("Subscribe to a search which will be run periodically by our service. The result can be sent to you via Email or stored in one of your baskets."),
}
if guest and CFG_WEBSESSION_DIFFERENTIATE_BETWEEN_GUESTS:
out += self.tmpl_warning_guest_user(type="alerts", ln = ln)
out += "
"
if CFG_CERN_SITE:
out += """
- %(your_loans)s
- %(explain_loans)s
""" % {
'your_loans' : _("Your Loans"),
'explain_loans' : _("Check out book you have on loan, submit borrowing requests, etc. Requires CERN ID."),
'ln': ln,
'CFG_SITE_SECURE_URL': CFG_SITE_SECURE_URL
}
out += """
"""
return out
def tmpl_warning_guest_user(self, ln, type):
"""
Displays a warning message about the specified type
Parameters:
- 'ln' *string* - The language to display the interface in
- 'type' *string* - The type of data that will get lost in case of guest account (for the moment: 'alerts' or 'baskets')
"""
# load the right message language
_ = gettext_set_language(ln)
if (type=='baskets'):
msg = _("You are logged in as a guest user, so your baskets will disappear at the end of the current session.") + ' '
elif (type=='alerts'):
msg = _("You are logged in as a guest user, so your alerts will disappear at the end of the current session.") + ' '
msg += _("If you wish you can %(x_url_open)slogin or register here%(x_url_close)s.") % {'x_url_open': '',
'x_url_close': ''}
return """
%s
""" % msg
def tmpl_account_body(self, ln, user):
"""
Displays the body of the actions of the user
Parameters:
- 'ln' *string* - The language to display the interface in
- 'user' *string* - The username (nickname or email)
"""
# load the right message language
_ = gettext_set_language(ln)
out = _("You are logged in as %(x_user)s. You may want to a) %(x_url1_open)slogout%(x_url1_close)s; b) edit your %(x_url2_open)saccount settings%(x_url2_close)s.") %\
{'x_user': user,
'x_url1_open': '',
'x_url1_close': '',
'x_url2_open': '',
'x_url2_close': '',
}
return out + "
"
def tmpl_account_template(self, title, body, ln, url):
"""
Displays a block of the your account page
Parameters:
- 'ln' *string* - The language to display the interface in
- 'title' *string* - The title of the block
- 'body' *string* - The body of the block
- 'url' *string* - The URL to go to the proper section
"""
out ="""
%s
%s
""" % (url, title, body)
return out
def tmpl_account_page(self, ln, warnings, warning_list, accBody, baskets, alerts, searches, messages, loans, groups, submissions, approvals, tickets, administrative):
"""
Displays the your account page
Parameters:
- 'ln' *string* - The language to display the interface in
- 'accBody' *string* - The body of the heading block
- 'baskets' *string* - The body of the baskets block
- 'alerts' *string* - The body of the alerts block
- 'searches' *string* - The body of the searches block
- 'messages' *string* - The body of the messages block
- 'groups' *string* - The body of the groups block
- 'submissions' *string* - The body of the submission block
- 'approvals' *string* - The body of the approvals block
- 'administrative' *string* - The body of the administrative block
"""
# load the right message language
_ = gettext_set_language(ln)
out = ""
if warnings == "1":
out += self.tmpl_general_warnings(warning_list)
out += self.tmpl_account_template(_("Your Account"), accBody, ln, '/youraccount/edit?ln=%s' % ln)
if messages:
out += self.tmpl_account_template(_("Your Messages"), messages, ln, '/yourmessages/display?ln=%s' % ln)
if loans:
out += self.tmpl_account_template(_("Your Loans"), loans, ln, '/yourloans/display?ln=%s' % ln)
if baskets:
out += self.tmpl_account_template(_("Your Baskets"), baskets, ln, '/yourbaskets/display?ln=%s' % ln)
if alerts:
out += self.tmpl_account_template(_("Your Alert Searches"), alerts, ln, '/youralerts/list?ln=%s' % ln)
if searches:
out += self.tmpl_account_template(_("Your Searches"), searches, ln, '/youralerts/display?ln=%s' % ln)
if groups:
groups_description = _("You can consult the list of %(x_url_open)syour groups%(x_url_close)s you are administering or are a member of.")
groups_description %= {'x_url_open': '',
'x_url_close': ''}
out += self.tmpl_account_template(_("Your Groups"), groups_description, ln, '/yourgroups/display?ln=%s' % ln)
if submissions:
submission_description = _("You can consult the list of %(x_url_open)syour submissions%(x_url_close)s and inquire about their status.")
submission_description %= {'x_url_open': '',
'x_url_close': ''}
out += self.tmpl_account_template(_("Your Submissions"), submission_description, ln, '/yoursubmissions.py?ln=%s' % ln)
if approvals:
approval_description = _("You can consult the list of %(x_url_open)syour approvals%(x_url_close)s with the documents you approved or refereed.")
approval_description %= {'x_url_open': '',
'x_url_close': ''}
out += self.tmpl_account_template(_("Your Approvals"), approval_description, ln, '/yourapprovals.py?ln=%s' % ln)
#check if this user might have tickets
if tickets:
ticket_description = _("You can consult the list of %(x_url_open)syour tickets%(x_url_close)s.")
ticket_description %= {'x_url_open': '',
'x_url_close': ''}
out += self.tmpl_account_template(_("Your Tickets"), ticket_description, ln, '/yourtickets?ln=%s' % ln)
if administrative:
out += self.tmpl_account_template(_("Your Administrative Activities"), administrative, ln, '/admin')
return out
def tmpl_account_emailMessage(self, ln, msg):
"""
Displays a link to retrieve the lost password
Parameters:
- 'ln' *string* - The language to display the interface in
- 'msg' *string* - Explicative message on top of the form.
"""
# load the right message language
_ = gettext_set_language(ln)
out =""
out +="""
%(msg)s %(try_again)s
""" % {
'ln' : ln,
'msg' : msg,
'try_again' : _("Try again")
}
return out
def tmpl_account_reset_password_email_body(self, email, reset_key, ip_address, ln=CFG_SITE_LANG):
"""
The body of the email that sends lost internal account
passwords to users.
"""
_ = gettext_set_language(ln)
out = """
%(intro)s
%(intro2)s
<%(link)s>
%(outro)s
%(outro2)s""" % {
'intro': _("Somebody (possibly you) coming from %(x_ip_address)s "
"has asked\nfor a password reset at %(x_sitename)s\nfor "
"the account \"%(x_email)s\"." % {
'x_sitename' :CFG_SITE_NAME_INTL.get(ln, CFG_SITE_NAME),
'x_email' : email,
'x_ip_address' : ip_address,
}
),
'intro2' : _("If you want to reset the password for this account, please go to:"),
'link' : "%s/youraccount/access%s" %
(CFG_SITE_SECURE_URL, make_canonical_urlargd({
'ln' : ln,
'mailcookie' : reset_key
}, {})),
'outro' : _("in order to confirm the validity of this request."),
'outro2' : _("Please note that this URL will remain valid for about %(days)s days only.") % {'days': CFG_WEBSESSION_RESET_PASSWORD_EXPIRE_IN_DAYS},
}
return out
def tmpl_account_address_activation_email_body(self, email, address_activation_key, ip_address, ln=CFG_SITE_LANG):
"""
The body of the email that sends email address activation cookie
passwords to users.
"""
_ = gettext_set_language(ln)
out = """
%(intro)s
%(intro2)s
<%(link)s>
%(outro)s
%(outro2)s""" % {
'intro': _("Somebody (possibly you) coming from %(x_ip_address)s "
"has asked\nto register a new account at %(x_sitename)s\nfor the "
"email address \"%(x_email)s\"." % {
'x_sitename' :CFG_SITE_NAME_INTL.get(ln, CFG_SITE_NAME),
'x_email' : email,
'x_ip_address' : ip_address,
}
),
'intro2' : _("If you want to complete this account registration, please go to:"),
'link' : "%s/youraccount/access%s" %
(CFG_SITE_SECURE_URL, make_canonical_urlargd({
'ln' : ln,
'mailcookie' : address_activation_key
}, {})),
'outro' : _("in order to confirm the validity of this request."),
'outro2' : _("Please note that this URL will remain valid for about %(days)s days only.") % {'days' : CFG_WEBSESSION_ADDRESS_ACTIVATION_EXPIRE_IN_DAYS},
}
return out
def tmpl_account_emailSent(self, ln, email):
"""
Displays a confirmation message for an email sent
Parameters:
- 'ln' *string* - The language to display the interface in
- 'email' *string* - The email to which the message has been sent
"""
# load the right message language
_ = gettext_set_language(ln)
out =""
out += _("Okay, a password reset link has been emailed to %s.") % email
return out
def tmpl_account_delete(self, ln):
"""
Displays a confirmation message about deleting the account
Parameters:
- 'ln' *string* - The language to display the interface in
"""
# load the right message language
_ = gettext_set_language(ln)
out = "" + _("""Deleting your account""") + '
'
return out
def tmpl_account_logout(self, ln):
"""
Displays a confirmation message about logging out
Parameters:
- 'ln' *string* - The language to display the interface in
"""
# load the right message language
_ = gettext_set_language(ln)
out = _("You are no longer recognized by our system.") + ' '
if CFG_EXTERNAL_AUTH_USING_SSO and CFG_EXTERNAL_AUTH_LOGOUT_SSO:
out += _("""You are still recognized by the centralized
%(x_fmt_open)sSSO%(x_fmt_close)s system. You can
%(x_url_open)slogout from SSO%(x_url_close)s, too.""") % \
{'x_fmt_open' : '', 'x_fmt_close' : '',
'x_url_open' : '' % CFG_EXTERNAL_AUTH_LOGOUT_SSO,
'x_url_close' : ''}
out += '
'
out += _("If you wish you can %(x_url_open)slogin here%(x_url_close)s.") % \
{'x_url_open': '',
'x_url_close': ''}
return out
def tmpl_login_form(self, ln, referer, internal, register_available, methods, selected_method, msg=None):
"""
Displays a login form
Parameters:
- 'ln' *string* - The language to display the interface in
- 'referer' *string* - The referer URL - will be redirected upon after login
- 'internal' *boolean* - If we are producing an internal authentication
- 'register_available' *boolean* - If users can register freely in the system
- 'methods' *array* - The available authentication methods
- 'selected_method' *string* - The default authentication method
- 'msg' *string* - The message to print before the form, if needed
"""
# load the right message language
_ = gettext_set_language(ln)
if msg is "":
out = "%(please_login)s
" % {
'please_login' : cgi.escape(_("If you already have an account, please login using the form below."))
}
if CFG_CERN_SITE:
out += "" + _("If you don't own a CERN account yet, you can register a %(x_url_open)snew CERN lightweight account%(x_url_close)s.") % {'x_url_open' : '', 'x_url_close' : ''} + "
"
else:
if register_available:
out += ""+_("If you don't own an account yet, please %(x_url_open)sregister%(x_url_close)s an internal account.") %\
{'x_url_open': '',
'x_url_close': ''} + "
"
else:
# users cannot register accounts, so advise them
# how to get one, or be silent about register
# facility if account level is more than 4:
if CFG_ACCESS_CONTROL_LEVEL_ACCOUNTS < 5:
out += "" + _("If you don't own an account yet, please contact %s.") % ('%s' % (cgi.escape(CFG_SITE_SUPPORT_EMAIL, True), cgi.escape(CFG_SITE_SUPPORT_EMAIL))) + "
"
else:
out = "%s
" % msg
out += """"""
out += """%(note)s: %(note_text)s
""" % {
'note' : cgi.escape(_("Note")),
'note_text': cgi.escape(_("You can use your nickname or your email address to login."))}
return out
def tmpl_lost_your_password_teaser(self, ln=CFG_SITE_LANG):
"""Displays a short sentence to attract user to the fact that
maybe he lost his password. Used by the registration page.
"""
_ = gettext_set_language(ln)
out = ""
out += """%(maybe_lost_pass)s""" % {
'ln' : ln,
'maybe_lost_pass': ("Maybe you have lost your password?")
}
return out
def tmpl_reset_password_form(self, ln, email, reset_key, msg=''):
"""Display a form to reset the password."""
_ = gettext_set_language(ln)
out = ""
out = "%s
" % _("Your request is valid. Please set the new "
"desired password in the following form.")
if msg:
out += """%s
""" % msg
out += """
""" % {
'ln' : ln,
'reset_key' : reset_key,
'email' : email,
'set_password_for' : _('Set a new password for'),
'type_new_password' : _('Type the new password'),
'type_it_again' : _('Type again the new password'),
'set_new_password' : _('Set the new password')
}
return out
def tmpl_register_page(self, ln, referer, level):
"""
Displays a login form
Parameters:
- 'ln' *string* - The language to display the interface in
- 'referer' *string* - The referer URL - will be redirected upon after login
- 'level' *int* - Login level (0 - all access, 1 - accounts activated, 2+ - no self-registration)
"""
# load the right message language
_ = gettext_set_language(ln)
out = ""
if level <= 1:
out += _("Please enter your email address and desired nickname and password:")
if level == 1:
out += _("It will not be possible to use the account before it has been verified and activated.")
out += """
%(note)s: %(explain_acc)s""" % {
'referer' : cgi.escape(referer),
'ln' : cgi.escape(ln),
'email_address' : _("Email address"),
'nickname' : _("Nickname"),
'password' : _("Password"),
'mandatory' : _("mandatory"),
'optional' : _("optional"),
'example' : _("Example"),
'note' : _("Note"),
'password_contain' : _("The password phrase may contain punctuation, spaces, etc."),
'retype' : _("Retype Password"),
'register' : _("register"),
'explain_acc' : _("Please do not use valuable passwords such as your Unix, AFS or NICE passwords with this service. Your email address will stay strictly confidential and will not be disclosed to any third party. It will be used to identify you for personal services of %s. For example, you may set up an automatic alert search that will look for new preprints and will notify you daily of new arrivals by email.") % CFG_SITE_NAME,
}
else:
# level >=2, so users cannot register accounts
out += "
" + _("It is not possible to create an account yourself. Contact %s if you want an account.") % ('%s' % (CFG_SITE_SUPPORT_EMAIL, CFG_SITE_SUPPORT_EMAIL)) + "
"
return out
def tmpl_account_adminactivities(self, ln, uid, guest, roles, activities):
"""
Displays the admin activities block for this user
Parameters:
- 'ln' *string* - The language to display the interface in
- 'uid' *string* - The used id
- 'guest' *boolean* - If the user is guest
- 'roles' *array* - The current user roles
- 'activities' *array* - The user allowed activities
"""
# load the right message language
_ = gettext_set_language(ln)
out = ""
# guest condition
if guest:
return _("You seem to be a guest user. You have to %(x_url_open)slogin%(x_url_close)s first.") % \
{'x_url_open': '',
'x_url_close': ''}
# no rights condition
if not roles:
return "" + _("You are not authorized to access administrative functions.") + "
"
# displaying form
out += "" + _("You are enabled to the following roles: %(x_role)s.") % {'x_role': ('' + ", ".join(roles) + "")} + '
'
if activities:
# print proposed links:
activities.sort(lambda x, y: cmp(x.lower(), y.lower()))
tmp_out = ''
for action in activities:
if action == "runbibedit":
tmp_out += """
%s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Editor"))
if action == "runbibeditmulti":
tmp_out += """
%s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Multi-Record Editor"))
if action == "runbibcirculation":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Run BibCirculation"))
if action == "runbibmerge":
tmp_out += """
%s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run Record Merger"))
if action == "runbibswordclient":
tmp_out += """
%s""" % (CFG_SITE_URL, CFG_SITE_RECORD, _("Run BibSword Client"))
if action == "runbatchuploader":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Run Batch Uploader"))
if action == "cfgbibformat":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure BibFormat"))
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure BibKnowledge"))
if action == "cfgoaiharvest":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure OAI Harvest"))
if action == "cfgoairepository":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure OAI Repository"))
if action == "cfgbibindex":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure BibIndex"))
if action == "cfgbibrank":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure BibRank"))
if action == "cfgwebaccess":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebAccess"))
if action == "cfgwebcomment":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebComment"))
if action == "cfgweblinkback":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebLinkback"))
if action == "cfgwebjournal":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebJournal"))
if action == "cfgwebsearch":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebSearch"))
if action == "cfgwebsubmit":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure WebSubmit"))
if action == "runbibdocfile":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Run Document File Manager"))
if action == "cfgbibsort":
tmp_out += """
%s""" % (CFG_SITE_URL, ln, _("Configure BibSort"))
if tmp_out:
out += _("Here are some interesting web admin links for you:") + tmp_out
out += "
" + _("For more admin-level activities, see the complete %(x_url_open)sAdmin Area%(x_url_close)s.") %\
{'x_url_open': '',
'x_url_close': ''}
return out
def tmpl_create_userinfobox(self, ln, url_referer, guest, username, submitter, referee, admin, usebaskets, usemessages, usealerts, usegroups, useloans, usestats):
"""
Displays the user block
Parameters:
- 'ln' *string* - The language to display the interface in
- 'url_referer' *string* - URL of the page being displayed
- 'guest' *boolean* - If the user is guest
- 'username' *string* - The username (nickname or email)
- 'submitter' *boolean* - If the user is submitter
- 'referee' *boolean* - If the user is referee
- 'admin' *boolean* - If the user is admin
- 'usebaskets' *boolean* - If baskets are enabled for the user
- 'usemessages' *boolean* - If messages are enabled for the user
- 'usealerts' *boolean* - If alerts are enabled for the user
- 'usegroups' *boolean* - If groups are enabled for the user
- 'useloans' *boolean* - If loans are enabled for the user
- 'usestats' *boolean* - If stats are enabled for the user
@note: with the update of CSS classes (cds.cds ->
invenio.css), the variables useloans etc are not used in
this function, since they are in the menus. But we keep
them in the function signature for backwards
compatibility.
"""
# load the right message language
_ = gettext_set_language(ln)
out = """
""" % CFG_SITE_URL
if guest:
out += """%(guest_msg)s ::
%(login)s""" % {
'sitesecureurl': CFG_SITE_SECURE_URL,
'ln' : ln,
'guest_msg' : _("guest"),
'referer' : url_referer and ('&referer=%s' % urllib.quote(url_referer)) or '',
'login' : _('login')
}
else:
out += """
%(username)s :: """ % {
'sitesecureurl' : CFG_SITE_SECURE_URL,
'ln' : ln,
'username' : username
}
out += """%(logout)s""" % {
'sitesecureurl' : CFG_SITE_SECURE_URL,
'ln' : ln,
'logout' : _("logout"),
}
return out
def tmpl_create_useractivities_menu(self, ln, selected, url_referer, guest, username, submitter, referee, admin, usebaskets, usemessages, usealerts, usegroups, useloans, usestats):
"""
Returns the main navigation menu with actions based on user's
priviledges
@param ln: The language to display the interface in
@type ln: string
@param selected: If the menu is currently selected
@type selected: boolean
@param url_referer: URL of the page being displayed
@type url_referer: string
@param guest: If the user is guest
@type guest: string
@param username: The username (nickname or email)
@type username: string
@param submitter: If the user is submitter
@type submitter: boolean
@param referee: If the user is referee
@type referee: boolean
@param admin: If the user is admin
@type admin: boolean
@param usebaskets: If baskets are enabled for the user
@type usebaskets: boolean
@param usemessages: If messages are enabled for the user
@type usemessages: boolean
@param usealerts: If alerts are enabled for the user
@type usealerts: boolean
@param usegroups: If groups are enabled for the user
@type usegroups: boolean
@param useloans: If loans are enabled for the user
@type useloans: boolean
@param usestats: If stats are enabled for the user
@type usestats: boolean
@return: html menu of the user activities
@rtype: string
"""
# load the right message language
_ = gettext_set_language(ln)
out = ''''
return out
def tmpl_create_adminactivities_menu(self, ln, selected, url_referer, guest, username, submitter, referee, admin, usebaskets, usemessages, usealerts, usegroups, useloans, usestats, activities):
"""
Returns the main navigation menu with actions based on user's
priviledges
@param ln: The language to display the interface in
@type ln: string
@param selected: If the menu is currently selected
@type selected: boolean
@param url_referer: URL of the page being displayed
@type url_referer: string
@param guest: If the user is guest
@type guest: string
@param username: The username (nickname or email)
@type username: string
@param submitter: If the user is submitter
@type submitter: boolean
@param referee: If the user is referee
@type referee: boolean
@param admin: If the user is admin
@type admin: boolean
@param usebaskets: If baskets are enabled for the user
@type usebaskets: boolean
@param usemessages: If messages are enabled for the user
@type usemessages: boolean
@param usealerts: If alerts are enabled for the user
@type usealerts: boolean
@param usegroups: If groups are enabled for the user
@type usegroups: boolean
@param useloans: If loans are enabled for the user
@type useloans: boolean
@param usestats: If stats are enabled for the user
@type usestats: boolean
@param activities: dictionary of admin activities
@rtype activities: dict
@return: html menu of the user activities
@rtype: string
"""
# load the right message language
_ = gettext_set_language(ln)
out = ''
if activities:
out += ''''
return out
def tmpl_warning(self, warnings, ln=CFG_SITE_LANG):
"""
Display len(warnings) warning fields
@param infos: list of strings
@param ln=language
@return: html output
"""
if not((type(warnings) is list) or (type(warnings) is tuple)):
warnings = [warnings]
warningbox = ""
if warnings != []:
warningbox = "\n Warning:\n"
for warning in warnings:
lines = warning.split("\n")
warningbox += " "
for line in lines[0:-1]:
warningbox += line + "
\n"
warningbox += lines[-1] + "
"
warningbox += "
\n"
return warningbox
def tmpl_error(self, error, ln=CFG_SITE_LANG):
"""
Display error
@param error: string
@param ln=language
@return: html output
"""
_ = gettext_set_language(ln)
errorbox = ""
if error != "":
errorbox = "\n Error:\n"
errorbox += " "
errorbox += error + "
"
errorbox += "
\n"
return errorbox
def tmpl_display_all_groups(self,
infos,
admin_group_html,
member_group_html,
external_group_html = None,
warnings=[],
ln=CFG_SITE_LANG):
"""
Displays the 3 tables of groups: admin, member and external
Parameters:
- 'ln' *string* - The language to display the interface in
- 'admin_group_html' *string* - HTML code for displaying all the groups
the user is the administrator of
- 'member_group_html' *string* - HTML code for displaying all the groups
the user is member of
- 'external_group_html' *string* - HTML code for displaying all the
external groups the user is member of
"""
_ = gettext_set_language(ln)
group_text = self.tmpl_infobox(infos)
group_text += self.tmpl_warning(warnings)
if external_group_html:
group_text += """
%s
%s
%s
""" %(admin_group_html, member_group_html, external_group_html)
else:
group_text += """
%s
%s
""" %(admin_group_html, member_group_html)
return group_text
def tmpl_display_admin_groups(self, groups, ln=CFG_SITE_LANG):
"""
Display the groups the user is admin of.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'groups' *list* - All the group the user is admin of
- 'infos' *list* - Display infos on top of admin group table
"""
_ = gettext_set_language(ln)
img_link = """
s/img/%(img)s)
%(text)s
"""
out = self.tmpl_group_table_title(img="/img/group_admin.png",
text=_("You are an administrator of the following groups:") )
out += """
%s
%s
""" %(_("Group"), _("Description"))
if len(groups) == 0:
out += """
%s
""" %(_("You are not an administrator of any groups."),)
for group_data in groups:
(grpID, name, description) = group_data
edit_link = img_link % {'siteurl' : CFG_SITE_URL,
'grpID' : grpID,
'ln': ln,
'img':"webbasket_create_small.png",
'text':_("Edit group"),
'action':"edit"
}
members_link = img_link % {'siteurl' : CFG_SITE_URL,
'grpID' : grpID,
'ln': ln,
'img':"webbasket_usergroup.png",
'text':_("Edit %s members") % '',
'action':"members"
}
out += """
%s
%s
%s
%s
""" % (cgi.escape(name), cgi.escape(description), edit_link, members_link)
out += """
""" % {'ln': ln,
'write_label': _("Create new group"),
}
return out
def tmpl_display_member_groups(self, groups, ln=CFG_SITE_LANG):
"""
Display the groups the user is member of.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'groups' *list* - All the group the user is member of
"""
_ = gettext_set_language(ln)
group_text = self.tmpl_group_table_title(img="/img/webbasket_us.png", text=_("You are a member of the following groups:"))
group_text += """
%s
%s
""" % (_("Group"), _("Description"))
if len(groups) == 0:
group_text += """
%s
""" %(_("You are not a member of any groups."),)
for group_data in groups:
(id, name, description) = group_data
group_text += """
%s
%s
""" % (cgi.escape(name), cgi.escape(description))
group_text += """
""" % {'ln': ln,
'join_label': _("Join new group"),
'leave_label':_("Leave group")
}
return group_text
def tmpl_display_external_groups(self, groups, ln=CFG_SITE_LANG):
"""
Display the external groups the user is member of.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'groups' *list* - All the group the user is member of
"""
_ = gettext_set_language(ln)
group_text = self.tmpl_group_table_title(img="/img/webbasket_us.png", text=_("You are a member of the following external groups:"))
group_text += """
%s
%s
""" % (_("Group"), _("Description"))
if len(groups) == 0:
group_text += """
%s
""" %(_("You are not a member of any external groups."),)
for group_data in groups:
(id, name, description) = group_data
group_text += """
%s
%s
""" % (cgi.escape(name), cgi.escape(description))
group_text += """
"""
return group_text
def tmpl_display_input_group_info(self,
group_name,
group_description,
join_policy,
act_type="create",
grpID=None,
warnings=[],
ln=CFG_SITE_LANG):
"""
Display group data when creating or updating a group:
Name, description, join_policy.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'group_name' *string* - name of the group
- 'group_description' *string* - description of the group
- 'join_policy' *string* - join policy
- 'act_type' *string* - info about action : create or edit(update)
- 'grpID' *int* - ID of the group(not None in case of group editing)
- 'warnings' *list* - Display warning if values are not correct
"""
_ = gettext_set_language(ln)
#default
hidden_id =""
form_name = "create_group"
action = CFG_SITE_URL + '/yourgroups/create'
button_label = _("Create new group")
button_name = "create_button"
label = _("Create new group")
delete_text = ""
if act_type == "update":
form_name = "update_group"
action = CFG_SITE_URL + '/yourgroups/edit'
button_label = _("Update group")
button_name = "update"
label = _('Edit group %s') % cgi.escape(group_name)
delete_text = """"""
delete_text %= (_("Delete group"),"delete")
if grpID is not None:
hidden_id = """"""
hidden_id %= grpID
out = self.tmpl_warning(warnings)
out += """
"""
out %= {'action' : action,
'logo': CFG_SITE_URL + '/img/webbasket_create.png',
'label': label,
'form_name' : form_name,
'name_label': _("Group name:"),
'delete_text': delete_text,
'description_label': _("Group description:"),
'join_policy_label': _("Group join policy:"),
'group_name': cgi.escape(group_name, 1),
'group_description': cgi.escape(group_description, 1),
'button_label': button_label,
'button_name':button_name,
'cancel_label':_("Cancel"),
'hidden_id':hidden_id,
'ln': ln,
'join_policy' :self.__create_join_policy_selection_menu("join_policy",
join_policy,
ln)
}
return out
def tmpl_display_input_join_group(self,
group_list,
group_name,
group_from_search,
search,
warnings=[],
ln=CFG_SITE_LANG):
"""
Display the groups the user can join.
He can use default select list or the search box
Parameters:
- 'ln' *string* - The language to display the interface in
- 'group_list' *list* - All the group the user can join
- 'group_name' *string* - Name of the group the user is looking for
- 'group_from search' *list* - List of the group the user can join matching group_name
- 'search' *int* - User is looking for group using group_name
- 'warnings' *list* - Display warning if two group are selected
"""
_ = gettext_set_language(ln)
out = self.tmpl_warning(warnings)
search_content = ""
if search:
search_content = """ """
if group_from_search != []:
search_content += self.__create_select_menu('grpID', group_from_search, _("Please select:"))
else:
search_content += _("No matching group")
search_content += """ %s
""" % (member_list,_("Remove member"))
else :
member_text = """%s """ % _("No members.")
if pending_members :
pending_list = self.__create_select_menu("pending_member_id", pending_members, _("Please select:"))
pending_text = """
%s
""" % (pending_list,_("Accept member"), _("Reject member"))
else :
pending_text = """%s """ % _("No members awaiting approval.")
header1 = self.tmpl_group_table_title(text=_("Current members"))
header2 = self.tmpl_group_table_title(text=_("Members awaiting approval"))
header3 = _("Invite new members")
write_a_message_url = create_url(
"%s/yourmessages/write" % CFG_SITE_URL,
{
'ln' : ln,
'msg_subject' : _('Invitation to join "%s" group' % escape_html(group_name)),
'msg_body' : _("""\
Hello:
I think you might be interested in joining the group "%(x_name)s".
You can join by clicking here: %(x_url)s.
Best regards.
""") % {'x_name': group_name,
'x_url': create_html_link("%s/yourgroups/join" % CFG_SITE_URL, { 'grpID' : grpID,
'join_button' : "1",
},
link_label=group_name, escape_urlargd=True, escape_linkattrd=True)}})
link_open = '' % escape_html(write_a_message_url)
invite_text = _("If you want to invite new members to join your group, please use the %(x_url_open)sweb message%(x_url_close)s system.") % \
{'x_url_open': link_open,
'x_url_close': ''}
action = CFG_SITE_URL + '/yourgroups/members?ln=' + ln
out %= {'title':_('Group: %s') % escape_html(group_name),
'member_text' : member_text,
'pending_text' :pending_text,
'action':action,
'grpID':grpID,
'header1': header1,
'header2': header2,
'header3': header3,
'img_alt_header1': _("Current members"),
'img_alt_header2': _("Members awaiting approval"),
'img_alt_header3': _("Invite new members"),
'invite_text': invite_text,
'imgurl': CFG_SITE_URL + '/img',
'cancel_label':_("Cancel"),
'ln':ln
}
return out
def tmpl_display_input_leave_group(self,
groups,
warnings=[],
ln=CFG_SITE_LANG):
"""Display groups the user can leave.
Parameters:
- 'ln' *string* - The language to display the interface in
- 'groups' *list* - List of groups the user is currently member of
- 'warnings' *list* - Display warning if no group is selected
"""
_ = gettext_set_language(ln)
out = self.tmpl_warning(warnings)
out += """
"""
if groups:
groups = self.__create_select_menu("grpID", groups, _("Please select:"))
list_label = _("Group list")
submit = """""" % _("Leave group")
else :
groups = _("You are not member of any group.")
list_label = ""
submit = ""
action = CFG_SITE_URL + '/yourgroups/leave?ln=%s'
action %= (ln)
out %= {'groups' : groups,
'list_label' : list_label,
'action':action,
'logo': CFG_SITE_URL + '/img/webbasket_create.png',
'label' : _("Leave group"),
'cancel_label':_("Cancel"),
'ln' :ln,
'submit' : submit
}
return out
def tmpl_confirm_delete(self, grpID, ln=CFG_SITE_LANG):
"""
display a confirm message when deleting a group
@param grpID *int* - ID of the group
@param ln: language
@return: html output
"""
_ = gettext_set_language(ln)
action = CFG_SITE_URL + '/yourgroups/edit'
out = """
"""% {'message': _("Are you sure you want to delete this group?"),
'ln':ln,
'yes_label': _("Yes"),
'no_label': _("No"),
'grpID':grpID,
'action': action
}
return out
def tmpl_confirm_leave(self, uid, grpID, ln=CFG_SITE_LANG):
"""
display a confirm message
@param grpID *int* - ID of the group
@param ln: language
@return: html output
"""
_ = gettext_set_language(ln)
action = CFG_SITE_URL + '/yourgroups/leave'
out = """
"""% {'message': _("Are you sure you want to leave this group?"),
'ln':ln,
'yes_label': _("Yes"),
'no_label': _("No"),
'grpID':grpID,
'action': action
}
return out
def __create_join_policy_selection_menu(self, name, current_join_policy, ln=CFG_SITE_LANG):
"""Private function. create a drop down menu for selection of join policy
@param current_join_policy: join policy as defined in CFG_WEBSESSION_GROUP_JOIN_POLICY
@param ln: language
"""
_ = gettext_set_language(ln)
elements = [(CFG_WEBSESSION_GROUP_JOIN_POLICY['VISIBLEOPEN'],
_("Visible and open for new members")),
(CFG_WEBSESSION_GROUP_JOIN_POLICY['VISIBLEMAIL'],
_("Visible but new members need approval"))
]
select_text = _("Please select:")
return self.__create_select_menu(name, elements, select_text, selected_key=current_join_policy)
def __create_select_menu(self, name, elements, select_text, multiple=0, selected_key=None):
""" private function, returns a popup menu
@param name: name of HTML control
@param elements: list of (key, value)
"""
if multiple :
out = """