.data = '').
Processors may stop the processing chain by raising StopIteration.
IMPORTANT: By default the method will execute custom post processors
defined in the webdeposit_config. If you override the method, be
sure to call this method to ensure extra processors are called::
super(MyField, self).post_process(
form, extra_processors=extra_processors
)
"""
# Check if post processing should run for this field
if self.name in formfields or not formfields:
stop = False
for p in (self.processors or []):
try:
p(form, self, submit)
except StopIteration:
stop = True
break
if not stop:
for p in (extra_processors or []):
p(form, self, submit)
def perform_autocomplete(self, form, name, term, limit=50):
"""
Run auto-complete method for field. This method should not be called
directly, instead use Form.autocomplete().
"""
if name == self.name and self.autocomplete:
return self.autocomplete(form, self, term, limit=limit)
return None
def add_message(self, msg, state=None):
"""
Add a message
@param msg: The message to set
@param state: State of message; info, warning, error, success.
"""
self._messages.append(msg)
if state:
self._message_state = state
def set_flags(self, flags):
"""
Set field flags
"""
field_flags = flags.get(self.name, [])
for check_flag in CFG_FIELD_FLAGS:
setattr(self.flags, check_flag, check_flag in field_flags)
@property
def messages(self):
"""
Retrieve field messages
"""
if self.errors:
return {self.name: dict(
state='error',
messages=self.errors
)}
else:
return {self.name: dict(
state=getattr(self, '_message_state', ''),
messages=self._messages
)}
diff --git a/invenio/modules/deposit/processor_utils.py b/invenio/modules/deposit/processor_utils.py
index d5d38161b..1a7db0ff6 100644
--- a/invenio/modules/deposit/processor_utils.py
+++ b/invenio/modules/deposit/processor_utils.py
@@ -1,276 +1,276 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# from wtforms.validators import ValidationError, StopValidation, Regexp
from werkzeug import MultiDict
from invenio.utils.datacite import DataciteMetadata
from invenio.utils.sherpa_romeo import SherpaRomeoSearch
-from invenio.legacy.bibfield import get_record
+from invenio.modules.records.api import get_record
from invenio.utils import persistentid as pidutils
#
# General purpose processors
#
def replace_field_data(field_name, getter=None):
"""
Returns a processor, which will replace the given field names value with
the value from the field where the processor is installed.
"""
def _inner(form, field, submit=False):
getattr(form, field_name).data = getter(field) if getter else field.data
return _inner
#
# PID processors
#
class PidSchemeDetection(object):
"""
Detect the persistent identifier scheme and store it in another field.
"""
def __init__(self, set_field=None):
self.set_field = set_field
def __call__(self, form, field, submit=False):
if field.data:
schemes = pidutils.detect_identifier_schemes(field.data)
if schemes:
getattr(form, self.set_field).data = schemes[0]
class PidNormalize(object):
"""
Normalize a persistent identifier
"""
def __init__(self, scheme_field=None):
self.scheme_field = scheme_field
def __call__(self, form, field, submit=False):
scheme = getattr(form, self.scheme_field).data
field.data = pidutils.normalize_pid(field.data, scheme=scheme)
#
# DOI-related processors
#
def datacite_dict_mapper(datacite, form, mapping):
"""
Helper function to map DataCite metadata to form fields based on a mapping
"""
for func_name, field_name in mapping.items():
setattr(form, field_name, getattr(datacite, func_name)())
class DataCiteLookup(object):
"""
Lookup DOI metadata in DataCite but only if DOI is not locally
administered.
"""
def __init__(self, display_info=False, mapping=None,
mapping_func=None, exclude_prefix='10.5072'):
self.display_info = display_info
self.mapping = mapping or dict(
get_publisher='publisher',
get_titles='title',
get_dates='date',
get_description='abstract',
)
self.mapping_func = mapping_func or datacite_dict_mapper
self.prefix = exclude_prefix
def __call__(self, form, field, submit=False):
if not field.errors and field.data and not field.data.startswith(self.prefix + '/'):
try:
datacite = DataciteMetadata(field.data)
if datacite.error:
if self.display_info:
field.add_message("DOI metadata could not be retrieved.", state='info')
return
if self.mapping_func:
self.mapping_func(datacite, form, self.mapping)
if self.display_info:
field.add_message("DOI metadata successfully imported from DataCite.", state='info')
except Exception:
# Ignore errors
pass
datacite_lookup = DataCiteLookup
def sherpa_romeo_issn_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return dict(error=0, error_message='')
s = SherpaRomeoSearch()
s.search_issn(value)
if s.error:
field.add_message(s.error_message, state='info')
return
if s.get_num_hits() == 1:
journal = s.parser.get_journals(attribute='jtitle')
journal = journal[0]
publisher = s.parser.get_publishers(journal=journal)
if publisher is not None and publisher != []:
if hasattr(form, 'journal'):
form.journal.data = journal
if hasattr(form, 'publisher'):
form.publisher.data = publisher['name']
return
else:
if hasattr(form, 'journal'):
form.journal.data = journal
return
field.add_message("Couldn't find Journal.", state='info')
def sherpa_romeo_publisher_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return
s = SherpaRomeoSearch()
s.search_publisher(value)
if s.error:
field.add_message(s.error_message, state='info')
conditions = s.parser.get_publishers(attribute='conditions')
if conditions is not None and s.get_num_hits() == 1:
conditions = conditions[0]
else:
conditions = []
if conditions != []:
conditions_html = "Conditions
"
if isinstance(conditions['condition'], str):
conditions_html += "- " + conditions['condition'] + "
"
else:
for condition in conditions['condition']:
conditions_html += "- " + condition + "
"
copyright_links = s.parser.get_publishers(attribute='copyrightlinks')
if copyright_links is not None and copyright_links != []:
copyright_links = copyright_links[0]
else:
copyright_links = None
if isinstance(copyright_links, list):
copyright_links_html = ""
for copyright_link in copyright_links['copyrightlink']:
copyright_links_html += '' + copyright_link['copyrightlinktext'] + "
"
elif isinstance(copyright_links, dict):
if isinstance(copyright_links['copyrightlink'], list):
for copyright_link in copyright_links['copyrightlink']:
copyright_links_html = '' + copyright_link['copyrightlinktext'] + "
"
else:
copyright_link = copyright_links['copyrightlink']
copyright_links_html = '' + copyright_link['copyrightlinktext'] + "
"
home_url = s.parser.get_publishers(attribute='homeurl')
if home_url is not None and home_url != []:
home_url = home_url[0]
home_url = '' + home_url + ""
else:
home_url = None
info_html = ""
if home_url is not None:
info_html += "" + home_url + "
"
if conditions is not None:
info_html += "" + conditions_html + "
"
if copyright_links is not None:
info_html += "" + copyright_links_html + "
"
if info_html != "":
field.add_message(info_html, state='info')
def sherpa_romeo_journal_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return
s = SherpaRomeoSearch()
s.search_journal(value, 'exact')
if s.error:
field.add_message(s.error_message, state='info')
return
if s.get_num_hits() == 1:
issn = s.parser.get_journals(attribute='issn')
if issn != [] and issn is not None:
issn = issn[0]
publisher = s.parser.get_publishers(journal=value)
if publisher is not None and publisher != []:
if hasattr(form, 'issn'):
form.issn.data = issn
if hasattr(form, 'publisher'):
form.publisher.data = publisher['name']
form.publisher.post_process(form)
return
field.add_message("Journal's Publisher not found", state='info')
if hasattr(form, 'issn'):
form.issn.data = issn
if hasattr(form, 'publisher'):
form.publisher.data = publisher
form.publisher.post_process(form)
else:
field.add_message("Couldn't find ISSN.", state='info')
def record_id_process(form, field, submit=False):
value = field.data or ''
if value == "" or value.isspace():
return
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
if is_number(field.data):
json_reader = get_record(value)
else:
field.add_message("Record id must be a number!", state='error')
return
if json_reader is not None:
webdeposit_json = form.uncook_json(json_reader, {}, value)
#FIXME: update current json, past self, what do you mean?? :S
field.add_message('Record was loaded successfully',
state='info')
form.process(MultiDict(webdeposit_json))
else:
field.add_message("Record doesn't exist", state='info')
diff --git a/invenio/modules/deposit/tasks.py b/invenio/modules/deposit/tasks.py
index af71ff5cd..19d46b325 100644
--- a/invenio/modules/deposit/tasks.py
+++ b/invenio/modules/deposit/tasks.py
@@ -1,263 +1,264 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2012, 2013 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
"""
import os
from tempfile import mkstemp
from flask import current_app, abort
from flask.ext.login import current_user
from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
bibtask_allocate_sequenceid
-from invenio.legacy.bibfield.bibfield_jsonreader import JsonReader
+from invenio.modules.records.api import Record
from invenio.config import CFG_TMPSHAREDDIR
from invenio.legacy.dbquery import run_sql
from invenio.modules.deposit.models import Deposition, Agent, \
DepositionDraftCacheManager
from invenio.ext.logging import register_exception
try:
from invenio.pidstore_model import PersistentIdentifier
HAS_PIDSUPPORT = True
except ImportError:
HAS_PIDSUPPORT = False
def authorize_user(action, **params):
"""
Check if current user is authorized to perform the action.
"""
def _authorize_user(obj, dummy_eng):
from invenio.modules.access.engine import acc_authorize_action
auth, message = acc_authorize_action(
current_user.get_id(),
action,
**dict((k, v() if callable(v) else v)
for (k, v) in params.items()))
if auth != 0:
current_app.logger.info(message)
abort(401)
return _authorize_user
def prefill_draft(form_class, draft_id='_default', clear=True):
"""
Fill draft values with values from pre-filled cache
"""
def _prefill_draft(obj, eng):
draft_cache = DepositionDraftCacheManager.get()
if draft_cache.has_data():
d = Deposition(obj)
draft_cache.fill_draft(
d, draft_id, form_class=form_class, clear=clear
)
d.update()
return _prefill_draft
def render_form(form_class, draft_id='_default'):
"""
Renders a form if the draft associated with it has not yet been completed.
:param form_class: The form class which should be rendered.
:param draft_id: The name of the draft to create. Must be specified if you
put more than two ``render_form'''s in your deposition workflow.
"""
def _render_form(obj, eng):
d = Deposition(obj)
draft = d.get_or_create_draft(draft_id, form_class=form_class)
if draft.is_completed():
eng.jumpCallForward(1)
else:
form = draft.get_form(validate_draft=draft.validate)
form.validate = True
d.set_render_context(dict(
template_name_or_list=form.get_template(),
deposition=d,
deposition_type=(
None if d.type.is_default() else d.type.get_identifier()
),
uuid=d.id,
draft=draft,
form=form,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
))
d.update()
eng.halt('Wait for form submission.')
return _render_form
def create_recid():
"""
Create a new record id.
"""
def _create_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(include_sealed=False)
if sip is None:
raise Exception("No submission information package found.")
if 'recid' not in sip.metadata:
sip.metadata['recid'] = run_sql(
"INSERT INTO bibrec (creation_date, modification_date) "
"VALUES (NOW(), NOW())"
)
d.update()
return _create_recid
def mint_pid(pid_field='doi', pid_creator=None, pid_store_type='doi',
existing_pid_checker=None):
"""
Register a persistent identifier internally.
:param pid_field: The recjson key for where to look for a pre-reserved pid.
Defaults to 'pid'.
:param pid_creator: Callable taking one argument (the recjson) that when
called will generate and return a pid string.
:param pid_store_type: The PID store type. Defaults to 'doi'.
:param existing_pid_checker: A callable taking two arguments
(pid_str, recjson) that will check if an pid found using ``pid_field''
should be registered or not.
"""
if not HAS_PIDSUPPORT:
def _mint_pid_dummy(dummy_obj, dummy_eng):
pass
return _mint_pid_dummy
def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(include_sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
current_app.logger.info("Registering pid %s" % pid_text)
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
return _mint_pid
def prepare_sip():
"""
Prepare a submission information package
"""
def _prepare_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(include_sealed=False)
if sip is None:
sip = d.create_sip()
sip.metadata['fft'] = sip.metadata['files']
del sip.metadata['files']
sip.agents = [Agent(role='creator', from_request_context=True)]
d.update()
return _prepare_sip
def finalize_record_sip():
"""
Finalizes the SIP by generating the MARC and storing it in the SIP.
"""
def _finalize_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(include_sealed=False)
- jsonreader = JsonReader()
+ json = Record()
for k, v in sip.metadata.items():
- jsonreader[k] = v
+ json.set(k, v, extend=True)
- sip.package = jsonreader.legacy_export_as_marc()
+ sip.package = json.legacy_export_as_marc()
- current_app.logger.info(jsonreader['__error_messages'])
+ current_app.logger.info(json['__meta_metadata__']['__errors__'])
+ current_app.logger.info(json['__meta_metadata__']['__continuable_errors__'])
current_app.logger.info(sip.package)
d.update()
return _finalize_sip
def upload_record_sip():
"""
Generates the record from marc.
The function requires the marc to be generated,
so the function export_marc_from_json must have been called successfully
before
"""
def create(obj, dummy_eng):
current_app.logger.info("Upload sip")
d = Deposition(obj)
sip = d.get_latest_sip(include_sealed=False)
sip.seal()
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.xml',
dir=CFG_TMPSHAREDDIR,
)
os.write(tmp_file_fd, sip.package)
os.close(tmp_file_fd)
# Trick to have access to task_sequence_id in subsequent tasks.
d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
task_low_level_submission(
'bibupload', 'webdeposit',
'-r' if 'recid' in sip.metadata else '-i', tmp_file_path, '-P5',
'-I', str(d.workflow_object.task_sequence_id)
)
d.update()
return create
diff --git a/invenio/modules/documents/api.py b/invenio/modules/documents/api.py
index 8324abb54..9709e170a 100644
--- a/invenio/modules/documents/api.py
+++ b/invenio/modules/documents/api.py
@@ -1,34 +1,34 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
from invenio.modules.jsonalchemy.wrappers import SmartJson
from invenio.modules.jsonalchemy.jsonext.engines.sqlalchemy import SQLAlchemyStorage
from invenio.modules.jsonalchemy.jsonext.readers.json_reader import reader
from .models import Document as DocumentModel
class Document(SmartJson):
storage_engine = SQLAlchemyStorage(DocumentModel)
@classmethod
def create(cls, data, model='common_document'):
- record = reader(data, model=model)
+ record = reader(data, namespace='documentext', model=model)
document = cls(record.translate())
return cls.storage_engine.save_one(document.dumps())
diff --git a/invenio/modules/documents/documentext/__init__.py b/invenio/modules/documents/documentext/__init__.py
deleted file mode 100644
index eb537e874..000000000
--- a/invenio/modules/documents/documentext/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-##
-## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
-##
-## Invenio is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License as
-## published by the Free Software Foundation; either version 2 of the
-## License, or (at your option) any later version.
-##
-## Invenio is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
diff --git a/invenio/modules/documents/documentext/fields/__init__.py b/invenio/modules/documents/documentext/fields/__init__.py
deleted file mode 100644
index eb537e874..000000000
--- a/invenio/modules/documents/documentext/fields/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-##
-## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
-##
-## Invenio is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License as
-## published by the Free Software Foundation; either version 2 of the
-## License, or (at your option) any later version.
-##
-## Invenio is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
diff --git a/invenio/modules/documents/documentext/fields/core_fields.cfg b/invenio/modules/documents/documentext/fields/base.cfg
similarity index 58%
rename from invenio/modules/documents/documentext/fields/core_fields.cfg
rename to invenio/modules/documents/documentext/fields/base.cfg
index f244e8446..8906b6e90 100644
--- a/invenio/modules/documents/documentext/fields/core_fields.cfg
+++ b/invenio/modules/documents/documentext/fields/base.cfg
@@ -1,49 +1,45 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
@persistent_identifier(0)
uuid:
+ """
+ This is the main persistent identifier of a document and will be used
+ internally as this, therefore the pid important should always be '0'.
+ """
+ schema:
+ {'_id': {'type':'objectid'}}
checker:
check_field_existence(1, continuable=False)
- documentation:
- """
- This is the main persistent identifier of a document and will be used
- internally as this, therefore the pid important should always be '0'.
- """
version:
- documentation:
- """Document version"""
+ """Document version"""
parent:
- calculated:
- @do_not_cache
- @parse_first(('parent_uuid', ))
- @depends_on(('parent_uuid', ))
- invenio.modules.documents.api.Document.storage_engine.get_one(self.get('parent_uuid'))
- documentation:
- """
- Return parent object
- """
+ """
+ Return parent object
+ """
+ # calculated:
+ # @depends_on(('parent_uuid', ))
+ # invenio.modules.documents.api.Document.storage_engine.get_one(self.get('parent_uuid'))
parent_uuid:
- documentation:
- """
- FIXME: add checker if not None document_exists(parent_uuid)
- """
+ """
+ FIXME: add checker if not None document_exists(parent_uuid)
+ """
diff --git a/invenio/modules/documents/documentext/models/__init__.py b/invenio/modules/documents/documentext/models/__init__.py
deleted file mode 100644
index eb537e874..000000000
--- a/invenio/modules/documents/documentext/models/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# -*- coding: utf-8 -*-
-##
-## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
-##
-## Invenio is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License as
-## published by the Free Software Foundation; either version 2 of the
-## License, or (at your option) any later version.
-##
-## Invenio is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
diff --git a/invenio/modules/documents/documentext/models/common_document.cfg b/invenio/modules/documents/documentext/models/base.cfg
similarity index 82%
rename from invenio/modules/documents/documentext/models/common_document.cfg
rename to invenio/modules/documents/documentext/models/base.cfg
index 0421a11f4..24b12bd12 100644
--- a/invenio/modules/documents/documentext/models/common_document.cfg
+++ b/invenio/modules/documents/documentext/models/base.cfg
@@ -1,31 +1,29 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+"""
+Common fields in all the documents inside any Invenio installation.
+"""
+
fields:
- @inherit_from(('common', ))
+ @inherit_from(('base', ))
_id = uuid
version
parent
parent_uuid
- modification_date
- creation_date
-documentation:
- """
- Common fields in all the documents inside any Invenio installation.
- """
diff --git a/invenio/modules/formatter/api.py b/invenio/modules/formatter/api.py
index 09f92ee62..ba8ee4c35 100644
--- a/invenio/modules/formatter/api.py
+++ b/invenio/modules/formatter/api.py
@@ -1,534 +1,534 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
invenio.modules.formatter.api
-----------------------------------------
Database access related functions for BibFormat engine and
administration pages.
"""
import zlib
from invenio.ext.sqlalchemy import db
from sqlalchemy.exc import SQLAlchemyError
from invenio.utils.date import convert_datetime_to_utc_string, strftime
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.search.models import Tag
from .models import Format, Formatname, Bibfmt
def get_creation_date(sysno, fmt="%Y-%m-%dT%H:%M:%SZ"):
"""
Returns the creation date of the record 'sysno'.
:param sysno: the record ID for which we want to retrieve creation date
:param fmt: output format for the returned date
:return: creation date of the record
:rtype: string
"""
try:
return convert_datetime_to_utc_string(
Bibrec.query.get(sysno).creation_date, fmt)
except SQLAlchemyError:
return ""
def get_modification_date(sysno, fmt="%Y-%m-%dT%H:%M:%SZ"):
"""
Returns the date of last modification for the record 'sysno'.
:param sysno: the record ID for which we want to retrieve modification date
:param fmt: output format for the returned date
:return: modification date of the record
:rtype: string
"""
try:
return convert_datetime_to_utc_string(
Bibrec.query.get(sysno).modification_date, fmt)
except SQLAlchemyError:
return ""
## XML Marc related functions
def get_tag_from_name(name):
"""
Returns the marc code corresponding the given name
:param name: name for which we want to retrieve the tag
:return: a tag corresponding to X{name} or None if not found
"""
try:
return Tag.query.filter(Tag.name.like(name)).one().value
except SQLAlchemyError:
return None
def get_tags_from_name(name):
"""
Returns the marc codes corresponding the given name,
ordered by value
:param name: name for which we want to retrieve the tags
:return: list of tags corresponding to X{name} or None if not found
"""
try:
return [tag.value for tag in
Tag.query.filter(Tag.name.like(name))
.order_by(Tag.value).all()]
except SQLAlchemyError:
return None
def tag_exists_for_name(name):
"""
Returns True if a tag exists for name in 'tag' table.
:param name: name for which we want to check if a tag exist
:return: True if a tag exist for X{name} or False
"""
return (Tag.query.filter(Tag.name.like(name)).count() > 0)
def get_name_from_tag(tag):
"""
Returns the name corresponding to a marc code
:param tag: tag to consider
:return: a name corresponding to X{tag}
"""
try:
return Tag.query.filter(Tag.value.like(tag)).one().name
except SQLAlchemyError:
return None
def name_exists_for_tag(tag):
"""
Returns True if a name exists for tag in 'tag' table.
:param tag: tag for which we want to check if a name exist
:return: True if a name exist for X{tag} or False
"""
return (Tag.query.filter(Tag.value.like(tag)).count() > 0)
def get_all_name_tag_mappings():
"""
Return the list of mappings name<->tag from 'tag' table.
The returned object is a dict with name as key (if 2 names are the same
we will take the value of one of them, as we cannot make the difference
in format templates)
:return: a dict containing list of mapping in 'tag' table
"""
result = dict()
for tag in Tag.query.all():
result[tag.name] = tag.value
return result
## Output formats related functions
def get_format_by_code(code):
"""
Returns the output format object given by code in the database.
Output formats are located inside 'format' table
:param code: the code of an output format
:return: Format object with given ID. None if not found
"""
f_code = code
if len(code) > 6:
f_code = code[:6]
try:
return Format.query.filter(Format.code == f_code.lower()).one()
except SQLAlchemyError:
return None
def get_format_property(code, property_name, default_value=None):
"""
Returns the value of a property of the output format given by code.
If code or property does not exist, return default_value
:param code: the code of the output format to get the value from
:param property_name: name of property to return
:param default_value: value to be returned if format not found
:return: output format property value
"""
return getattr(get_format_by_code(code), property_name, default_value)
def set_format_property(code, property_name, value):
"""
Sets the property of an output format, given by its code
If 'code' does not exist, create format
:param code: the code of the output format to update
:param property_name: name of property to set
:param value: value to assign
"""
format = get_format_by_code(code)
if format is None:
format = Format()
setattr(format, property_name, value)
if(property == 'name'):
format.set_name(value)
db.session.add(format)
db.session.commit()
def get_output_format_id(code):
"""
Returns the id of output format given by code in the database.
Output formats are located inside 'format' table
:param code: the code of an output format
:return: the id in the database of the output format. None if not found
"""
return get_format_property(code, 'id', None)
def add_output_format(code, name="", description="",
content_type="text/html", visibility=1):
"""
Add output format into format table.
If format with given code already exists, do nothing
:param code: the code of the new format
:param name: a new for the new format
:param description: a description for the new format
:param content_type: the content_type (if applicable)
of the new output format
:param visibility: if the output format is shown to users (1) or not (0)
:return: None
"""
format = get_format_by_code(code)
if format is None:
format = Format()
format.code = code.lower()
format.description = description
format.content_type = content_type
format.visibility = visibility
format.set_name(name)
db.session.add(format)
db.session.commit()
def remove_output_format(code):
"""
Removes the output format with 'code'
If code does not exist in database, do nothing
The function also removes all localized names in formatname table
:param code: the code of the output format to remove
:return: None
"""
format = get_format_by_code(code)
if format is not None:
db.session.query(Formatname)\
.filter(Formatname.id_format == format.id).delete()
db.session.delete(format)
db.session.commit()
def get_output_format_description(code):
"""
Returns the description of the output format given by code
If code or description does not exist, return empty string
:param code: the code of the output format to get the description from
:return: output format description
"""
return get_format_property(code, 'description', '')
def set_output_format_description(code, description):
"""
Sets the description of an output format, given by its code
If 'code' does not exist, create format
:param code: the code of the output format to update
:param description: the new description
:return: None
"""
set_format_property(code, 'description', description)
def get_output_format_visibility(code):
"""
Returns the visibility of the output format, given by its code
If code does not exist, return 0
:param code: the code of an output format
:return: output format visibility (0 if not visible, 1 if visible
"""
visibility = get_format_property(code, 'visibility', 0)
if visibility is not None and int(visibility) in range(0, 2):
return int(visibility)
else:
return 0
def set_output_format_visibility(code, visibility):
"""
Sets the visibility of an output format, given by its code
If 'code' does not exist, create format
:param code: the code of the output format to update
:param visibility: the new visibility (0: not visible, 1:visible)
:return: None
"""
set_format_property(code, 'visibility', visibility)
def get_existing_content_types():
"""
Returns the list of all MIME content-types used in existing output
formats.
Always returns at least a list with 'text/html'
:return: a list of content-type strings
"""
types = db.session.query(Format.content_type).distinct()
if 'text/html' not in types:
types.append('text/html')
return types
def get_output_format_content_type(code):
"""
Returns the content_type of the output format given by code
If code or content_type does not exist, return empty string
:param code: the code of the output format to get the description from
:return: output format content_type
"""
return get_format_property(code, 'content_type', '') or ''
def set_output_format_content_type(code, content_type):
"""
Sets the content_type of an output format, given by its code
If 'code' does not exist, create format
:param code: the code of the output format to update
:param content_type: the content type for the format
:return: None
"""
set_format_property(code, 'content_type', content_type)
def get_output_format_names(code):
"""
Returns the localized names of the output format designated by 'code'
The returned object is a dict with keys 'ln' (for long name)
and 'sn' (for short name), containing each a dictionary
with languages as keys.
The key 'generic' contains the generic name of the output format
(for use in admin interface)
For eg::
{'ln':
{'en': "a long name", 'fr': "un long nom", 'de': "ein lange Name"},
'sn':{'en': "a name", 'fr': "un nom", 'de': "ein Name"}
'generic': "a name"}
The returned dictionary is never None.
The keys 'ln' and 'sn' are always present.
However only languages present in the database are in dicts 'sn' and 'ln'.
Language "CFG_SITE_LANG" is always in dict.
The localized names of output formats are located in formatname table.
:param code: the code of the output format to get the names from
:return: a dict containing output format names
"""
out = {'sn': {}, 'ln': {}, 'generic': ''}
format = get_format_by_code(code)
if format is None:
return out
out['generic'] = format.name
for fname in Formatname.query\
.filter(Formatname.id_format == format.id).all():
if fname.type == 'sn' or fname.type == 'ln':
out[fname.type][fname.ln] = fname.value
return out
def set_output_format_name(code, name, lang="generic", type='ln'):
"""
Sets the name of an output format given by code.
if 'type' different from 'ln' or 'sn', do nothing
if 'name' exceeds 256 chars, 'name' is truncated to first 256 chars.
if 'code' does not correspond to exisiting output format,
create format if "generic" is given as lang
The localized names of output formats are located in formatname table.
:param code: the code of an ouput format
:param type: either 'ln' (for long name) and 'sn' (for short name)
:param lang: the language in which the name is given
:param name: the name to give to the output format
:return: None
"""
if type.lower() != "sn" and type.lower() != "ln":
return
format = get_format_by_code(code)
if format is None and lang == "generic" and type.lower() == "ln":
# Create output format inside table if it did not exist
# Happens when the output format was added not through web interface
format = Format()
if format is not None:
format.set_name(name, lang, type)
def change_output_format_code(old_code, new_code):
"""
Change the code of an output format
:param old_code: the code of the output format to change
:param new_code: the new code
:return: None
"""
set_format_property(old_code, 'code', new_code.lower())
def get_preformatted_record(recID, of, decompress=zlib.decompress):
"""
Returns the preformatted record with id 'recID' and format 'of'
If corresponding record does not exist for given output format,
returns None
:param recID: the id of the record to fetch
:param of: the output format code
:param decompress: the method used to decompress
the preformatted record in database
:return: formatted record as String, or None if not exist
"""
try:
value = Bibfmt.query\
.filter(Bibfmt.id_bibrec == recID)\
.filter(Bibfmt.format == of)\
.one().value
return str(decompress(value))
except SQLAlchemyError:
return None
# Decide whether to use DB slave:
# if of in ('xm', 'recstruct'):
# run_on_slave = False # for master formats, use DB master
# else:
# run_on_slave = True # for other formats, we can use DB slave
def get_preformatted_record_date(recID, of):
"""
Returns the date of the last update of the cache for the considered
preformatted record in bibfmt
If corresponding record does not exist for given output format,
returns None
:param recID: the id of the record to fetch
:param of: the output format code
:return: the date of the last update of the cache, or None if not exist
"""
try:
last_updated = Bibfmt.query\
.filter(Bibfmt.id_bibrec == recID)\
.filter(Bibfmt.format == of)\
.one().last_updated
return strftime("%Y-%m-%d %H:%M:%S", last_updated)
except SQLAlchemyError:
return None
## def keep_formats_in_db(output_formats):
## """
## Remove from db formats that are not in the list
## TOBE USED ONLY ONCE OLD BIBFORMAT IS REMOVED
## (or old behaviours will be erased...)
## """
## query = "SELECT code FROM format"
## res = run_sql(query)
## for row in res:
## if not row[0] in output_formats:
## query = "DELETE FROM format WHERE code='%s'"%row[0]
## def add_formats_in_db(output_formats):
## """
## Add given formats in db (if not already there)
## """
## for output_format in output_format:
## if get_format_from_db(output_format) is None:
## #Add new
## query = "UPDATE TABLE format "
## else:
## #Update
## query = "UPDATE TABLE format "
## query = "UPDATE TABLE format "
## res = run_sql(query)
## for row in res:
## if not row[0] in output_formats:
## query = "DELETE FROM format WHERE code='%s'"%row[0]
diff --git a/invenio/modules/formatter/engine.py b/invenio/modules/formatter/engine.py
index 8045777bc..80c6bcb0d 100644
--- a/invenio/modules/formatter/engine.py
+++ b/invenio/modules/formatter/engine.py
@@ -1,2256 +1,2256 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Formats a single XML Marc record using specified format.
There is no API for the engine. Instead use module L{bibformat}.
You can have a look at the various escaping modes available in
X{BibFormatObject} in function L{escape_field}
Still it is useful sometimes for debugging purpose to use the
L{BibFormatObject} class directly. For eg:
>>> from invenio.modules.formatter.engine import BibFormatObject
>>> bfo = BibFormatObject(102)
>>> bfo.field('245__a')
The order Rodentia in South America
>>> from invenio.modules.formatter.format_elements import bfe_title
>>> bfe_title.format_element(bfo)
The order Rodentia in South America
@see: bibformat.py, bibformat_utils.py
"""
__revision__ = "$Id$"
import re
import sys
import os
import inspect
import traceback
import zlib
import cgi
import types
from flask import has_app_context
from operator import itemgetter
from werkzeug.utils import cached_property
from invenio.base.globals import cfg
from invenio.base.utils import (autodiscover_template_context_functions,
autodiscover_format_elements)
from invenio.config import \
CFG_PATH_PHP, \
CFG_BINDIR, \
CFG_SITE_LANG
from invenio.ext.logging import \
register_exception
from invenio.legacy.bibrecord import \
create_record, \
record_get_field_instances, \
record_get_field_value, \
record_get_field_values, \
record_xml_output
from . import registry
from .engines.xslt import format
from invenio.legacy.dbquery import run_sql
from invenio.base.i18n import \
language_list_long, \
wash_language, \
gettext_set_language
from . import api as bibformat_dblayer
from .config import \
CFG_BIBFORMAT_TEMPLATES_DIR, \
CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION, \
CFG_BIBFORMAT_FORMAT_JINJA_TEMPLATE_EXTENSION, \
CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION, \
CFG_BIBFORMAT_OUTPUTS_PATH, \
InvenioBibFormatError
from invenio.modules.formatter.utils import \
record_get_xml, \
parse_tag
from invenio.utils.html import \
HTMLWasher, \
CFG_HTML_BUFFER_ALLOWED_TAG_WHITELIST, \
CFG_HTML_BUFFER_ALLOWED_ATTRIBUTE_WHITELIST
from invenio.modules.knowledge.api import get_kbr_values
from invenio.ext.template import render_template_to_string
from HTMLParser import HTMLParseError
from invenio.utils.shell import escape_shell_arg
if CFG_PATH_PHP: #Remove when call_old_bibformat is removed
from xml.dom import minidom
import tempfile
# Cache for data we have already read and parsed
format_templates_cache = {}
format_elements_cache = {}
format_outputs_cache = {}
html_field = '' # String indicating that field should be
# treated as HTML (and therefore no escaping of
# HTML tags should occur.
# Appears in some field values.
washer = HTMLWasher() # Used to remove dangerous tags from HTML
# sources
# Regular expression for finding ... tag in format templates
pattern_lang = re.compile(r'''
#closing start tag
(?P.*?) #anything but the next group (greedy)
() #end tag
''', re.IGNORECASE | re.DOTALL | re.VERBOSE)
# Builds regular expression for finding each known language in tags
ln_pattern_text = r"<("
for lang in language_list_long(enabled_langs_only=False):
ln_pattern_text += lang[0] +r"|"
ln_pattern_text = ln_pattern_text.rstrip(r"|")
ln_pattern_text += r")>(.*?)"
ln_pattern = re.compile(ln_pattern_text, re.IGNORECASE | re.DOTALL)
# Regular expression for finding text to be translated
translation_pattern = re.compile(r'_\((?P.*?)\)_', \
re.IGNORECASE | re.DOTALL | re.VERBOSE)
# Regular expression for finding tag in format templates
pattern_format_template_name = re.compile(r'''
#closing start tag
(?P.*?) #name value. any char that is not end tag
()(\n)? #end tag
''', re.IGNORECASE | re.DOTALL | re.VERBOSE)
# Regular expression for finding tag in format templates
pattern_format_template_desc = re.compile(r'''
#closing start tag
(?P.*?) #description value. any char that is not end tag
(\n)? #end tag
''', re.IGNORECASE | re.DOTALL | re.VERBOSE)
# Regular expression for finding tags in format templates
pattern_tag = re.compile(r'''
[^/\s]+) #any char but a space or slash
\s* #any number of spaces
(?P(\s* #params here
(?P([^=\s])*)\s* #param name: any chars that is not a white space or equality. Followed by space(s)
=\s* #equality: = followed by any number of spaces
(?P[\'"]) #one of the separators
(?P.*?) #param value: any chars that is not a separator like previous one
(?P=sep) #same separator as starting one
)*) #many params
\s* #any number of spaces
(/)?> #end of the tag
''', re.IGNORECASE | re.DOTALL | re.VERBOSE)
# Regular expression for finding params inside tags in format templates
pattern_function_params = re.compile('''
(?P([^=\s])*)\s* # Param name: any chars that is not a white space or equality. Followed by space(s)
=\s* # Equality: = followed by any number of spaces
(?P[\'"]) # One of the separators
(?P.*?) # Param value: any chars that is not a separator like previous one
(?P=sep) # Same separator as starting one
''', re.VERBOSE | re.DOTALL )
# Regular expression for finding format elements "params" attributes
# (defined by @param)
pattern_format_element_params = re.compile('''
@param\s* # Begins with AT param keyword followed by space(s)
(?P[^\s=]*):\s* # A single keyword and comma, then space(s)
#(=\s*(?P[\'"]) # Equality, space(s) and then one of the separators
#(?P.*?) # Default value: any chars that is not a separator like previous one
#(?P=sep) # Same separator as starting one
#)?\s* # Default value for param is optional. Followed by space(s)
(?P.*) # Any text that is not end of line (thanks to MULTILINE parameter)
''', re.VERBOSE | re.MULTILINE)
# Regular expression for finding format elements "see also" attribute
# (defined by @see)
pattern_format_element_seealso = re.compile('''@see:\s*(?P.*)''',
re.VERBOSE | re.MULTILINE)
#Regular expression for finding 2 expressions in quotes, separated by
#comma (as in template("1st","2nd") )
#Used when parsing output formats
## pattern_parse_tuple_in_quotes = re.compile('''
## (?P[\'"])
## (?P.*)
## (?P=sep1)
## \s*,\s*
## (?P[\'"])
## (?P.*)
## (?P=sep2)
## ''', re.VERBOSE | re.MULTILINE)
sub_non_alnum = re.compile('[^0-9a-zA-Z]+')
fix_tag_name = lambda s: sub_non_alnum.sub('_', s.lower())
from invenio.utils.memoise import memoize
class LazyTemplateContextFunctionsCache(object):
"""Loads bibformat elements using plugin builder and caches results."""
@cached_property
def template_context_functions(self):
"""Returns template context functions"""
modules = autodiscover_template_context_functions()
elem = {}
for m in modules:
register_func = getattr(m, 'template_context_function', None)
if register_func and isinstance(register_func, types.FunctionType):
elem[m.__name__.split('.')[-1]] = register_func
return elem
@memoize
def bibformat_elements(self, modules=None):
"""Returns bibformat elements."""
if modules is None:
modules = registry.format_elements
elem = {}
for m in modules:
if m is None:
continue
name = m.__name__.split('.')[-1]
filename = m.__file__[:-1] if m.__file__.endswith('.pyc') \
else m.__file__
register_func = getattr(m, 'format_element',
getattr(m, 'format', None))
escape_values = getattr(m, 'escape_values', None)
if register_func and isinstance(register_func, types.FunctionType):
register_func._escape_values = escape_values
register_func.__file__ = filename
elem[name] = register_func
return elem
@cached_property
def functions(self):
def insert(name):
def _bfe_element(bfo, **kwargs):
# convert to utf-8 for legacy app
kwargs = dict((k, v.encode('utf-8') if isinstance(v, unicode) else v)
for k, v in kwargs.iteritems())
format_element = get_format_element(name)
(out, dummy) = eval_format_element(format_element,
bfo,
kwargs)
# returns unicode for jinja2
return out.decode('utf-8')
return _bfe_element
# Old bibformat templates
tfn_from_files = dict((name.lower(), insert(name.lower()))
for name in self.bibformat_elements().keys())
# Update with new template context functions
tfn_from_files.update(self.template_context_functions)
bfe_from_tags = {}
if has_app_context():
from invenio.ext.sqlalchemy import db
from invenio.modules.search.models import Tag
# get functions from tag table
bfe_from_tags = dict(('bfe_'+fix_tag_name(name),
insert(fix_tag_name(name)))
for name in map(itemgetter(0),
db.session.query(Tag.name).all()))
# overwrite functions from tag table with functions from files
bfe_from_tags.update(tfn_from_files)
return bfe_from_tags
TEMPLATE_CONTEXT_FUNCTIONS_CACHE = LazyTemplateContextFunctionsCache()
def get_format_element_path(filename):
if filename.endswith('.py'):
filename = filename[:-3]
return TEMPLATE_CONTEXT_FUNCTIONS_CACHE.bibformat_elements()[filename].__file__
def call_old_bibformat(recID, of="HD", on_the_fly=False, verbose=0):
"""
FIXME: REMOVE FUNCTION WHEN MIGRATION IS DONE
Calls BibFormat for the record RECID in the desired output format 'of'.
Note: this functions always try to return HTML, so when
bibformat returns XML with embedded HTML format inside the tag
FMT $g, as is suitable for prestoring output formats, we
perform un-XML-izing here in order to return HTML body only.
@param recID: record ID to format
@param of: output format to be used for formatting
@param on_the_fly: if False, try to return an already preformatted version of the record in the database
@param verbose: verbosity
@return: a formatted output using old BibFormat
"""
out = ""
res = []
if not on_the_fly:
# look for formatted record existence:
query = "SELECT value, last_updated FROM bibfmt WHERE "\
"id_bibrec='%s' AND format='%s'" % (recID, of)
res = run_sql(query, None, 1)
if res:
# record 'recID' is formatted in 'of', so print it
if verbose == 9:
last_updated = res[0][1]
out += """\n
Found preformatted output for record %i (cache updated on %s).
""" % (recID, last_updated)
decompress = zlib.decompress
return "%s" % decompress(res[0][0])
else:
# record 'recID' is not formatted in 'of',
# so try to call BibFormat on the fly or use default format:
if verbose == 9:
out += """\n
Formatting record %i on-the-fly with old BibFormat.
""" % recID
# Retrieve MARCXML
# Build it on-the-fly only if 'call_old_bibformat' was called
# with format=xm and on_the_fly=True
xm_record = record_get_xml(recID, 'xm',
on_the_fly=(on_the_fly and of == 'xm'))
## import platform
## # Some problem have been found using either popen() or os.system().
## # Here is a temporary workaround until the issue is solved.
## if platform.python_compiler().find('Red Hat') > -1:
## # use os.system
(result_code, result_path) = tempfile.mkstemp()
command = "( %s/bibformat otype=%s ) > %s" % \
(CFG_BINDIR, escape_shell_arg(of), result_path)
(xm_code, xm_path) = tempfile.mkstemp()
xm_file = open(xm_path, "w")
xm_file.write(xm_record)
xm_file.close()
command = command + " <" + xm_path
os.system(command)
result_file = open(result_path,"r")
bibformat_output = result_file.read()
result_file.close()
os.close(result_code)
os.remove(result_path)
os.close(xm_code)
os.remove(xm_path)
## else:
## # use popen
## pipe_input, pipe_output, pipe_error = os.popen3(["%s/bibformat" % CFG_BINDIR,
## "otype=%s" % format],
## 'rw')
## pipe_input.write(xm_record)
## pipe_input.flush()
## pipe_input.close()
## bibformat_output = pipe_output.read()
## pipe_output.close()
## pipe_error.close()
if bibformat_output.startswith(""):
dom = minidom.parseString(bibformat_output)
for e in dom.getElementsByTagName('subfield'):
if e.getAttribute('code') == 'g':
for t in e.childNodes:
out += t.data.encode('utf-8')
else:
out += bibformat_output
return out
def format_record(recID, of, ln=CFG_SITE_LANG, verbose=0,
search_pattern=None, xml_record=None, user_info=None, qid=""):
"""
Formats a record given output format. Main entry function of
bibformat engine.
Returns a formatted version of the record in the specified
language, search pattern, and with the specified output format.
The function will define which format template must be applied.
You can either specify an record ID to format, or give its xml
representation. if 'xml_record' is not None, then use it instead
of recID.
'user_info' allows to grant access to some functionalities on a
page depending on the user's priviledges. 'user_info' is the same
object as the one returned by 'webuser.collect_user_info(req)'
@param recID: the ID of record to format
@param of: an output format code (or short identifier for the output format)
@param ln: the language to use to format the record
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings, stop if error in format elements
9: errors and warnings, stop if error (debug mode ))
@param search_pattern: list of strings representing the user request in web interface
@param xml_record: an xml string representing the record to format
@param user_info: the information of the user who will view the formatted page
@return: formatted record
"""
if search_pattern is None:
search_pattern = []
out = ""
ln = wash_language(ln)
_ = gettext_set_language(ln)
# Temporary workflow (during migration of formats):
# Call new BibFormat
# But if format not found for new BibFormat, then call old BibFormat
#Create a BibFormat Object to pass that contain record and context
bfo = BibFormatObject(recID, ln, search_pattern, xml_record, user_info, of)
if of.lower() != 'xm' and \
(not bfo.get_record() or len(bfo.get_record()) <= 1):
# Record only has recid: do not format, excepted
# for xm format
return ""
#Find out which format template to use based on record and output format.
template = decide_format_template(bfo, of)
if verbose == 9 and template is not None:
out += """\n
Using %s template for record %i.
""" % (template, recID)
############### FIXME: REMOVE WHEN MIGRATION IS DONE ###############
path = "%s%s%s" % (cfg['CFG_BIBFORMAT_TEMPLATES_PATH'], os.sep, template)
if template is None or not (
os.access(path, os.R_OK) or
template.endswith("." + CFG_BIBFORMAT_FORMAT_JINJA_TEMPLATE_EXTENSION)):
# template not found in new BibFormat. Call old one
if verbose == 9:
if template is None:
out += """\n
No template found for output format %s and record %i.
(Check invenio.err log file for more details)
""" % (of, recID)
else:
out += """\n
Template %s could not be read.
""" % (template)
if CFG_PATH_PHP and os.path.isfile(os.path.join(CFG_BINDIR, 'bibformat')):
if verbose == 9:
out += """\n
Using old BibFormat for record %s.
""" % recID
return out + call_old_bibformat(recID, of=of, on_the_fly=True,
verbose=verbose)
############################# END ##################################
try:
raise InvenioBibFormatError(_('No template could be found for output format %(code)s.', code=of))
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
if verbose > 5:
return out + str(exc.message)
return out
# Format with template
out_ = format_with_format_template(template, bfo, verbose, qid=qid)
out += out_
return out
def decide_format_template(bfo, of):
"""
Returns the format template name that should be used for formatting
given output format and L{BibFormatObject}.
Look at of rules, and take the first matching one.
If no rule matches, returns None
To match we ignore lettercase and spaces before and after value of
rule and value of record
@param bfo: a L{BibFormatObject}
@param of: the code of the output format to use
@return: name of a format template
"""
output_format = get_output_format(of)
for rule in output_format['rules']:
if rule['field'].startswith('00'):
# Rule uses controlfield
values = [bfo.control_field(rule['field']).strip()] #Remove spaces
else:
# Rule uses datafield
values = bfo.fields(rule['field'])
# loop over multiple occurences, but take the first match
if len(values) > 0:
for value in values:
value = value.strip() #Remove spaces
pattern = rule['value'].strip() #Remove spaces
match_obj = re.match(pattern, value, re.IGNORECASE)
if match_obj is not None and \
match_obj.end() == len(value):
return rule['template']
template = output_format['default']
if template != '':
return template
else:
return None
def format_with_format_template(format_template_filename, bfo,
verbose=0, format_template_code=None, qid=""):
""" Format a record given a
format template.
Returns a formatted version of the record represented by bfo,
in the language specified in bfo, and with the specified format template.
If format_template_code is provided, the template will not be loaded from
format_template_filename (but format_template_filename will still be used to
determine if bft or xsl transformation applies). This allows to preview format
code without having to save file on disk.
@param format_template_filename: the dilename of a format template
@param bfo: the object containing parameters for the current formatting
@param format_template_code: if not empty, use code as template instead of reading format_template_filename (used for previews)
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: formatted text
"""
_ = gettext_set_language(bfo.lang)
def translate(match):
"""
Translate matching values
"""
word = match.group("word")
translated_word = _(word)
return translated_word
if format_template_code is not None:
format_content = str(format_template_code)
elif not format_template_filename.endswith("." + CFG_BIBFORMAT_FORMAT_JINJA_TEMPLATE_EXTENSION):
format_content = get_format_template(format_template_filename)['code']
if format_template_filename is None or \
format_template_filename.endswith("." + CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION):
# .bft
filtered_format = filter_languages(format_content, bfo.lang)
localized_format = translation_pattern.sub(translate, filtered_format)
evaluated_format = eval_format_template_elements(localized_format,
bfo,
verbose)
elif format_template_filename.endswith("." + CFG_BIBFORMAT_FORMAT_JINJA_TEMPLATE_EXTENSION):
evaluated_format = ''
#try:
from functools import wraps
- from invenio.legacy.bibfield import \
- create_record as bibfield_create_record, \
- get_record as bibfield_get_record
+ from invenio.modules.records.api import \
+ create_record as new_create_record, \
+ get_record as new_get_record
from invenio.legacy.search_engine import print_record
from flask.ext.login import current_user
from invenio.base.helpers import unicodifier
def _format_record(recid, of='hb', user_info=current_user, *args, **kwargs):
return print_record(recid, format=of, user_info=user_info, *args, **kwargs)
# Fixes unicode problems in Jinja2 templates.
def encode_utf8(f):
@wraps(f)
def wrapper(*args, **kwds):
return unicodifier(f(*args, **kwds))
return wrapper
if bfo.recID:
- record = bibfield_get_record(bfo.recID)
+ record = new_get_record(bfo.recID)
else:
- record = bibfield_create_record(bfo.xml_record, master_format='marc')
+ record = new_create_record(bfo.xml_record, master_format='marc')
bfo.recID = bfo.recID if bfo.recID else 0
record.__getitem__ = encode_utf8(record.__getitem__)
record.get = encode_utf8(record.get)
evaluated_format = render_template_to_string(
'format/record/'+format_template_filename,
recid=bfo.recID,
record=record,
format_record=_format_record,
qid=qid,
bfo=bfo, **TEMPLATE_CONTEXT_FUNCTIONS_CACHE.functions).encode('utf-8')
#except Exception:
# register_exception()
else:
#.xsl
if bfo.xml_record:
# bfo was initialized with a custom MARCXML
xml_record = '\n' + \
record_xml_output(bfo.record)
else:
# Fetch MARCXML. On-the-fly xm if we are now formatting in xm
xml_record = '\n' + \
record_get_xml(bfo.recID, 'xm', on_the_fly=False)
# Transform MARCXML using stylesheet
evaluated_format = format(xml_record, template_source=format_content)
try:
evaluated_format = evaluated_format.decode('utf8').encode('utf8')
except:
try:
evaluated_format = evaluated_format.encode('utf8')
except:
evaluated_format = ''.encode('utf8')
return evaluated_format
def eval_format_template_elements(format_template, bfo, verbose=0):
"""
Evalutes the format elements of the given template and replace each element with its value.
Prepare the format template content so that we can directly replace the marc code by their value.
This implies:
1. Look for special tags
2. replace special tags by their evaluation
@param format_template: the format template code
@param bfo: the object containing parameters for the current formatting
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors, 7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: tuple (result, errors)
"""
_ = gettext_set_language(bfo.lang)
# First define insert_element_code(match), used in re.sub() function
def insert_element_code(match):
"""
Analyses 'match', interpret the corresponding code, and return the result of the evaluation.
Called by substitution in 'eval_format_template_elements(...)'
@param match: a match object corresponding to the special tag that must be interpreted
"""
function_name = match.group("function_name")
try:
format_element = get_format_element(function_name, verbose)
except Exception, e:
format_element = None
if verbose >= 5:
return '' + \
cgi.escape(str(e)).replace('\n', '
') + \
''
if format_element is None:
try:
raise InvenioBibFormatError(
_('Could not find format element named %(function_name)s.',
function_name=function_name))
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
if verbose >= 5:
return '' + \
str(exc.message)+''
else:
params = {}
# Look for function parameters given in format template code
all_params = match.group('params')
if all_params is not None:
function_params_iterator = pattern_function_params.finditer(all_params)
for param_match in function_params_iterator:
name = param_match.group('param')
value = param_match.group('value')
params[name] = value
# Evaluate element with params and return (Do not return errors)
(result, dummy) = eval_format_element(format_element,
bfo,
params,
verbose)
return result
# Substitute special tags in the format by our own text.
# Special tags have the form
format = pattern_tag.sub(insert_element_code, format_template)
return format
def eval_format_element(format_element, bfo, parameters=None, verbose=0):
"""
Returns the result of the evaluation of the given format element
name, with given L{BibFormatObject} and parameters. Also returns
the errors of the evaluation.
@param format_element: a format element structure as returned by get_format_element
@param bfo: a L{BibFormatObject} used for formatting
@param parameters: a dict of parameters to be used for formatting. Key is parameter and value is value of parameter
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: tuple (result, errors)
"""
if parameters is None:
parameters = {}
errors = []
#Load special values given as parameters
prefix = parameters.get('prefix', "")
suffix = parameters.get('suffix', "")
default_value = parameters.get('default', "")
escape = parameters.get('escape', "")
output_text = ''
_ = gettext_set_language(bfo.lang)
# 3 possible cases:
# a) format element file is found: we execute it
# b) format element file is not found, but exist in tag table (e.g. bfe_isbn)
# c) format element is totally unknown. Do nothing or report error
if format_element is not None and format_element['type'] == "python":
# a) We found an element with the tag name, of type "python"
# Prepare a dict 'params' to pass as parameter to 'format'
# function of element
params = {}
# Look for parameters defined in format element
# Fill them with specified default values and values
# given as parameters.
# Also remember if the element overrides the 'escape'
# parameter
format_element_overrides_escape = False
for param in format_element['attrs']['params']:
name = param['name']
default = param['default']
params[name] = parameters.get(name, default)
if name == 'escape':
format_element_overrides_escape = True
# Add BibFormatObject
params['bfo'] = bfo
# Execute function with given parameters and return result.
function = format_element['code']
_ = gettext_set_language(bfo.lang)
try:
output_text = apply(function, (), params)
except Exception, e:
name = format_element['attrs']['name']
try:
raise InvenioBibFormatError(_(
'Error when evaluating format element %(format_element)s with parameters %(parameters)s.',
format_element=name, parameters=str(params)))
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
errors.append(exc.message)
if verbose >= 5:
tb = sys.exc_info()[2]
stack = traceback.format_exception(Exception, e, tb, limit=None)
output_text = ''+ \
str(exc.message) + "".join(stack) +' '
# None can be returned when evaluating function
if output_text is None:
output_text = ""
else:
output_text = str(output_text)
# Escaping:
# (1) By default, everything is escaped in mode 1
# (2) If evaluated element has 'escape_values()' function, use
# its returned value as escape mode, and override (1)
# (3) If template has a defined parameter 'escape' (in allowed
# values), use it, and override (1) and (2). If this
# 'escape' parameter is overriden by the format element
# (defined in the 'format' function of the element), leave
# the escaping job to this element
# (1)
escape_mode = 1
# (2)
escape_function = format_element['escape_function']
if escape_function is not None:
try:
escape_mode = apply(escape_function, (), {'bfo': bfo})
except Exception, e:
try:
raise InvenioBibFormatError(_('Escape mode for format element %s could not be retrieved. Using default mode instead.') % name)
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
errors.append(exc.message)
if verbose >= 5:
tb = sys.exc_info()[2]
output_text += ''+ \
str(exc.message) +' '
# (3)
if escape in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']:
escape_mode = int(escape)
# If escape is equal to 1, then escape all
# HTML reserved chars.
if escape_mode > 0 and not format_element_overrides_escape:
output_text = escape_field(output_text, mode=escape_mode)
# Add prefix and suffix if they have been given as parameters and if
# the evaluation of element is not empty
if output_text.strip() != "":
output_text = prefix + output_text + suffix
# Add the default value if output_text is empty
if output_text == "":
output_text = default_value
return (output_text, errors)
elif format_element is not None and format_element['type'] == "field":
# b) We have not found an element in files that has the tag
# name. Then look for it in the table "tag"
#
#
#
# Load special values given as parameters
separator = parameters.get('separator ', "")
nbMax = parameters.get('nbMax', "")
escape = parameters.get('escape', "1") # By default, escape here
# Get the fields tags that have to be printed
tags = format_element['attrs']['tags']
output_text = []
# Get values corresponding to tags
for tag in tags:
p_tag = parse_tag(tag)
values = record_get_field_values(bfo.get_record(),
p_tag[0],
p_tag[1],
p_tag[2],
p_tag[3])
if len(values)>0 and isinstance(values[0], dict):
#flatten dict to its values only
values_list = map(lambda x: x.values(), values)
#output_text.extend(values)
for values in values_list:
output_text.extend(values)
else:
output_text.extend(values)
if nbMax != "":
try:
nbMax = int(nbMax)
output_text = output_text[:nbMax]
except:
name = format_element['attrs']['name']
try:
raise InvenioBibFormatError(_('"nbMax" parameter for %s must be an "int".') % name)
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
errors.append(exc.message)
if verbose >= 5:
output_text = output_text.append(exc.message)
# Add prefix and suffix if they have been given as parameters and if
# the evaluation of element is not empty.
# If evaluation is empty string, return default value if it exists.
# Else return empty string
if ("".join(output_text)).strip() != "":
# If escape is equal to 1, then escape all
# HTML reserved chars.
if escape == '1':
output_text = cgi.escape(separator.join(output_text))
else:
output_text = separator.join(output_text)
output_text = prefix + output_text + suffix
else:
#Return default value
output_text = default_value
return (output_text, errors)
else:
# c) Element is unknown
try:
raise InvenioBibFormatError(_('Could not find format element named %(format_element)s.', format_element=format_element))
except InvenioBibFormatError, exc:
register_exception(req=bfo.req)
errors.append(exc.message)
if verbose < 5:
return ("", errors)
elif verbose >= 5:
if verbose >= 9:
sys.exit(exc.message)
return ('' + \
str(exc.message)+'', errors)
def filter_languages(format_template, ln='en'):
"""
Filters the language tags that do not correspond to the specified language.
@param format_template: the format template code
@param ln: the language that is NOT filtered out from the template
@return: the format template with unnecessary languages filtered out
"""
# First define search_lang_tag(match) and clean_language_tag(match), used
# in re.sub() function
def search_lang_tag(match):
"""
Searches for the ... tag and remove inner localized tags
such as , , that are not current_lang.
If current_lang cannot be found inside ... , try to use 'CFG_SITE_LANG'
@param match: a match object corresponding to the special tag that must be interpreted
"""
current_lang = ln
def clean_language_tag(match):
"""
Return tag text content if tag language of match is output language.
Called by substitution in 'filter_languages(...)'
@param match: a match object corresponding to the special tag that must be interpreted
"""
if match.group(1) == current_lang:
return match.group(2)
else:
return ""
# End of clean_language_tag
lang_tag_content = match.group("langs")
# Try to find tag with current lang. If it does not exists,
# then current_lang becomes CFG_SITE_LANG until the end of this
# replace
pattern_current_lang = re.compile(r"<("+current_lang+ \
r")\s*>(.*?)()", re.IGNORECASE | re.DOTALL)
if re.search(pattern_current_lang, lang_tag_content) is None:
current_lang = CFG_SITE_LANG
cleaned_lang_tag = ln_pattern.sub(clean_language_tag, lang_tag_content)
return cleaned_lang_tag
# End of search_lang_tag
filtered_format_template = pattern_lang.sub(search_lang_tag, format_template)
return filtered_format_template
def get_format_template(filename, with_attributes=False):
"""
Returns the structured content of the given formate template.
if 'with_attributes' is true, returns the name and description. Else 'attrs' is not
returned as key in dictionary (it might, if it has already been loaded previously)::
{'code':"Some template code"
'attrs': {'name': "a name", 'description': "a description"}
}
@param filename: the filename of an format template
@param with_attributes: if True, fetch the attributes (names and description) for format'
@return: strucured content of format template
"""
_ = gettext_set_language(CFG_SITE_LANG)
# Get from cache whenever possible
global format_templates_cache
if not filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION) and \
not filename.endswith(".xsl"):
return None
if format_templates_cache.has_key(filename):
# If we must return with attributes and template exist in
# cache with attributes then return cache.
# Else reload with attributes
if with_attributes and \
format_templates_cache[filename].has_key('attrs'):
return format_templates_cache[filename]
format_template = {'code': ""}
try:
path = registry.format_templates_lookup[filename]
format_file = open(path)
format_content = format_file.read()
format_file.close()
# Load format template code
# Remove name and description
if filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION):
code_and_description = pattern_format_template_name.sub("",
format_content, 1)
code = pattern_format_template_desc.sub("", code_and_description, 1)
else:
code = format_content
format_template['code'] = code
except Exception, e:
try:
raise InvenioBibFormatError(_('Could not read format template named %(filename)s. %(error)s.', filename=filename, error=str(e)))
except InvenioBibFormatError:
register_exception()
# Save attributes if necessary
if with_attributes:
format_template['attrs'] = get_format_template_attrs(filename)
# Cache and return
format_templates_cache[filename] = format_template
return format_template
def get_format_templates(with_attributes=False):
"""
Returns the list of all format templates, as dictionary with filenames as keys
if 'with_attributes' is true, returns the name and description. Else 'attrs' is not
returned as key in each dictionary (it might, if it has already been loaded previously)::
[{'code':"Some template code"
'attrs': {'name': "a name", 'description': "a description"}
},
...
}
@param with_attributes: if True, fetch the attributes (names and description) for formats
@return: the list of format templates (with code and info)
"""
format_templates = {}
for filename in registry.format_templates:
if filename.endswith("."+CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION) or \
filename.endswith(".xsl"):
filename = os.path.basename(filename)
format_templates[filename] = get_format_template(filename,
with_attributes)
return format_templates
def get_format_template_attrs(filename):
"""
Returns the attributes of the format template with given filename
The attributes are {'name', 'description'}
Caution: the function does not check that path exists or
that the format element is valid.
@param filename: the name of a format template
@return: a structure with detailed information about given format template
"""
_ = gettext_set_language(CFG_SITE_LANG)
attrs = {}
attrs['name'] = ""
attrs['description'] = ""
try:
template_file = open(registry.format_templates_lookup[filename])
code = template_file.read()
template_file.close()
match = None
if filename.endswith(".xsl"):
# .xsl
attrs['name'] = filename[:-4]
else:
# .bft
match = pattern_format_template_name.search(code)
if match is not None:
attrs['name'] = match.group('name')
else:
attrs['name'] = filename
match = pattern_format_template_desc.search(code)
if match is not None:
attrs['description'] = match.group('desc').rstrip('.')
except Exception, e:
try:
raise InvenioBibFormatError(_('Could not read format template named %(filename)s. %(error)s.', filename=filename, error=str(e)))
except InvenioBibFormatError:
register_exception()
attrs['name'] = filename
return attrs
def get_format_element(element_name, verbose=0, with_built_in_params=False):
"""
Returns the format element structured content.
Return None if element cannot be loaded (file not found, not readable or
invalid)
The returned structure is::
{'attrs': {some attributes in dict. See get_format_element_attrs_from_*}
'code': the_function_code,
'type':"field" or "python" depending if element is defined in file or table,
'escape_function': the function to call to know if element output must be escaped}
@param element_name: the name of the format element to load
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@param with_built_in_params: if True, load the parameters built in all elements
@return: a dictionary with format element attributes
"""
_ = gettext_set_language(CFG_SITE_LANG)
# Get from cache whenever possible
global format_elements_cache
errors = []
# Resolve filename and prepare 'name' as key for the cache
filename = resolve_format_element_filename(element_name)
if filename is not None:
name = filename.upper()
else:
name = element_name.upper()
if format_elements_cache.has_key(name):
element = format_elements_cache[name]
if not with_built_in_params or \
(with_built_in_params and \
element['attrs'].has_key('builtin_params')):
return element
if filename is None:
# Element is maybe in tag table
if bibformat_dblayer.tag_exists_for_name(element_name):
format_element = {'attrs': get_format_element_attrs_from_table( \
element_name,
with_built_in_params),
'code':None,
'escape_function':None,
'type':"field"}
# Cache and returns
format_elements_cache[name] = format_element
return format_element
else:
try:
raise InvenioBibFormatError(_('Format element %(element_name)s could not be found.', element_name=element_name))
except InvenioBibFormatError, exc:
register_exception()
if verbose >= 5:
sys.stderr.write(exc.message)
return None
else:
format_element = {}
module_name = filename
if module_name.endswith(".py"):
module_name = module_name[:-3]
# Load function 'format_element()' inside element
try:
function_format = TEMPLATE_CONTEXT_FUNCTIONS_CACHE.\
bibformat_elements()[module_name]
format_element['code'] = function_format
except KeyError:
try:
raise InvenioBibFormatError(_('Format element %(element_name)s has no function named "format".', element_name=element_name))
except InvenioBibFormatError, exc:
register_exception()
errors.append(exc.message)
if verbose >= 5:
sys.stderr.write(exc.message)
if errors:
if verbose >= 7:
raise Exception, exc.message
return None
# Load function 'escape_values()' inside element
format_element['escape_function'] = function_format._escape_values
# Prepare, cache and return
format_element['attrs'] = get_format_element_attrs_from_function( \
function_format,
element_name,
with_built_in_params)
format_element['type'] = "python"
format_elements_cache[name] = format_element
return format_element
def get_format_elements(with_built_in_params=False):
"""
Returns the list of format elements attributes as dictionary structure
Elements declared in files have priority over element declared in 'tag' table
The returned object has this format::
{element_name1: {'attrs': {'description':..., 'seealso':...
'params':[{'name':..., 'default':..., 'description':...}, ...]
'builtin_params':[{'name':..., 'default':..., 'description':...}, ...]
},
'code': code_of_the_element
},
element_name2: {...},
...}
Returns only elements that could be loaded (not error in code)
@return: a dict of format elements with name as key, and a dict as attributes
@param with_built_in_params: if True, load the parameters built in all elements
"""
format_elements = {}
mappings = bibformat_dblayer.get_all_name_tag_mappings()
for name in mappings:
format_elements[name.upper().replace(" ", "_").strip()] = get_format_element(name, with_built_in_params=with_built_in_params)
for module in registry.format_elements:
filename = os.path.basename(module.__file__)
filename_test = filename.upper().replace(" ", "_")
if filename_test.endswith(".PYC"):
filename_test = filename_test[:-1]
if filename_test.endswith(".PY") and not filename.upper().startswith("__INIT__.PY"):
if filename_test.startswith("BFE_"):
filename_test = filename_test[4:]
element_name = filename_test[:-3]
element = get_format_element(element_name,
with_built_in_params=with_built_in_params)
if element is not None:
format_elements[element_name] = element
return format_elements
def get_format_element_attrs_from_function(function, element_name,
with_built_in_params=False):
"""
Returns the attributes of the function given as parameter.
It looks for standard parameters of the function, default
values and comments in the docstring.
The attributes are::
{'name' : "name of element" #basically the name of 'name' parameter
'description': "a string description of the element",
'seealso' : ["element_1.py", "element_2.py", ...] #a list of related elements
'params': [{'name':"param_name", #a list of parameters for this element (except 'bfo')
'default':"default value",
'description': "a description"}, ...],
'builtin_params': {name: {'name':"param_name",#the parameters builtin for all elem of this kind
'default':"default value",
'description': "a description"}, ...},
}
@param function: the formatting function of a format element
@param element_name: the name of the element
@param with_built_in_params: if True, load the parameters built in all elements
@return: a structure with detailed information of a function
"""
attrs = {}
attrs['description'] = ""
attrs['name'] = element_name.replace(" ", "_").upper()
attrs['seealso'] = []
docstring = function.__doc__
if isinstance(docstring, str):
# Look for function description in docstring
#match = pattern_format_element_desc.search(docstring)
description = docstring.split("@param")[0]
description = description.split("@see:")[0]
attrs['description'] = description.strip().rstrip('.')
# Look for @see: in docstring
match = pattern_format_element_seealso.search(docstring)
if match is not None:
elements = match.group('see').rstrip('.').split(",")
for element in elements:
attrs['seealso'].append(element.strip())
params = {}
# Look for parameters in function definition
(args, varargs, varkw, defaults) = inspect.getargspec(function)
# Prepare args and defaults_list such that we can have a mapping
# from args to defaults
args.reverse()
if defaults is not None:
defaults_list = list(defaults)
defaults_list.reverse()
else:
defaults_list = []
for arg, default in map(None, args, defaults_list):
if arg == "bfo":
#Don't keep this as parameter. It is hidden to users, and
#exists in all elements of this kind
continue
param = {}
param['name'] = arg
if default is None:
#In case no check is made inside element, we prefer to
#print "" (nothing) than None in output
param['default'] = ""
else:
param['default'] = default
param['description'] = "(no description provided)"
params[arg] = param
if isinstance(docstring, str):
# Look for AT param descriptions in docstring.
# Add description to existing parameters in params dict
params_iterator = pattern_format_element_params.finditer(docstring)
for match in params_iterator:
name = match.group('name')
if params.has_key(name):
params[name]['description'] = match.group('desc').rstrip('.')
attrs['params'] = params.values()
# Load built-in parameters if necessary
if with_built_in_params:
builtin_params = []
# Add 'prefix' parameter
param_prefix = {}
param_prefix['name'] = "prefix"
param_prefix['default'] = ""
param_prefix['description'] = """A prefix printed only if the
record has a value for this element"""
builtin_params.append(param_prefix)
# Add 'suffix' parameter
param_suffix = {}
param_suffix['name'] = "suffix"
param_suffix['default'] = ""
param_suffix['description'] = """A suffix printed only if the
record has a value for this element"""
builtin_params.append(param_suffix)
# Add 'default' parameter
param_default = {}
param_default['name'] = "default"
param_default['default'] = ""
param_default['description'] = """A default value printed if the
record has no value for this element"""
builtin_params.append(param_default)
# Add 'escape' parameter
param_escape = {}
param_escape['name'] = "escape"
param_escape['default'] = ""
param_escape['description'] = """0 keeps value as it is. Refer to main
documentation for escaping modes
1 to 7"""
builtin_params.append(param_escape)
attrs['builtin_params'] = builtin_params
return attrs
def get_format_element_attrs_from_table(element_name,
with_built_in_params=False):
"""
Returns the attributes of the format element with given name in 'tag' table.
Returns None if element_name does not exist in tag table.
The attributes are::
{'name' : "name of element" #basically the name of 'element_name' parameter
'description': "a string description of the element",
'seealso' : [] #a list of related elements. Always empty in this case
'params': [], #a list of parameters for this element. Always empty in this case
'builtin_params': [{'name':"param_name", #the parameters builtin for all elem of this kind
'default':"default value",
'description': "a description"}, ...],
'tags':["950.1", 203.a] #the list of tags printed by this element
}
@param element_name: an element name in database
@param element_name: the name of the element
@param with_built_in_params: if True, load the parameters built in all elements
@return: a structure with detailed information of an element found in DB
"""
attrs = {}
tags = bibformat_dblayer.get_tags_from_name(element_name)
field_label = "field"
if len(tags)>1:
field_label = "fields"
attrs['description'] = "Prints %s %s of the record" % (field_label,
", ".join(tags))
attrs['name'] = element_name.replace(" ", "_").upper()
attrs['seealso'] = []
attrs['params'] = []
attrs['tags'] = tags
# Load built-in parameters if necessary
if with_built_in_params:
builtin_params = []
# Add 'prefix' parameter
param_prefix = {}
param_prefix['name'] = "prefix"
param_prefix['default'] = ""
param_prefix['description'] = """A prefix printed only if the
record has a value for this element"""
builtin_params.append(param_prefix)
# Add 'suffix' parameter
param_suffix = {}
param_suffix['name'] = "suffix"
param_suffix['default'] = ""
param_suffix['description'] = """A suffix printed only if the
record has a value for this element"""
builtin_params.append(param_suffix)
# Add 'separator' parameter
param_separator = {}
param_separator['name'] = "separator"
param_separator['default'] = " "
param_separator['description'] = """A separator between elements of
the field"""
builtin_params.append(param_separator)
# Add 'nbMax' parameter
param_nbMax = {}
param_nbMax['name'] = "nbMax"
param_nbMax['default'] = ""
param_nbMax['description'] = """The maximum number of values to
print for this element. No limit if not
specified"""
builtin_params.append(param_nbMax)
# Add 'default' parameter
param_default = {}
param_default['name'] = "default"
param_default['default'] = ""
param_default['description'] = """A default value printed if the
record has no value for this element"""
builtin_params.append(param_default)
# Add 'escape' parameter
param_escape = {}
param_escape['name'] = "escape"
param_escape['default'] = ""
param_escape['description'] = """If set to 1, replaces special
characters '&', '<' and '>' of this
element by SGML entities"""
builtin_params.append(param_escape)
attrs['builtin_params'] = builtin_params
return attrs
def get_output_format(code, with_attributes=False, verbose=0):
"""
Returns the structured content of the given output format
If 'with_attributes' is true, also returns the names and description of the output formats,
else 'attrs' is not returned in dict (it might, if it has already been loaded previously).
if output format corresponding to 'code' is not found return an empty structure.
See get_output_format_attrs() to learn more about the attributes::
{'rules': [ {'field': "980__a",
'value': "PREPRINT",
'template': "filename_a.bft",
},
{...}
],
'attrs': {'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}}
'description': "a description"
'code': "fnm1",
'content_type': "application/ms-excel",
'visibility': 1
}
'default':"filename_b.bft"
}
@param code: the code of an output_format
@param with_attributes: if True, fetch the attributes (names and description) for format
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: strucured content of output format
"""
_ = gettext_set_language(CFG_SITE_LANG)
output_format = {'rules':[], 'default':""}
filename = resolve_output_format_filename(code, verbose)
if filename is None:
try:
raise InvenioBibFormatError(_('Output format with code %(code)s could not be found.', code=code))
except InvenioBibFormatError, exc:
register_exception()
if with_attributes: #Create empty attrs if asked for attributes
output_format['attrs'] = get_output_format_attrs(code, verbose)
return output_format
# Get from cache whenever possible
global format_outputs_cache
if format_outputs_cache.has_key(filename):
# If was must return with attributes but cache has not
# attributes, then load attributes
if with_attributes and not \
format_outputs_cache[filename].has_key('attrs'):
format_outputs_cache[filename]['attrs'] = get_output_format_attrs(code, verbose)
return format_outputs_cache[filename]
try:
if with_attributes:
output_format['attrs'] = get_output_format_attrs(code, verbose)
path = "%s%s%s" % (CFG_BIBFORMAT_OUTPUTS_PATH, os.sep, filename )
format_file = open(path)
current_tag = ''
for line in format_file:
line = line.strip()
if line == "":
# Ignore blank lines
continue
if line.endswith(":"):
# Retrieve tag
# Remove : spaces and eol at the end of line
clean_line = line.rstrip(": \n\r")
# The tag starts at second position
current_tag = "".join(clean_line.split()[1:]).strip()
elif line.find('---') != -1:
words = line.split('---')
template = words[-1].strip()
condition = ''.join(words[:-1])
value = ""
output_format['rules'].append({'field': current_tag,
'value': condition,
'template': template,
})
elif line.find(':') != -1:
# Default case
default = line.split(':')[1].strip()
output_format['default'] = default
except Exception, e:
try:
raise InvenioBibFormatError(_('Output format %(filename)s cannot not be read. %(error)s.', filename=filename, error=str(e)))
except InvenioBibFormatError, exc:
register_exception()
# Cache and return
format_outputs_cache[filename] = output_format
return output_format
def get_output_format_attrs(code, verbose=0):
"""
Returns the attributes of an output format.
The attributes contain 'code', which is the short identifier of the output format
(to be given as parameter in format_record function to specify the output format),
'description', a description of the output format, 'visibility' the visibility of
the format in the output format list on public pages and 'names', the localized names
of the output format. If 'content_type' is specified then the search_engine will
send a file with this content type and with result of formatting as content to the user.
The 'names' dict always contais 'generic', 'ln' (for long name) and 'sn' (for short names)
keys. 'generic' is the default name for output format. 'ln' and 'sn' contain long and short
localized names of the output format. Only the languages for which a localization exist
are used::
{'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}}
'description': "a description"
'code': "fnm1",
'content_type': "application/ms-excel",
'visibility': 1
}
@param code: the short identifier of the format
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: strucured content of output format attributes
"""
if code.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION):
code = code[:-(len(CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION) + 1)]
attrs = {'names':{'generic':"",
'ln':{},
'sn':{}},
'description':'',
'code':code.upper(),
'content_type':"",
'visibility':1}
filename = resolve_output_format_filename(code, verbose)
if filename is None:
return attrs
attrs['names'] = bibformat_dblayer.get_output_format_names(code)
attrs['description'] = bibformat_dblayer.get_output_format_description(code)
attrs['content_type'] = bibformat_dblayer.get_output_format_content_type(code)
attrs['visibility'] = bibformat_dblayer.get_output_format_visibility(code)
return attrs
def get_output_formats(with_attributes=False):
"""
Returns the list of all output format, as a dictionary with their filename as key
If 'with_attributes' is true, also returns the names and description of the output formats,
else 'attrs' is not returned in dicts (it might, if it has already been loaded previously).
See get_output_format_attrs() to learn more on the attributes::
{'filename_1.bfo': {'rules': [ {'field': "980__a",
'value': "PREPRINT",
'template': "filename_a.bft",
},
{...}
],
'attrs': {'names': {'generic':"a name", 'sn':{'en': "a name", 'fr':"un nom"}, 'ln':{'en':"a long name"}}
'description': "a description"
'code': "fnm1"
}
'default':"filename_b.bft"
},
'filename_2.bfo': {...},
...
}
@param with_attributes: if returned output formats contain detailed info, or not
@type with_attributes: boolean
@return: the list of output formats
"""
output_formats = {}
files = os.listdir(CFG_BIBFORMAT_OUTPUTS_PATH)
for filename in files:
if filename.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION):
code = "".join(filename.split(".")[:-1])
output_formats[filename] = get_output_format(code, with_attributes)
return output_formats
def resolve_format_element_filename(element_name):
"""
Returns the filename of element corresponding to x{element_name}
This is necessary since format templates code call
elements by ignoring case, for eg. is the
same as .
It is also recommended that format elements filenames are
prefixed with bfe_ . We need to look for these too.
The name of the element has to start with "BFE_".
@param element_name: a name for a format element
@return: the corresponding filename, with right case
"""
if not element_name.endswith(".py"):
name = element_name.replace(" ", "_").upper() +".PY"
else:
name = element_name.replace(" ", "_").upper()
files = registry.format_elements
for element in files:
filename = element.__file__
if filename.endswith('.pyc'):
filename = filename[:-1]
basename = os.path.basename(filename)
test_filename = basename.replace(" ", "_").upper()
if test_filename == name or \
test_filename == "BFE_" + name or \
"BFE_" + test_filename == name:
return basename
# No element with that name found
# Do not log error, as it might be a normal execution case:
# element can be in database
return None
def resolve_output_format_filename(code, verbose=0):
"""
Returns the filename of output corresponding to code
This is necessary since output formats names are not case sensitive
but most file systems are.
@param code: the code for an output format
@param verbose: the level of verbosity from 0 to 9 (O: silent,
5: errors,
7: errors and warnings,
9: errors and warnings, stop if error (debug mode ))
@return: the corresponding filename, with right case, or None if not found
"""
_ = gettext_set_language(CFG_SITE_LANG)
#Remove non alphanumeric chars (except . and _)
code = re.sub(r"[^.0-9a-zA-Z_]", "", code)
if not code.endswith("."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION):
code = re.sub(r"\W", "", code)
code += "."+CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION
files = os.listdir(CFG_BIBFORMAT_OUTPUTS_PATH)
for filename in files:
if filename.upper() == code.upper():
return filename
# No output format with that name found
try:
raise InvenioBibFormatError(_('Could not find output format named %(code)s.', code=code))
except InvenioBibFormatError, exc:
register_exception()
if verbose >= 5:
sys.stderr.write(exc.message)
if verbose >= 9:
sys.exit(exc.message)
return None
def get_fresh_format_template_filename(name):
"""
Returns a new filename and name for template with given name.
Used when writing a new template to a file, so that the name
has no space, is unique in template directory
Returns (unique_filename, modified_name)
@param name: name for a format template
@return: the corresponding filename, and modified name if necessary
"""
#name = re.sub(r"\W", "", name) #Remove non alphanumeric chars
name = name.replace(" ", "_")
filename = name
# Remove non alphanumeric chars (except .)
filename = re.sub(r"[^.0-9a-zA-Z]", "", filename)
index = 1
def _get_fullname(filename):
return filename + '.' + CFG_BIBFORMAT_FORMAT_TEMPLATE_EXTENSION
while _get_fullname(filename) in registry.format_templates_lookup:
index += 1
filename = name + str(index)
if index > 1:
returned_name = (name + str(index)).replace("_", " ")
else:
returned_name = name.replace("_", " ")
return (_get_fullname(filename), returned_name)
def get_fresh_output_format_filename(code):
"""
Returns a new filename for output format with given code.
Used when writing a new output format to a file, so that the code
has no space, is unique in output format directory. The filename
also need to be at most 6 chars long, as the convention is that
filename == output format code (+ .extension)
We return an uppercase code
Returns (unique_filename, modified_code)
@param code: the code of an output format
@return: the corresponding filename, and modified code if necessary
"""
_ = gettext_set_language(CFG_SITE_LANG)
#code = re.sub(r"\W", "", code) #Remove non alphanumeric chars
code = code.upper().replace(" ", "_")
# Remove non alphanumeric chars (except . and _)
code = re.sub(r"[^.0-9a-zA-Z_]", "", code)
if len(code) > 6:
code = code[:6]
filename = code
path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename \
+ "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION
index = 2
while os.path.exists(path):
filename = code + str(index)
if len(filename) > 6:
filename = code[:-(len(str(index)))]+str(index)
index += 1
path = CFG_BIBFORMAT_OUTPUTS_PATH + os.sep + filename \
+ "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION
# We should not try more than 99999... Well I don't see how we
# could get there.. Sanity check.
if index >= 99999:
try:
raise InvenioBibFormatError(_('Could not find a fresh name for output format %s.') % code)
except InvenioBibFormatError, exc:
register_exception()
sys.exit("Output format cannot be named as %s"%code)
return (filename + "." + CFG_BIBFORMAT_FORMAT_OUTPUT_EXTENSION, filename)
def clear_caches():
"""
Clear the caches (Output Format, Format Templates and Format Elements)
@return: None
"""
global format_templates_cache, format_elements_cache, format_outputs_cache
format_templates_cache = {}
format_elements_cache = {}
format_outputs_cache = {}
class BibFormatObject:
"""
An object that encapsulates a record and associated methods, and that is given
as parameter to all format elements 'format' function.
The object is made specifically for a given formatting, i.e. it includes
for example the language for the formatting.
The object provides basic accessors to the record. For full access, one can get
the record with get_record() and then use BibRecord methods on the returned object.
"""
# The record
record = None
# The language in which the formatting has to be done
lang = CFG_SITE_LANG
# A list of string describing the context in which the record has
# to be formatted.
# It represents the words of the user request in web interface search
search_pattern = []
# The id of the record
recID = 0
# The information about the user, as returned by
# 'webuser.collect_user_info(req)'
user_info = None
# The format in which the record is being formatted
output_format = ''
req = None # DEPRECATED: use bfo.user_info instead. Used by WebJournal.
def __init__(self, recID, ln=CFG_SITE_LANG, search_pattern=None,
xml_record=None, user_info=None, output_format=''):
"""
Creates a new bibformat object, with given record.
You can either specify an record ID to format, or give its xml representation.
if 'xml_record' is not None, use 'xml_record' instead of recID for the record.
'user_info' allows to grant access to some functionalities on
a page depending on the user's priviledges. It is a dictionary
in the following form::
user_info = {
'remote_ip' : '',
'remote_host' : '',
'referer' : '',
'uri' : '',
'agent' : '',
'uid' : -1,
'nickname' : '',
'email' : '',
'group' : [],
'guest' : '1'
}
@param recID: the id of a record
@param ln: the language in which the record has to be formatted
@param search_pattern: list of string representing the request used by the user in web interface
@param xml_record: a xml string of the record to format
@param user_info: the information of the user who will view the formatted page
@param output_format: the output_format used for formatting this record
"""
self.xml_record = None # *Must* remain empty if recid is given
if xml_record is not None:
# If record is given as parameter
self.xml_record = xml_record
self.record = create_record(xml_record)[0]
recID = record_get_field_value(self.record, "001")
self.lang = wash_language(ln)
if search_pattern is None:
search_pattern = []
self.search_pattern = search_pattern
self.recID = recID
self.output_format = output_format
self.user_info = user_info
if self.user_info is None:
from invenio.ext.login.legacy_user import UserInfo
self.user_info = UserInfo(None)
def get_record(self):
"""
Returns the record structure of this L{BibFormatObject} instance
@return: the record structure as defined by BibRecord library
"""
from invenio.legacy.search_engine import get_record
# Create record if necessary
if self.record is None:
# on-the-fly creation if current output is xm
self.record = get_record(self.recID)
return self.record
def control_field(self, tag, escape=0):
"""
Returns the value of control field given by tag in record
@param tag: the marc code of a field
@param escape: 1 if returned value should be escaped. Else 0.
@return: value of field tag in record
"""
if self.get_record() is None:
#Case where BibRecord could not parse object
return ''
p_tag = parse_tag(tag)
field_value = record_get_field_value(self.get_record(),
p_tag[0],
p_tag[1],
p_tag[2],
p_tag[3])
if escape == 0:
return field_value
else:
return escape_field(field_value, escape)
def field(self, tag, escape=0):
"""
Returns the value of the field corresponding to tag in the
current record.
If the value does not exist, return empty string. Else
returns the same as bfo.fields(..)[0] (see docstring below).
'escape' parameter allows to escape special characters
of the field. The value of escape can be:
0. no escaping
1. escape all HTML characters
2. remove unsafe HTML tags (Eg. keep
)
3. Mix of mode 1 and 2. If value of field starts with
, then use mode 2. Else use mode 1.
4. Remove all HTML tags
5. Same as 2, with more tags allowed (like ![]()
)
6. Same as 3, with more tags allowed (like ![]()
)
7. Mix of mode 0 and mode 1. If field_value
starts with , then use mode 0.
Else use mode 1.
8. Same as mode 1, but also escape double-quotes
9. Same as mode 4, but also escape double-quotes
@param tag: the marc code of a field
@param escape: 1 if returned value should be escaped. Else 0. (see above for other modes)
@return: value of field tag in record
"""
list_of_fields = self.fields(tag)
if len(list_of_fields) > 0:
# Escaping below
if escape == 0:
return list_of_fields[0]
else:
return escape_field(list_of_fields[0], escape)
else:
return ""
def fields(self, tag, escape=0, repeatable_subfields_p=False):
"""
Returns the list of values corresonding to "tag".
If tag has an undefined subcode (such as 999C5),
the function returns a list of dictionaries, whoose keys
are the subcodes and the values are the values of tag.subcode.
If the tag has a subcode, simply returns list of values
corresponding to tag.
Eg. for given MARC::
999C5 $a value_1a $b value_1b
999C5 $b value_2b
999C5 $b value_3b $b value_3b_bis
>>> bfo.fields('999C5b')
>>> ['value_1b', 'value_2b', 'value_3b', 'value_3b_bis']
>>> bfo.fields('999C5')
>>> [{'a':'value_1a', 'b':'value_1b'},
{'b':'value_2b'},
{'b':'value_3b'}]
By default the function returns only one value for each
subfield (that is it considers that repeatable subfields are
not allowed). It is why in the above example 'value3b_bis' is
not shown for bfo.fields('999C5'). (Note that it is not
defined which of value_3b or value_3b_bis is returned). This
is to simplify the use of the function, as most of the time
subfields are not repeatable (in that way we get a string
instead of a list). You can allow repeatable subfields by
setting 'repeatable_subfields_p' parameter to True. In
this mode, the above example would return:
>>> bfo.fields('999C5b', repeatable_subfields_p=True)
>>> ['value_1b', 'value_2b', 'value_3b']
>>> bfo.fields('999C5', repeatable_subfields_p=True)
>>> [{'a':['value_1a'], 'b':['value_1b']},
{'b':['value_2b']},
{'b':['value_3b', 'value3b_bis']}]
NOTICE THAT THE RETURNED STRUCTURE IS DIFFERENT. Also note
that whatever the value of 'repeatable_subfields_p' is,
bfo.fields('999C5b') always show all fields, even repeatable
ones. This is because the parameter has no impact on the
returned structure (it is always a list).
'escape' parameter allows to escape special characters
of the fields. The value of escape can be:
0. No escaping
1. Escape all HTML characters
2. Remove unsafe HTML tags (Eg. keep
)
3. Mix of mode 1 and 2. If value of field starts with
, then use mode 2. Else use mode 1.
4. Remove all HTML tags
5. Same as 2, with more tags allowed (like ![]()
)
6. Same as 3, with more tags allowed (like ![]()
)
7. Mix of mode 0 and mode 1. If field_value
starts with , then use mode 0.
Else use mode 1.
8. Same as mode 1, but also escape double-quotes
9. Same as mode 4, but also escape double-quotes
@param tag: the marc code of a field
@param escape: 1 if returned values should be escaped. Else 0.
@repeatable_subfields_p if True, returns the list of subfields in the dictionary
@return: values of field tag in record
"""
if self.get_record() is None:
# Case where BibRecord could not parse object
return []
p_tag = parse_tag(tag)
if p_tag[3] != "":
# Subcode has been defined. Simply returns list of values
values = record_get_field_values(self.get_record(),
p_tag[0],
p_tag[1],
p_tag[2],
p_tag[3])
if escape == 0:
return values
else:
return [escape_field(value, escape) for value in values]
else:
# Subcode is undefined. Returns list of dicts.
# However it might be the case of a control field.
instances = record_get_field_instances(self.get_record(),
p_tag[0],
p_tag[1],
p_tag[2])
if repeatable_subfields_p:
list_of_instances = []
for instance in instances:
instance_dict = {}
for subfield in instance[0]:
if not instance_dict.has_key(subfield[0]):
instance_dict[subfield[0]] = []
if escape == 0:
instance_dict[subfield[0]].append(subfield[1])
else:
instance_dict[subfield[0]].append(escape_field(subfield[1], escape))
list_of_instances.append(instance_dict)
return list_of_instances
else:
if escape == 0:
return [dict(instance[0]) for instance in instances]
else:
return [dict([ (subfield[0], escape_field(subfield[1], escape)) \
for subfield in instance[0] ]) \
for instance in instances]
def kb(self, kb, string, default=""):
"""
Returns the value of the "string" in the knowledge base "kb".
If kb does not exist or string does not exist in kb,
returns 'default' string or empty string if not specified.
@param kb: a knowledge base name
@param string: the string we want to translate
@param default: a default value returned if 'string' not found in 'kb'
@return: a string value corresponding to translated input with given kb
"""
if not string:
return default
val = get_kbr_values(kb, searchkey=string, searchtype='e')
try:
return val[0][0]
except:
return default
def escape_field(value, mode=0):
"""
Utility function used to escape the value of a field in given mode.
- mode 0: no escaping
- mode 1: escaping all HTML/XML characters (escaped chars are shown as escaped)
- mode 2: escaping unsafe HTML tags to avoid XSS, but
keep basic one (such as
)
Escaped tags are removed.
- mode 3: mix of mode 1 and mode 2. If field_value starts with ,
then use mode 2. Else use mode 1.
- mode 4: escaping all HTML/XML tags (escaped tags are removed)
- mode 5: same as 2, but allows more tags, like ![]()
- mode 6: same as 3, but allows more tags, like ![]()
- mode 7: mix of mode 0 and mode 1. If field_value starts with ,
then use mode 0. Else use mode 1.
- mode 8: same as mode 1, but also escape double-quotes
- mode 9: same as mode 4, but also escape double-quotes
@param value: value to escape
@param mode: escaping mode to use
@return: an escaped version of X{value} according to chosen X{mode}
"""
if mode == 1:
return cgi.escape(value)
elif mode == 8:
return cgi.escape(value, True)
elif mode in [2, 5]:
allowed_attribute_whitelist = CFG_HTML_BUFFER_ALLOWED_ATTRIBUTE_WHITELIST
allowed_tag_whitelist = CFG_HTML_BUFFER_ALLOWED_TAG_WHITELIST + \
('class',)
if mode == 5:
allowed_attribute_whitelist += ('src', 'alt',
'width', 'height',
'style', 'summary',
'border', 'cellspacing',
'cellpadding')
allowed_tag_whitelist += ('img', 'table', 'td',
'tr', 'th', 'span', 'caption')
try:
return washer.wash(value,
allowed_attribute_whitelist=\
allowed_attribute_whitelist,
allowed_tag_whitelist= \
allowed_tag_whitelist
)
except HTMLParseError:
# Parsing failed
return cgi.escape(value)
elif mode in [3, 6]:
if value.lstrip(' \n').startswith(html_field):
allowed_attribute_whitelist = CFG_HTML_BUFFER_ALLOWED_ATTRIBUTE_WHITELIST
allowed_tag_whitelist = CFG_HTML_BUFFER_ALLOWED_TAG_WHITELIST + \
('class',)
if mode == 6:
allowed_attribute_whitelist += ('src', 'alt',
'width', 'height',
'style', 'summary',
'border', 'cellspacing',
'cellpadding')
allowed_tag_whitelist += ('img', 'table', 'td',
'tr', 'th', 'span', 'caption')
try:
return washer.wash(value,
allowed_attribute_whitelist=\
allowed_attribute_whitelist,
allowed_tag_whitelist=\
allowed_tag_whitelist
)
except HTMLParseError:
# Parsing failed
return cgi.escape(value)
else:
return cgi.escape(value)
elif mode in [4, 9]:
try:
out = washer.wash(value,
allowed_attribute_whitelist=[],
allowed_tag_whitelist=[]
)
if mode == 9:
out = out.replace('"', '"')
return out
except HTMLParseError:
# Parsing failed
if mode == 4:
return cgi.escape(value)
else:
return cgi.escape(value, True)
elif mode == 7:
if value.lstrip(' \n').startswith(html_field):
return value
else:
return cgi.escape(value)
else:
return value
def bf_profile():
"""
Runs a benchmark
@return: None
"""
for i in range(1, 51):
format_record(i, "HD", ln=CFG_SITE_LANG, verbose=9, search_pattern=[])
return
if __name__ == "__main__":
import profile
import pstats
#bf_profile()
profile.run('bf_profile()', "bibformat_profile")
p = pstats.Stats("bibformat_profile")
p.strip_dirs().sort_stats("cumulative").print_stats()
diff --git a/invenio/modules/formatter/models.py b/invenio/modules/formatter/models.py
index 3c6b04b2f..23d7012bc 100644
--- a/invenio/modules/formatter/models.py
+++ b/invenio/modules/formatter/models.py
@@ -1,163 +1,163 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
invenio.modules.formatter.models
-----------------------------------
Database access related functions for Formatter engine and
administration pages.
"""
from invenio.ext.sqlalchemy import db
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
class Format(db.Model):
"""Represents a Format record."""
__tablename__ = 'format'
id = db.Column(
db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
name = db.Column(db.String(255), nullable=False)
code = db.Column(db.String(6), nullable=False, unique=True)
description = db.Column(db.String(255), server_default='')
content_type = db.Column(db.String(255), server_default='')
visibility = db.Column(
db.TinyInteger(4),
nullable=False,
server_default='1')
last_updated = db.Column(db.DateTime, nullable=True)
@classmethod
def get_export_formats(cls):
return cls.query.filter(db.and_(
Format.content_type != 'text/html',
Format.visibility == 1)
).order_by(Format.name).all()
def set_name(self, name, lang="generic", type='ln'):
"""
Sets the name of an output format.
if 'type' different from 'ln' or 'sn', do nothing
if 'name' exceeds 256 chars, 'name' is truncated to first 256 chars.
The localized names of output formats are located in formatname table.
:param type: either 'ln' (for long name) and 'sn' (for short name)
:param lang: the language in which the name is given
:param name: the name to give to the output format
"""
if len(name) > 256:
name = name[:256]
if type.lower() != "sn" and type.lower() != "ln":
return
if lang == "generic" and type.lower() == "ln":
self.name = name
else:
# Save inside formatname table for name variations
fname = db.session.query(Formatname)\
.get((self.id, lang, type.lower()))
if not fname:
fname = db.session.merge(Formatname())
fname.id_format = self.id
fname.ln = lang
fname.type = type.lower()
fname.value = name
db.session.save(fname)
db.session.add(self)
db.session.commit()
class Formatname(db.Model):
"""Represents a Formatname record."""
__tablename__ = 'formatname'
id_format = db.Column(
db.MediumInteger(9, unsigned=True),
db.ForeignKey(Format.id),
primary_key=True)
ln = db.Column(
db.Char(5),
primary_key=True,
server_default='')
type = db.Column(
db.Char(3),
primary_key=True,
server_default='sn')
value = db.Column(db.String(255), nullable=False)
format = db.relationship(Format, backref='names')
#TODO add association proxy with key (id_format, ln, type)
class Bibfmt(db.Model):
"""Represents a Bibfmt record."""
def __init__(self):
pass
__tablename__ = 'bibfmt'
id_bibrec = db.Column(
db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False,
server_default='0',
primary_key=True,
autoincrement=False)
format = db.Column(
db.String(10),
nullable=False,
server_default='',
primary_key=True,
index=True)
last_updated = db.Column(
db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00',
index=True)
value = db.Column(db.iLargeBinary)
bibrec = db.relationship(Bibrec, backref='bibfmt')
__all__ = [
'Format',
'Formatname',
'Bibfmt',
]
diff --git a/invenio/modules/indexer/models.py b/invenio/modules/indexer/models.py
index a1779b8bc..f413bc82a 100644
--- a/invenio/modules/indexer/models.py
+++ b/invenio/modules/indexer/models.py
@@ -1,2290 +1,2290 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
bibindex database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
# Create your models here.
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.search.models import Field
class IdxINDEX(db.Model):
"""Represents a IdxINDEX record."""
def __init__(self):
pass
__tablename__ = 'idxINDEX'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True)
name = db.Column(db.String(50), unique=True, nullable=False,
server_default='')
description = db.Column(db.String(255), nullable=False,
server_default='')
last_updated = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
stemming_language = db.Column(db.String(10), nullable=False,
server_default='')
indexer = db.Column(db.String(10), nullable=False, server_default='native')
synonym_kbrs = db.Column(db.String(255), nullable=False, server_default='')
remove_stopwords = db.Column(db.String(255), nullable=False, server_default='')
remove_html_markup = db.Column(db.String(10), nullable=False, server_default='')
remove_latex_markup = db.Column(db.String(10), nullable=False, server_default='')
tokenizer = db.Column(db.String(50), nullable=False, server_default='')
class IdxINDEXIdxINDEX(db.Model):
"""Represents an IdxINDEXIdxINDEX record."""
def __init__(self):
pass
__tablename__ = 'idxINDEX_idxINDEX'
id_virtual = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(IdxINDEX.id), nullable=False,
server_default='0', primary_key=True)
id_normal = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(IdxINDEX.id), nullable=False,
server_default='0', primary_key=True)
class IdxINDEXNAME(db.Model):
"""Represents a IdxINDEXNAME record."""
def __init__(self):
pass
__tablename__ = 'idxINDEXNAME'
id_idxINDEX = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(IdxINDEX.id), primary_key=True)
ln = db.Column(db.Char(5), primary_key=True,
server_default='')
type = db.Column(db.Char(3), primary_key=True,
server_default='sn')
value = db.Column(db.String(255), nullable=False)
idxINDEX = db.relationship(IdxINDEX, backref='names')
class IdxINDEXField(db.Model):
"""Represents a IdxINDEXField record."""
def __init__(self):
pass
__tablename__ = 'idxINDEX_field'
id_idxINDEX = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(IdxINDEX.id), primary_key=True)
id_field = db.Column(db.MediumInteger(9, unsigned=True), db.ForeignKey(Field.id),
primary_key=True)
regexp_punctuation = db.Column(db.String(255),
nullable=False,
server_default='[.,:;?!"]')
regexp_alphanumeric_separators = db.Column(db.String(255),
nullable=False) #FIX ME ,
#server_default='[!"#$\\%&''()*+,-./:;<=>?@[\\]^\\_`{|}~]')
idxINDEX = db.relationship(IdxINDEX, backref='fields')
field = db.relationship(Field, backref='idxINDEXes')
#GENERATED
class IdxPAIR01F(db.Model):
"""Represents a IdxPAIR01F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR01F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR01R(db.Model):
"""Represents a IdxPAIR01R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR01R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR02F(db.Model):
"""Represents a IdxPAIR02F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR02F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR02R(db.Model):
"""Represents a IdxPAIR02R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR02R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR03F(db.Model):
"""Represents a IdxPAIR03F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR03F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR03R(db.Model):
"""Represents a IdxPAIR03R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR03R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR04F(db.Model):
"""Represents a IdxPAIR04F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR04F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR04R(db.Model):
"""Represents a IdxPAIR04R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR04R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR05F(db.Model):
"""Represents a IdxPAIR05F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR05F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR05R(db.Model):
"""Represents a IdxPAIR05R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR05R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR06F(db.Model):
"""Represents a IdxPAIR06F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR06F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR06R(db.Model):
"""Represents a IdxPAIR06R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR06R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR07F(db.Model):
"""Represents a IdxPAIR07F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR07F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR07R(db.Model):
"""Represents a IdxPAIR07R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR07R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR08F(db.Model):
"""Represents a IdxPAIR08F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR08F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR08R(db.Model):
"""Represents a IdxPAIR08R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR08R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR09F(db.Model):
"""Represents a IdxPAIR09F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR09F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR09R(db.Model):
"""Represents a IdxPAIR09R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR09R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR10F(db.Model):
"""Represents a IdxPAIR10F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR10F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR10R(db.Model):
"""Represents a IdxPAIR10R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR10R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR11F(db.Model):
"""Represents a IdxPAIR11F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR11F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR11R(db.Model):
"""Represents a IdxPAIR11R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR11R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR12F(db.Model):
"""Represents a IdxPAIR12F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR12F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR12R(db.Model):
"""Represents a IdxPAIR12R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR12R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR13F(db.Model):
"""Represents a IdxPAIR13F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR13F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR13R(db.Model):
"""Represents a IdxPAIR13R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR13R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR14F(db.Model):
"""Represents a IdxPAIR14F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR14F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR14R(db.Model):
"""Represents a IdxPAIR14R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR14R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR15F(db.Model):
"""Represents a IdxPAIR15F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR15F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR15R(db.Model):
"""Represents a IdxPAIR15R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR15R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR16F(db.Model):
"""Represents a IdxPAIR16F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR16F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR16R(db.Model):
"""Represents a IdxPAIR16R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR16R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR17F(db.Model):
"""Represents a IdxPAIR17F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR17F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR17R(db.Model):
"""Represents a IdxPAIR17R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR17R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR18F(db.Model):
"""Represents a IdxPAIR18F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR18F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR18R(db.Model):
"""Represents a IdxPAIR18R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR18R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR19F(db.Model):
"""Represents a IdxPAIR19F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR19F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR19R(db.Model):
"""Represents a IdxPAIR19R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR19R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR20F(db.Model):
"""Represents a IdxPAIR20F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR20F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR20R(db.Model):
"""Represents a IdxPAIR20R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR20R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR21F(db.Model):
"""Represents a IdxPAIR21F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR21F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR21R(db.Model):
"""Represents a IdxPAIR21R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR21R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR22F(db.Model):
"""Represents a IdxPAIR22F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR22F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR22R(db.Model):
"""Represents a IdxPAIR22R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR22R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR23F(db.Model):
"""Represents a IdxPAIR23F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR23F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR23R(db.Model):
"""Represents a IdxPAIR23R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR23R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR24F(db.Model):
"""Represents a IdxPAIR24F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR24F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR24R(db.Model):
"""Represents a IdxPAIR24R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR24R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR25F(db.Model):
"""Represents a IdxPAIR25F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR25F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR25R(db.Model):
"""Represents a IdxPAIR25R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR25R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPAIR26F(db.Model):
"""Represents a IdxPAIR26F record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR26F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPAIR26R(db.Model):
"""Represents a IdxPAIR26R record."""
def __init__(self):
pass
__tablename__ = 'idxPAIR26R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE01F(db.Model):
"""Represents a IdxPHRASE01F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE01F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE01R(db.Model):
"""Represents a IdxPHRASE01R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE01R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE02F(db.Model):
"""Represents a IdxPHRASE02F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE02F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE02R(db.Model):
"""Represents a IdxPHRASE02R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE02R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE03F(db.Model):
"""Represents a IdxPHRASE03F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE03F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE03R(db.Model):
"""Represents a IdxPHRASE03R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE03R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE04F(db.Model):
"""Represents a IdxPHRASE04F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE04F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE04R(db.Model):
"""Represents a IdxPHRASE04R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE04R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE05F(db.Model):
"""Represents a IdxPHRASE05F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE05F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE05R(db.Model):
"""Represents a IdxPHRASE05R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE05R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE06F(db.Model):
"""Represents a IdxPHRASE06F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE06F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE06R(db.Model):
"""Represents a IdxPHRASE06R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE06R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE07F(db.Model):
"""Represents a IdxPHRASE07F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE07F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE07R(db.Model):
"""Represents a IdxPHRASE07R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE07R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE08F(db.Model):
"""Represents a IdxPHRASE08F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE08F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE08R(db.Model):
"""Represents a IdxPHRASE08R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE08R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE09F(db.Model):
"""Represents a IdxPHRASE09F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE09F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE09R(db.Model):
"""Represents a IdxPHRASE09R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE09R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE10F(db.Model):
"""Represents a IdxPHRASE10F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE10F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE10R(db.Model):
"""Represents a IdxPHRASE10R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE10R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE11F(db.Model):
"""Represents a IdxPHRASE11F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE11F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE11R(db.Model):
"""Represents a IdxPHRASE11R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE11R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE12F(db.Model):
"""Represents a IdxPHRASE12F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE12F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE12R(db.Model):
"""Represents a IdxPHRASE12R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE12R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE13F(db.Model):
"""Represents a IdxPHRASE13F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE13F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE13R(db.Model):
"""Represents a IdxPHRASE13R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE13R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE14F(db.Model):
"""Represents a IdxPHRASE14F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE14F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE14R(db.Model):
"""Represents a IdxPHRASE14R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE14R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE15F(db.Model):
"""Represents a IdxPHRASE15F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE15F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE15R(db.Model):
"""Represents a IdxPHRASE15R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE15R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE16F(db.Model):
"""Represents a IdxPHRASE16F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE16F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE16R(db.Model):
"""Represents a IdxPHRASE16R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE16R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE17F(db.Model):
"""Represents a IdxPHRASE17F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE17F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE17R(db.Model):
"""Represents a IdxPHRASE17R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE17R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE18F(db.Model):
"""Represents a IdxPHRASE18F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE18F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE18R(db.Model):
"""Represents a IdxPHRASE18R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE18R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE19F(db.Model):
"""Represents a IdxPHRASE19F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE19F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE19R(db.Model):
"""Represents a IdxPHRASE19R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE19R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE20F(db.Model):
"""Represents a IdxPHRASE20F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE20F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE20R(db.Model):
"""Represents a IdxPHRASE20R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE20R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE21F(db.Model):
"""Represents a IdxPHRASE21F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE21F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE21R(db.Model):
"""Represents a IdxPHRASE21R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE21R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE22F(db.Model):
"""Represents a IdxPHRASE22F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE22F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE22R(db.Model):
"""Represents a IdxPHRASE22R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE22R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE23F(db.Model):
"""Represents a IdxPHRASE23F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE23F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE23R(db.Model):
"""Represents a IdxPHRASE23R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE23R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE24F(db.Model):
"""Represents a IdxPHRASE24F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE24F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE24R(db.Model):
"""Represents a IdxPHRASE24R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE24R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE25F(db.Model):
"""Represents a IdxPHRASE25F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE25F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE25R(db.Model):
"""Represents a IdxPHRASE25R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE25R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxPHRASE26F(db.Model):
"""Represents a IdxPHRASE26F record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE26F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(100), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxPHRASE26R(db.Model):
"""Represents a IdxPHRASE26R record."""
def __init__(self):
pass
__tablename__ = 'idxPHRASE26R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD01F(db.Model):
"""Represents a IdxWORD01F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD01F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD01R(db.Model):
"""Represents a IdxWORD01R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD01R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD02F(db.Model):
"""Represents a IdxWORD02F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD02F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD02R(db.Model):
"""Represents a IdxWORD02R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD02R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD03F(db.Model):
"""Represents a IdxWORD03F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD03F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD03R(db.Model):
"""Represents a IdxWORD03R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD03R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD04F(db.Model):
"""Represents a IdxWORD04F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD04F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD04R(db.Model):
"""Represents a IdxWORD04R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD04R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD05F(db.Model):
"""Represents a IdxWORD05F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD05F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD05R(db.Model):
"""Represents a IdxWORD05R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD05R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD06F(db.Model):
"""Represents a IdxWORD06F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD06F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD06R(db.Model):
"""Represents a IdxWORD06R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD06R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD07F(db.Model):
"""Represents a IdxWORD07F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD07F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD07R(db.Model):
"""Represents a IdxWORD07R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD07R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD08F(db.Model):
"""Represents a IdxWORD08F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD08F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD08R(db.Model):
"""Represents a IdxWORD08R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD08R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD09F(db.Model):
"""Represents a IdxWORD09F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD09F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD09R(db.Model):
"""Represents a IdxWORD09R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD09R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD10F(db.Model):
"""Represents a IdxWORD10F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD10F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD10R(db.Model):
"""Represents a IdxWORD10R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD10R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD11F(db.Model):
"""Represents a IdxWORD11F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD11F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD11R(db.Model):
"""Represents a IdxWORD11R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD11R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD12F(db.Model):
"""Represents a IdxWORD12F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD12F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD12R(db.Model):
"""Represents a IdxWORD12R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD12R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD13F(db.Model):
"""Represents a IdxWORD13F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD13F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD13R(db.Model):
"""Represents a IdxWORD13R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD13R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD14F(db.Model):
"""Represents a IdxWORD14F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD14F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD14R(db.Model):
"""Represents a IdxWORD14R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD14R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD15F(db.Model):
"""Represents a IdxWORD15F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD15F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD15R(db.Model):
"""Represents a IdxWORD15R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD15R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD16F(db.Model):
"""Represents a IdxWORD16F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD16F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD16R(db.Model):
"""Represents a IdxWORD16R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD16R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD17F(db.Model):
"""Represents a IdxWORD17F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD17F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD17R(db.Model):
"""Represents a IdxWORD17R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD17R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD18F(db.Model):
"""Represents a IdxWORD18F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD18F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD18R(db.Model):
"""Represents a IdxWORD18R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD18R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD19F(db.Model):
"""Represents a IdxWORD19F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD19F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD19R(db.Model):
"""Represents a IdxWORD19R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD19R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD20F(db.Model):
"""Represents a IdxWORD20F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD20F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD20R(db.Model):
"""Represents a IdxWORD20R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD20R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD21F(db.Model):
"""Represents a IdxWORD21F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD21F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD21R(db.Model):
"""Represents a IdxWORD21R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD21R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD22F(db.Model):
"""Represents a IdxWORD22F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD22F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD22R(db.Model):
"""Represents a IdxWORD22R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD22R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD23F(db.Model):
"""Represents a IdxWORD23F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD23F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD23R(db.Model):
"""Represents a IdxWORD23R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD23R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD24F(db.Model):
"""Represents a IdxWORD24F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD24F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD24R(db.Model):
"""Represents a IdxWORD24R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD24R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD25F(db.Model):
"""Represents a IdxWORD25F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD25F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD25R(db.Model):
"""Represents a IdxWORD25R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD25R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
class IdxWORD26F(db.Model):
"""Represents a IdxWORD26F record."""
def __init__(self):
pass
__tablename__ = 'idxWORD26F'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
term = db.Column(db.String(50), nullable=True,
unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class IdxWORD26R(db.Model):
"""Represents a IdxWORD26R record."""
def __init__(self):
pass
__tablename__ = 'idxWORD26R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
termlist = db.Column(db.iLargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False,
server_default='CURRENT',
primary_key=True)
__all__ = ['IdxINDEX',
'IdxINDEXIdxINDEX',
'IdxINDEXNAME',
'IdxINDEXField',
'IdxPAIR01F',
'IdxPAIR01R',
'IdxPAIR02F',
'IdxPAIR02R',
'IdxPAIR03F',
'IdxPAIR03R',
'IdxPAIR04F',
'IdxPAIR04R',
'IdxPAIR05F',
'IdxPAIR05R',
'IdxPAIR06F',
'IdxPAIR06R',
'IdxPAIR07F',
'IdxPAIR07R',
'IdxPAIR08F',
'IdxPAIR08R',
'IdxPAIR09F',
'IdxPAIR09R',
'IdxPAIR10F',
'IdxPAIR10R',
'IdxPAIR11F',
'IdxPAIR11R',
'IdxPAIR12F',
'IdxPAIR12R',
'IdxPAIR13F',
'IdxPAIR13R',
'IdxPAIR14F',
'IdxPAIR14R',
'IdxPAIR15F',
'IdxPAIR15R',
'IdxPAIR16F',
'IdxPAIR16R',
'IdxPAIR17F',
'IdxPAIR17R',
'IdxPAIR18F',
'IdxPAIR18R',
'IdxPAIR19F',
'IdxPAIR19R',
'IdxPAIR20F',
'IdxPAIR20R',
'IdxPAIR21F',
'IdxPAIR21R',
'IdxPAIR22F',
'IdxPAIR22R',
'IdxPAIR23F',
'IdxPAIR23R',
'IdxPAIR24F',
'IdxPAIR24R',
'IdxPAIR25F',
'IdxPAIR25R',
'IdxPAIR26F',
'IdxPAIR26R',
'IdxPHRASE01F',
'IdxPHRASE01R',
'IdxPHRASE02F',
'IdxPHRASE02R',
'IdxPHRASE03F',
'IdxPHRASE03R',
'IdxPHRASE04F',
'IdxPHRASE04R',
'IdxPHRASE05F',
'IdxPHRASE05R',
'IdxPHRASE06F',
'IdxPHRASE06R',
'IdxPHRASE07F',
'IdxPHRASE07R',
'IdxPHRASE08F',
'IdxPHRASE08R',
'IdxPHRASE09F',
'IdxPHRASE09R',
'IdxPHRASE10F',
'IdxPHRASE10R',
'IdxPHRASE11F',
'IdxPHRASE11R',
'IdxPHRASE12F',
'IdxPHRASE12R',
'IdxPHRASE13F',
'IdxPHRASE13R',
'IdxPHRASE14F',
'IdxPHRASE14R',
'IdxPHRASE15F',
'IdxPHRASE15R',
'IdxPHRASE16F',
'IdxPHRASE16R',
'IdxPHRASE17F',
'IdxPHRASE17R',
'IdxPHRASE18F',
'IdxPHRASE18R',
'IdxPHRASE19F',
'IdxPHRASE19R',
'IdxPHRASE20F',
'IdxPHRASE20R',
'IdxPHRASE21F',
'IdxPHRASE21R',
'IdxPHRASE22F',
'IdxPHRASE22R',
'IdxPHRASE23F',
'IdxPHRASE23R',
'IdxPHRASE24F',
'IdxPHRASE24R',
'IdxPHRASE25F',
'IdxPHRASE25R',
'IdxPHRASE26F',
'IdxPHRASE26R',
'IdxWORD01F',
'IdxWORD01R',
'IdxWORD02F',
'IdxWORD02R',
'IdxWORD03F',
'IdxWORD03R',
'IdxWORD04F',
'IdxWORD04R',
'IdxWORD05F',
'IdxWORD05R',
'IdxWORD06F',
'IdxWORD06R',
'IdxWORD07F',
'IdxWORD07R',
'IdxWORD08F',
'IdxWORD08R',
'IdxWORD09F',
'IdxWORD09R',
'IdxWORD10F',
'IdxWORD10R',
'IdxWORD11F',
'IdxWORD11R',
'IdxWORD12F',
'IdxWORD12R',
'IdxWORD13F',
'IdxWORD13R',
'IdxWORD14F',
'IdxWORD14R',
'IdxWORD15F',
'IdxWORD15R',
'IdxWORD16F',
'IdxWORD16R',
'IdxWORD17F',
'IdxWORD17R',
'IdxWORD18F',
'IdxWORD18R',
'IdxWORD19F',
'IdxWORD19R',
'IdxWORD20F',
'IdxWORD20R',
'IdxWORD21F',
'IdxWORD21R',
'IdxWORD22F',
'IdxWORD22R',
'IdxWORD23F',
'IdxWORD23R',
'IdxWORD24F',
'IdxWORD24R',
'IdxWORD25F',
'IdxWORD25R',
'IdxWORD26F',
'IdxWORD26R']
diff --git a/invenio/modules/indexer/tokenizers/BibIndexFiletypeTokenizer.py b/invenio/modules/indexer/tokenizers/BibIndexFiletypeTokenizer.py
index 00342a33b..4111dca80 100644
--- a/invenio/modules/indexer/tokenizers/BibIndexFiletypeTokenizer.py
+++ b/invenio/modules/indexer/tokenizers/BibIndexFiletypeTokenizer.py
@@ -1,63 +1,63 @@
# -*- coding:utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2010, 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""BibIndexFiletypeTokenizer: 'tokenizes' for file extensions.
- Tokenizer is adapted to work with bibfield and its get_record function.
+ Tokenizer is adapted to work with recjson and its get_record function.
"""
from invenio.modules.indexer.tokenizers.BibIndexEmptyTokenizer import BibIndexEmptyTokenizer
class BibIndexFiletypeTokenizer(BibIndexEmptyTokenizer):
"""
Tokenizes for file extensions.
- Tokenizer is adapted to work with bibfield and its get_record function.
+ Tokenizer is adapted to work with recjson and its get_record function.
It accepts as an input a record created by a get_record function:
- from bibfield import get_record
+ from invenio.modules.records.api import get_record
record16 = get_record(16)
tokenizer = BibIndexFiletypeTokenizer()
new_words = tokenizer.tokenize(record16)
"""
def __init__(self, stemming_language = None, remove_stopwords = False, remove_html_markup = False, remove_latex_markup = False):
pass
def tokenize(self, record):
- """'record' is a recjson record from bibfield.
+ """'record' is a recjson record.
Function uses derived field 'filetypes'
from the record.
@param urls: recjson record
"""
values = []
try:
if record.has_key('filetypes'):
values = record['filetypes']
except KeyError:
pass
except TypeError:
return []
return values
def get_tokenizing_function(self, wordtable_type):
return self.tokenize
diff --git a/invenio/core/__init__.py b/invenio/modules/jsonalchemy/__init__.py
similarity index 100%
copy from invenio/core/__init__.py
copy to invenio/modules/jsonalchemy/__init__.py
diff --git a/invenio/legacy/bibfield/functions/get_number_of_comments.py b/invenio/modules/jsonalchemy/errors.py
similarity index 68%
rename from invenio/legacy/bibfield/functions/get_number_of_comments.py
rename to invenio/modules/jsonalchemy/errors.py
index 2adbb4182..2aa2f937f 100644
--- a/invenio/legacy/bibfield/functions/get_number_of_comments.py
+++ b/invenio/modules/jsonalchemy/errors.py
@@ -1,31 +1,31 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+class FieldParserException(Exception):
+ """Exception raised when some error happens parsing field definitions"""
+ pass
-def get_number_of_comments(recid):
- """
- Returns number of comments for given record.
- @param recid:
+class ModelParserException(Exception):
+ """Exception raised when some error happens parsing model definitions"""
+ pass
- @return: Number of comments
- """
- from invenio.legacy.webcomment.adminlib import get_nb_comments
- if recid:
- return get_nb_comments(recid)
+class ReaderException(Exception):
+ """Exception raised when some error happens reading a blob"""
+ pass
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/jsonext/__init__.py
similarity index 100%
copy from invenio/legacy/bibfield/functions/__init__.py
copy to invenio/modules/jsonalchemy/jsonext/__init__.py
diff --git a/invenio/core/record/__init__.py b/invenio/modules/jsonalchemy/jsonext/engines/__init__.py
similarity index 100%
rename from invenio/core/record/__init__.py
rename to invenio/modules/jsonalchemy/jsonext/engines/__init__.py
diff --git a/invenio/utils/jsonalchemy/engines/mongodb_pymongo.py b/invenio/modules/jsonalchemy/jsonext/engines/mongodb_pymongo.py
similarity index 65%
rename from invenio/utils/jsonalchemy/engines/mongodb_pymongo.py
rename to invenio/modules/jsonalchemy/jsonext/engines/mongodb_pymongo.py
index bd3eb0354..13d9ab96e 100644
--- a/invenio/utils/jsonalchemy/engines/mongodb_pymongo.py
+++ b/invenio/modules/jsonalchemy/jsonext/engines/mongodb_pymongo.py
@@ -1,104 +1,101 @@
# -*- coding: utf-8 -*-
-
+##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
- invenio.utils.jsonalchemy.engines.mongo_pymongo
+ invenio.modules.jsonalchemy.engines.mongo_pymongo
------------------------------------------
"""
-from itertools import izip, imap
-
import pymongo
-from invenio.utils.jsonalchemy.storage import Storage
-
+from invenio.modules.jsonalchemy.storage import Storage
class MongoDBStorage(Storage):
"""
Implements storage engine for MongoDB using the driver pymongo
"""
def __init__(self, model, **kwards):
"""
- TBC
- See also :meth:`~invenio.utils.jsonalchemy.storage:Storage.__init__`
+ See also :meth:`~invenio.modules.jsonalchemy.storage:Storage.__init__`
"""
self.model = model
host = kwards.get('host', 'localhost')
port = kwards.get('port', 27017)
database = kwards.get('database', 'invenio')
self.__connection = pymongo.MongoClient(host=host,port=port)
self.__database = self.__connection[database]
self.__collection = self.__database[model]
def save_one(self, json, id=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.save_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.save_one`"""
if not id is None:
json['_id'] = id
return self.__collection.insert(json)
def save_many(self, jsons, ids=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.save_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.save_many`"""
if not ids is None:
def add_id(t):
t[0]['_id'] = t[1]
return t[0]
- jsons = impa(add_id, izip(jsons, ids))
+ jsons = impa(add_id, zip(jsons, ids))
return self.__collection.insert(jsons, continue_on_error=True)
def update_one(self, json, id=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.update_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.update_one`"""
#FIXME: what if we get only the fields that have change
if not recid is None:
json['_id'] = recid
return self.__collection.save(json)
def update_many(self, jsons, ids=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.update_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.update_many`"""
if not recids is None:
def add_id(t):
t[0]['_id'] = t[1]
return t[0]
- jsons = impa(add_id, izip(jsons, ids))
+ jsons = impa(add_id, zip(jsons, ids))
- return imap(self.__collection.save, jsons)
+ return map(self.__collection.save, jsons)
def get_one(self, id):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_one`"""
return self.__collection.find_one(id)
def get_many(self, ids):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_many`"""
return self.__collection.find({'_id': {'$in':ids}})
- def get_field_values(recids, field, repetitive_values=True, count=False,
+ def get_field_values(ids, field, repetitive_values=True, count=False,
include_recid=False, split_by=0):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_field_values`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_field_values`"""
raise NotImplementedError()
- def get_fields_values(recids, fields, repetitive_values=True, count=False,
+ def get_fields_values(ids, fields, repetitive_values=True, count=False,
include_recid=False, split_by=0):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_fields_values`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_fields_values`"""
raise NotImplementedError()
-
-
+ def search(self, query):
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.search`"""
+ return self.__collection.find(query)
diff --git a/invenio/utils/jsonalchemy/engines/sqlalchemy.py b/invenio/modules/jsonalchemy/jsonext/engines/sqlalchemy.py
similarity index 67%
rename from invenio/utils/jsonalchemy/engines/sqlalchemy.py
rename to invenio/modules/jsonalchemy/jsonext/engines/sqlalchemy.py
index 37a7afbb4..153b4ae2e 100644
--- a/invenio/utils/jsonalchemy/engines/sqlalchemy.py
+++ b/invenio/modules/jsonalchemy/jsonext/engines/sqlalchemy.py
@@ -1,123 +1,119 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-"""
- invenio.core.record.backends.sqlalchemy
- ---------------------------------------
+import six
-"""
-from itertools import izip, imap
-
-from flask import current_app
from flask.helpers import locked_cached_property
from werkzeug import import_string
-from invenio.utils.jsonalchemy.storage import Storage
-
+from invenio.modules.jsonalchemy.storage import Storage
class SQLAlchemyStorage(Storage):
"""
Implements database backend for SQLAlchemy model storage.
"""
#FIXME: This storage engine should use transactions!
def __init__(self, model, **kwards):
"""
- TBC
- See also :meth:`~invenio.utils.jsonalchemy.storage:Storage.__init__`
+ See also :meth:`~invenio.modules.jsonalchemy.storage:Storage.__init__`
"""
self.__db = kwards.get('sqlalchemy_backend', 'invenio.ext.sqlalchemy:db')
self.__model = model
- if not self.db.engine.dialect.has_table(self.db.engine,
- self.model.__tablename__):
- self.model.__table__.create(bind=self.db.engine)
- self.db.session.commit()
@locked_cached_property
def db(self):
"""Returns SQLAlchemy database object."""
- if isinstance(self.__db, basestring):
- return import_string(self.__db)
+ if isinstance(self.__db, six.string_types):
+ self.__db = import_string(self.__db)
+ if not self.__db.engine.dialect.has_table(self.__db.engine,
+ self.model.__tablename__):
+ self.model.__table__.create(bind=self.__db.engine)
+ self.__db.session.commit()
return self.__db
@locked_cached_property
def model(self):
"""Returns SQLAchemy model."""
- if isinstance(self.__model, basestring):
+ if isinstance(self.__model, six.string_types):
return import_string(self.__model)
return self.__model
def save_one(self, json, id=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.save_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.save_one`"""
if id is None:
- id = json_record['_id']
+ id = json['_id']
self.db.session.add(self.model(id=id, json=json))
self.db.session.commit()
def save_many(self, jsons, ids=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.save_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.save_many`"""
if ids is None:
- ids = imap(lambda j: j['_id'], jsons)
+ ids = map(lambda j: j['_id'], jsons)
self.db.session.add_all([self.model(id=id, json=json)
- for id, json in izip(ids, jsons)])
+ for id, json in zip(ids, jsons)])
self.db.session.commit()
def update_one(self, json, id=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.update_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.update_one`"""
#FIXME: what if we get only the fields that have change
if id is None:
id = json['id']
self.db.session.merge(self.model(id=id, json=json))
self.db.session.commit()
def update_many(self, jsons, ids=None):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.update_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.update_many`"""
#FIXME: what if we get only the fields that have change
if ids is None:
- ids = imap(lambda j: j['_id'], jsons)
+ ids = map(lambda j: j['_id'], jsons)
- for id, json in izip(ids, jsons):
+ for id, json in zip(ids, jsons):
self.db.session.merge(self.model(id=id, json=json))
self.db.session.commit()
def get_one(self, id):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_one`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_one`"""
return self.db.session.query(self.model.json)\
.filter_by(id=id).one().json
def get_many(self, ids):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_many`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_many`"""
for json in self.db.session.query(self.model.json)\
.filter(RecordMetadata.id.in_(ids))\
.all():
yield json[0]
def get_field_values(recids, field, repetitive_values=True, count=False,
include_recid=False, split_by=0):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_field_values`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_field_values`"""
#TODO
raise NotImplementedError()
def get_fields_values(recids, fields, repetitive_values=True, count=False,
include_recid=False, split_by=0):
- """See :meth:`~invenio.utils.jsonalchemy.storage:Storage.get_fields_values`"""
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.get_fields_values`"""
#TODO
raise NotImplementedError()
+
+ def search(query):
+ """See :meth:`~invenio.modules.jsonalchemy.storage:Storage.search`"""
+ raise NotImplementedError()
diff --git a/invenio/utils/jsonalchemy/__init__.py b/invenio/modules/jsonalchemy/jsonext/functions/__init__.py
similarity index 100%
rename from invenio/utils/jsonalchemy/__init__.py
rename to invenio/modules/jsonalchemy/jsonext/functions/__init__.py
diff --git a/invenio/legacy/bibfield/functions/is_local_url.py b/invenio/modules/jsonalchemy/jsonext/functions/is_local_url.py
similarity index 100%
rename from invenio/legacy/bibfield/functions/is_local_url.py
rename to invenio/modules/jsonalchemy/jsonext/functions/is_local_url.py
diff --git a/invenio/legacy/bibfield/functions/util_split.py b/invenio/modules/jsonalchemy/jsonext/functions/util_split.py
similarity index 100%
rename from invenio/legacy/bibfield/functions/util_split.py
rename to invenio/modules/jsonalchemy/jsonext/functions/util_split.py
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/jsonext/parsers/__init__.py
similarity index 100%
copy from invenio/legacy/bibfield/functions/__init__.py
copy to invenio/modules/jsonalchemy/jsonext/parsers/__init__.py
diff --git a/invenio/modules/jsonalchemy/jsonext/parsers/checker_parser.py b/invenio/modules/jsonalchemy/jsonext/parsers/checker_parser.py
new file mode 100644
index 000000000..0de9d2887
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/parsers/checker_parser.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 60 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from pyparsing import Optional, Suppress, OneOrMore, indentedBlock, nestedExpr,\
+ originalTextFor
+
+from invenio.modules.jsonalchemy.parser import BaseExtensionParser, \
+ function_call
+
+
+class CheckerParser(BaseExtensionParser):
+ """
+ """
+
+ @classmethod
+ def parse_element(cls, indent_stack):
+ master_format = (Suppress("@master_format") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("master_format")\
+ .setParseAction(lambda toks: toks[0])
+ checker_body = indentedBlock((Optional (master_format) + \
+ function_call.setResultsName('func')), indent_stack)
+
+ return (Suppress('checker:') + OneOrMore(checker_body))\
+ .setResultsName('checker')\
+ .setParseAction(lambda toks: toks[0])
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ return None
+
+CheckerParser.__name__ = 'checker'
+parser = CheckerParser
diff --git a/invenio/modules/jsonalchemy/jsonext/parsers/description_parser.py b/invenio/modules/jsonalchemy/jsonext/parsers/description_parser.py
new file mode 100644
index 000000000..b0acb4586
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/parsers/description_parser.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 60 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from pyparsing import QuotedString, Suppress, indentedBlock
+
+from invenio.modules.jsonalchemy.parser import BaseExtensionParser
+
+
+class DescriptionParser(BaseExtensionParser):
+ """
+
+ """
+
+ @classmethod
+ def parse_element(cls, indent_stack):
+ doc_double = QuotedString(quoteChar='"""', multiline=True)
+ doc_single = QuotedString(quoteChar="'''", multiline=True)
+ doc_string = indentedBlock((doc_double | doc_single), indent_stack)
+ description = (Suppress('description:') + doc_string).\
+ setParseAction(lambda toks: toks[0][0])
+ return (description | doc_double | doc_single).setResultsName('description')
+
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ return rule.description
+
+DescriptionParser.__name__ = 'description'
+parser = DescriptionParser
diff --git a/invenio/modules/jsonalchemy/jsonext/parsers/json_extra_parser.py b/invenio/modules/jsonalchemy/jsonext/parsers/json_extra_parser.py
new file mode 100644
index 000000000..fe1cdc038
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/parsers/json_extra_parser.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 60 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from pyparsing import Optional, Suppress, indentedBlock, Each
+
+from invenio.base.utils import try_to_eval
+
+from invenio.modules.jsonalchemy.registry import functions
+from invenio.modules.jsonalchemy.parser import BaseExtensionParser, FieldParser, \
+ python_allowed_expr
+
+
+class JsonExtraParser(BaseExtensionParser):
+ """
+ Class to parse and store the information related with how to load and dump
+ a non-json object.
+
+ It parses something like this::
+
+ json:
+ loads, function_to_load(field)
+ dumps, function_to_dump(field)
+
+ The functions to load and dump must have one parameter which is the field
+ to parse.
+ """
+
+ @classmethod
+ def parse_element(cls, indent_stack):
+ json_dumps = (Suppress('dumps') + Suppress(',') + python_allowed_expr)\
+ .setResultsName("dumps")\
+ .setParseAction(lambda toks: toks.value[0])
+ json_loads = (Suppress("loads") + Suppress(",") + python_allowed_expr)\
+ .setResultsName("loads")\
+ .setParseAction(lambda toks: toks.value[0])
+
+ func = indentedBlock(Each((json_dumps, json_loads)), indent_stack)
+ return (Suppress('json:') + func)\
+ .setResultsName('json_ext')\
+ .setParseAction(lambda toks: toks[0][0])
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ json_id = rule.json_id[0]
+
+ return {'loads': try_to_eval(rule.json_ext.loads.strip(), functions(namespace)),
+ 'dumps': try_to_eval(rule.json_ext.dumps.strip(), functions(namespace))}
+
+ @classmethod
+ def add_info_to_field(cls, json_id, rule):
+ info = {}
+ if 'json_ext' in rule:
+ info['dumps'] = (json_id, 'json_ext', 'dumps')
+ info['loads'] = (json_id, 'json_ext', 'loads')
+ return info
+
+
+JsonExtraParser.__name__ = 'json_ext'
+parser = JsonExtraParser
diff --git a/invenio/modules/jsonalchemy/jsonext/parsers/producer_parser.py b/invenio/modules/jsonalchemy/jsonext/parsers/producer_parser.py
new file mode 100644
index 000000000..40c4066c4
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/parsers/producer_parser.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 60 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from pyparsing import Optional, Suppress, OneOrMore, Word, alphas, alphanums, \
+ indentedBlock, nestedExpr, originalTextFor
+
+from invenio.modules.jsonalchemy.parser import BaseExtensionParser, FieldParser,\
+ python_allowed_expr
+
+
+class ProducerParser(BaseExtensionParser):
+ """
+ """
+
+ @classmethod
+ def parse_element(cls, indent_stack):
+ producer_code = (Word(alphas, alphanums + "_")\
+ + originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName('producer_code')
+ producer_rule = (Suppress(',') + python_allowed_expr)\
+ .setResultsName('producer_rule')\
+ .setParseAction(lambda toks: toks[0])
+ producer_body = indentedBlock(producer_code + producer_rule, indent_stack)\
+ .setParseAction(lambda toks: toks[0])
+
+ return (Suppress('producer:') + OneOrMore(producer_body))\
+ .setResultsName('producer')
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ json_id = rule.json_id[0]
+ assert json_id in FieldParser.field_definitions(namespace)
+
+ producers = {}
+ for producer in rule.producer:
+ if producer.producer_code[0] not in producers:
+ producers[producer.producer_code[0]] = []
+ producers[producer.producer_code[0]].append(
+ (eval(producer.producer_code[1]), eval(producer.producer_rule)))#FIXME: remove eval
+ return producers
+
+ProducerParser.__name__ = 'producer'
+parser = ProducerParser
diff --git a/invenio/legacy/bibfield/functions/get_cited_by_count.py b/invenio/modules/jsonalchemy/jsonext/parsers/schema_parser.py
similarity index 52%
rename from invenio/legacy/bibfield/functions/get_cited_by_count.py
rename to invenio/modules/jsonalchemy/jsonext/parsers/schema_parser.py
index 36dcd1258..14c9a5636 100644
--- a/invenio/legacy/bibfield/functions/get_cited_by_count.py
+++ b/invenio/modules/jsonalchemy/jsonext/parsers/schema_parser.py
@@ -1,31 +1,44 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+## 60 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+from pyparsing import QuotedString, Suppress, indentedBlock
-def get_cited_by_count(recid):
- """
- Return how many records cite given record.
+from invenio.modules.jsonalchemy.parser import BaseExtensionParser, \
+ dict_def
- @param recid:
- @return: Number of records citing given record
+class SchemaParser(BaseExtensionParser):
+ """
+
"""
- from invenio.legacy.bibrank.citation_searcher import get_cited_by_count
- if recid:
- return get_cited_by_count(recid)
+
+ @classmethod
+ def parse_element(cls, indent_stack):
+ schema = indentedBlock(dict_def,indent_stack)
+ return (Suppress('schema:') + schema)\
+ .setParseAction(lambda toks: toks[0][0][0])\
+ .setResultsName('schema')
+
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ return eval(rule.schema)
+
+SchemaParser.__name__ = 'schema'
+parser = SchemaParser
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/jsonext/producers/__init__.py
similarity index 100%
copy from invenio/legacy/bibfield/functions/__init__.py
copy to invenio/modules/jsonalchemy/jsonext/producers/__init__.py
diff --git a/invenio/core/record/recordext/producers/marc.py b/invenio/modules/jsonalchemy/jsonext/producers/json_for_marc.py
similarity index 87%
rename from invenio/core/record/recordext/producers/marc.py
rename to invenio/modules/jsonalchemy/jsonext/producers/json_for_marc.py
index 260507242..9782cbe79 100644
--- a/invenio/core/record/recordext/producers/marc.py
+++ b/invenio/modules/jsonalchemy/jsonext/producers/json_for_marc.py
@@ -1,64 +1,64 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
def produce(self, fields=None):
"""
Export the record in marc format.
@param tags: list of tags to include in the output, if None or
empty list all available tags will be included.
"""
- from invenio.legacy.bibfield.bibfield_utils import get_producer_rules
+ from invenio.modules.jsonalchemy.parser import get_producer_rules
if not fields:
fields = self.keys()
out = []
for field in fields:
if field.startswith('__'):
continue
try:
- marc_rules = get_producer_rules(field, 'xm')
+ marc_rules = get_producer_rules(field, 'json_for_marc')
for rule in marc_rules:
field = self.get(rule[0], None)
if field is None:
continue
if not isinstance(field, list):
field = [field, ]
for f in field:
for r in rule[1]:
tmp_dict = {}
- for key, subfield in r.iteritems():
+ #FIXME: check field meta_metadata
+ for key, subfield in r[1].iteritems():
if not subfield:
tmp_dict[key] = f
else:
try:
tmp_dict[key] = f[subfield]
except:
try:
tmp_dict[key] = self._try_to_eval(subfield, value=f)
- except Exception,e:
+ except Exception as e:
self['__error_messages.cerror[n]'] = 'Producer CError - Unable to produce %s - %s' % (field, str(e))
if tmp_dict:
out.append(tmp_dict)
except KeyError:
self['__error_messages.cerror[n]'] = 'Producer CError - No producer rule for field %s' % field
return out
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/jsonext/readers/__init__.py
similarity index 100%
copy from invenio/legacy/bibfield/functions/__init__.py
copy to invenio/modules/jsonalchemy/jsonext/readers/__init__.py
diff --git a/invenio/modules/jsonalchemy/jsonext/readers/json_reader.py b/invenio/modules/jsonalchemy/jsonext/readers/json_reader.py
new file mode 100644
index 000000000..ce3a49af1
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/readers/json_reader.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.jsonext.readers.json_reader
+ --------------------------------------------------------
+
+"""
+import re
+
+from invenio.modules.jsonalchemy.reader import Reader
+
+class JsonReader(Reader):
+ """Default reader"""
+
+ __master_format__ = 'json'
+
+ def __init__(self, blob, **kwargs):
+ """
+ :param blob:
+ """
+ super(JsonReader, self).__init__(blob=blob, **kwargs)
+ self._additional_info['master_format'] = 'json'
+
+ @staticmethod
+ def split_blob(blob, schema=None, **kwargs):
+ """
+ In case of several records inside the blob this method specify how to
+ split then and work one by one afterwards.
+ """
+ return blob.splitlines()
+
+ def _prepare_blob(self, *args, **kwargs):
+ self.json.update(self.blob)
+
+ def _get_elements_from_blob(self, regex_key):
+ if regex_key in ('entire_record', '*'):
+ return self.blob
+ return [self.blob.get(key) for key in regex_key]
+
+ def _apply_rules(self, json_id, field_name, rule_def):
+ try:
+ info = self._find_meta_metadata(json_id, field_name, 'creator', {'source_tag':json_id}, rule_def)
+ if 'json_ext' in rule_def and field_name in self.json:
+ self.json[field_name] = rule_def['json_ext']['dumps'](self.json[field_name])
+ self.json['__meta_metadata__.%s' % (field_name, )] = info
+ except KeyError:
+ self._set_default_value(json_id, field_name)
+ except Exception, e:
+ self.json['__meta_metadata__']['__errors__']\
+ .append('Rule Error - Unable to apply rule for field %s - %s' % (field_name, str(e)),)
+ return False
+ return True
+
+ def _apply_virtual_rules(self, json_id, field_name, rule_def):
+ if field_name in self.json:
+ try:
+ info = self._find_meta_metadata(json_id, field_name, rule_type, rule, rule_def)
+ if rule_type == 'derived' or rule['memoize']:
+ if 'json_ext' in rule_def:
+ self.json[field_name] = rule_def['json_ext']['dumps'](self.json[field_name])
+ else:
+ self.json[field_name] = None
+ except Exception, e:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append('Virtual Rule CError - Unable to evaluate %s - %s' % (field_name, str(e)))
+ return False
+ else:
+ return super(JsonReader, self)._apply_virtual_rules(json_id, field_name, rule_def)
+
+reader = JsonReader
diff --git a/invenio/modules/jsonalchemy/jsonext/readers/marc_reader.py b/invenio/modules/jsonalchemy/jsonext/readers/marc_reader.py
new file mode 100644
index 000000000..cf933b3b7
--- /dev/null
+++ b/invenio/modules/jsonalchemy/jsonext/readers/marc_reader.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.jsonext.readers.marc_reader
+ --------------------------------------------------------
+
+"""
+import re
+
+from invenio.modules.jsonalchemy.reader import Reader
+
+class MarcReader(Reader):
+ """Marc reader"""
+
+ __master_format__ = 'marc'
+
+ split_marc = re.compile('.*?', re.DOTALL)
+
+ def __init__(self, blob=None, **kwargs):
+ """
+ :param blob:
+ """
+ super(MarcReader, self).__init__(blob=blob, **kwargs)
+ self._additional_info['master_format'] = 'marc'
+
+ @staticmethod
+ def split_blob(blob, schema=None, **kwargs):
+ """
+ Splits the blob using .*? as pattern.
+
+ Note 1: Taken from invenio.legacy.bibrecord:create_records
+ Note 2: Use the DOTALL flag to include newlines.
+ """
+ if schema in (None, 'xml'):
+ for match in MarcReader.split_marc.finditer(blob):
+ yield match.group()
+ else:
+ raise StopIteration()
+
+ def _get_elements_from_blob(self, regex_key):
+ if regex_key in ('entire_record', '*'):
+ return self.rec_tree
+ elements = []
+ for k in regex_key:
+ regex = re.compile(k)
+ keys = filter(regex.match, self.rec_tree.keys())
+ values = []
+ for key in keys:
+ values.append(self.rec_tree.get(key))
+ elements.extend(values)
+ return elements
+
+ def _prepare_blob(self, *args, **kwargs):
+ #FIXME stop using recstruct!
+ from invenio.legacy.bibrecord import create_record
+
+ class SaveDict(dict):
+ __getitem__ = dict.get
+
+ def dict_extend_helper(d, key, value):
+ """
+ If the key is present inside the dictionary it creates a list (it not
+ present) and extends it with the new value. Almost as in C{list.extend}
+ """
+ if key in d:
+ current_value = d.get(key)
+ if not isinstance(current_value, list):
+ current_value = [current_value]
+ current_value.append(value)
+ value = current_value
+ d[key] = value
+
+ self.rec_tree = SaveDict()
+ tmp = create_record(self.blob)[0]
+ for key, values in tmp.iteritems():
+ if key < '010' and key.isdigit():
+ self.rec_tree[key] = [value[3] for value in values]
+ else:
+ for value in values:
+ field = SaveDict()
+ for subfield in value[0]:
+ dict_extend_helper(field, subfield[0], subfield[1])
+ dict_extend_helper(self.rec_tree, (key + value[1] + value[2]).replace(' ', '_'), field)
+
+reader = MarcReader
diff --git a/invenio/modules/jsonalchemy/parser.py b/invenio/modules/jsonalchemy/parser.py
new file mode 100644
index 000000000..54aee4b5d
--- /dev/null
+++ b/invenio/modules/jsonalchemy/parser.py
@@ -0,0 +1,724 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.parser
+ ---------------------------------
+
+ Fields and models configuration loader.
+
+ This module uses `pyparsing ` to read from
+ thedifferent configuration files the field and model definitions.
+
+ Extensions to both parsers could be added inside jsonext.parsers
+"""
+import os
+import re
+import six
+
+from pyparsing import ParseException, FollowedBy, Suppress, OneOrMore, Word, \
+ LineEnd, ZeroOrMore, Optional, Literal, alphas, alphanums, \
+ originalTextFor, oneOf, nestedExpr, quotedString, removeQuotes, lineEnd, \
+ empty, col, restOfLine, delimitedList, Each, indentedBlock
+
+
+from .errors import FieldParserException, ModelParserException
+from .registry import fields_definitions, models_definitions, parsers
+
+json_id = (Word(alphas + "_", alphanums + "_") + Optional(oneOf("[0] [n]")))\
+ .setResultsName("json_id", listAllMatches=True)\
+ .setParseAction(lambda toks: "".join(toks))
+
+ident = Word(alphas + "_", alphanums + "_")
+dict_def = originalTextFor(nestedExpr('{', '}'))
+list_def = originalTextFor(nestedExpr('[', ']'))
+dict_access = list_access = originalTextFor(ident + nestedExpr('[', ']'))
+function_call = originalTextFor(ZeroOrMore(ident + ".") + ident + nestedExpr('(', ')'))
+
+python_allowed_expr = (dict_def ^ list_def ^ dict_access ^ \
+ list_access ^ function_call ^ restOfLine)\
+ .setResultsName("value", listAllMatches=True)
+
+def _create_record_field_parser():
+ """
+ Creates the base parser that can handle field definitions and adds any
+ extension placed inside jsonext.parsers.
+
+ BFN like base libray::
+ line ::= python_comment | include | field_def
+
+ include ::= "include(" PATH ")"
+
+ field_def ::= [persitent_identifier] [inherit_from] [override] [extend] json_id ["[0]" | "[n]"] "," aliases":" INDENT field_def_body UNDENT
+ field_def_body ::= [default] (creator | derived | calculated)
+ aliases ::= json_id ["[0]" | "[n]"] ["," aliases]
+ json_id ::= (alphas + '_') (alphanums + '_')
+
+ creator ::= "creator:" INDENT creator_body+ UNDENT
+ creator_body ::= [decorators] source_format "," source_tags "," python_allowed_expr
+ source_format ::= MASTER_FORMATS
+ source_tag ::= QUOTED_STRING #reStrucuredText
+
+ derived ::= "derived:" INDENT derived_calculated_body UNDENT
+ calculated ::= "calculated:" INDENT derived_calculated_body UNDENT
+ derived_calculated_body ::= [decorators] "," python_allowed_exp
+
+ decorators ::= (legacy | memoize | parse_first | depends_on | only_if | only_if_master_value)*
+ legacy ::= "@legacy(" correspondences+ ")"
+ correspondences ::= "(" source_tag [ "," field_tag_name ] "," subfield ")" # If no subfield needed empty string
+ parse_first ::= "@parse_first(" json_id+ ")"
+ depends_on ::= "@depends_on(" json_id+ ")"
+ only_if ::= "@only_if(" python_condition+ ")"
+ only_if_master_value ::= "@only_if_master_value(" python_condition+ ")"
+
+ persistent_identifier ::= @persistent_identifier( level )
+ inherit_from ::= "@inherit_from(" json_id+ ")"
+ override ::= "@override"
+ extend ::= "@extend"
+ memoize ::= "@memoize(time)"
+
+ python_allowed_exp ::= ident | list_def | dict_def | list_access | dict_access | function_call | one_line_expr
+ """
+ indent_stack = [1]
+
+ def check_sub_indent(string, location, tokens):
+ cur_col = col(location, string)
+ if cur_col > indent_stack[-1]:
+ indent_stack.append(cur_col)
+ else:
+ raise ParseException(string, location, "not a subentry")
+
+ def check_unindent(string, location, tokens):
+ if location >= len(string):
+ return
+ cur_col = col(location, string)
+ if not(cur_col < indent_stack[-1] and cur_col <= indent_stack[-2]):
+ raise ParseException(string, location, "not an unindent")
+
+ def do_unindent():
+ indent_stack.pop()
+
+ INDENT = lineEnd.suppress() + empty + empty.copy().setParseAction(check_sub_indent)
+ UNDENT = FollowedBy(empty).setParseAction(check_unindent)
+ UNDENT.setParseAction(do_unindent)
+
+ aliases = delimitedList((Word(alphanums + "_") + Optional(oneOf("[0] [n]")))\
+ .setParseAction(lambda toks: "".join(toks)))\
+ .setResultsName("aliases")
+
+ persistent_identifier = (Suppress("@persistent_identifier") + \
+ nestedExpr("(", ")"))\
+ .setResultsName("persistent_identifier")
+ legacy = (Suppress("@legacy") + originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("legacy", listAllMatches=True)
+ only_if = (Suppress("@only_if") + originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("only_if")
+ only_if_master_value = (Suppress("@only_if_value") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("only_if_master_value")
+ depends_on = (Suppress("@depends_on") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("depends_on")
+ parse_first = (Suppress("@parse_first") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("parse_first")
+ memoize = (Suppress("@memoize") + nestedExpr("(", ")"))\
+ .setResultsName("memoize")
+ field_decorator = parse_first ^ depends_on ^ only_if ^ \
+ only_if_master_value ^ memoize ^ legacy
+
+ #Independent decorators
+ inherit_from = (Suppress("@inherit_from") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("inherit_from")
+ override = (Suppress("@") + "override")\
+ .setResultsName("override")
+ extend = (Suppress("@") + "extend")\
+ .setResultsName("extend")
+
+ derived_calculated_body = (ZeroOrMore(field_decorator) + python_allowed_expr)\
+ .setResultsName('derived_calculated_def')
+
+ derived = "derived" + Suppress(":") + \
+ INDENT + derived_calculated_body + UNDENT
+ calculated = "calculated" + Suppress(":") + \
+ INDENT + derived_calculated_body + UNDENT
+
+ source_tag = quotedString\
+ .setParseAction(removeQuotes)\
+ .setResultsName("source_tag", listAllMatches=True)
+ source_format = Word(alphas, alphanums + "_")\
+ .setResultsName("source_format", listAllMatches=True)
+ creator_body = (ZeroOrMore(field_decorator) + source_format + \
+ Suppress(",") + source_tag + Suppress(",") + python_allowed_expr)\
+ .setResultsName("creator_def", listAllMatches=True)
+ creator = "creator" + Suppress(":") + \
+ INDENT + OneOrMore(creator_body) + UNDENT
+
+ field_def = (creator | derived | calculated)\
+ .setResultsName("type_field", listAllMatches=True)
+
+ body = Each([Optional(field_def),] + \
+ [Optional(p.parser.parse_element(indent_stack)) for p in parsers])
+ comment = Literal("#") + restOfLine + LineEnd()
+ include = (Suppress("include") + quotedString)\
+ .setResultsName("includes", listAllMatches=True)
+ rule = (Optional(persistent_identifier) + Optional(inherit_from) + \
+ Optional(override) + Optional(extend) +json_id + \
+ Optional(Suppress(",") + aliases) + Suppress(":") + \
+ INDENT + body + UNDENT)\
+ .setResultsName("rules", listAllMatches=True)
+
+ return OneOrMore(rule | include | comment.suppress())
+
+
+def _create_record_model_parser():
+ """
+ Creates a parser that can handle model definitions.
+
+ BFN like grammar::
+
+ record_model ::= python_comment | fields
+ fields ::= "fields:" INDENT [inherit_from] [list_of_fields]
+ inherit_from ::= "@inherit_from(" json_id+ ")"
+ list_of_fields ::= json_id [ "=" json_id ] # new field name = existing field name
+
+ Note: Unlike the field configuration files where you can specify more than
+ one field inside each file for the record models only one definition is
+ allowed by file.
+ """
+ indent_stack = [1]
+
+ field_def = (Word(alphas + "_", alphanums + "_") + \
+ Optional(Suppress("=") + \
+ Word(alphas + "_", alphanums + "_")))\
+ .setResultsName("field_definition")
+ inherit_from = (Suppress("@inherit_from") + \
+ originalTextFor(nestedExpr("(", ")")))\
+ .setResultsName("inherit_from")
+
+ fields = (Suppress("fields:") + \
+ indentedBlock(inherit_from | field_def, indent_stack))\
+ .setResultsName("fields")
+ comment = Literal("#") + restOfLine + LineEnd()
+ return OneOrMore(comment | Each([fields, ] + \
+ [Optional(p.parser.parse_element(indent_stack)) for p in parsers]))
+
+
+class FieldParser(object):
+ """Field definitions parser"""
+
+ _field_definitions = {}
+ """Dictionary containing all the rules needed to create and validate json fields"""
+
+ _legacy_field_matchings = {}
+ """Dictionary containing matching between the legacy master format and the current json"""
+
+ def __init__(self, namespace):
+ self.files = list(fields_definitions(namespace))
+ self.__namespace = namespace
+ self.__inherit_rules = []
+ self.__unresolved_inheritence = []
+ self.__override_rules = []
+ self.__extend_rules = []
+
+ @classmethod
+ def field_definitions(cls, namespace):
+ if namespace not in cls._field_definitions:
+ cls.reparse(namespace)
+ return cls._field_definitions.get(namespace)
+
+ @classmethod
+ def field_definition_model_based(cls, field_name, model_name, namespace):
+ if model_name in ModelParser.model_definitions(namespace):
+ field_name = ModelParser.model_definitions(namespace)[model_name] \
+ ['fields'].get(field_name, field_name)
+ return cls.field_definitions(namespace).get(field_name, None)
+
+ @classmethod
+ def legacy_field_matchings(cls, namespace):
+ if namespace not in cls._legacy_field_matchings:
+ cls.reparse(namespace)
+ return cls._legacy_field_matchings
+
+ @classmethod
+ def reparse(cls, namespace):
+ cls._field_definitions[namespace] = {}
+ cls._legacy_field_matchings = {}
+ cls(namespace)._create()
+ # It invalidates the Model definitions too as they relay on the field definitions
+ ModelParser.reparse(namespace)
+
+ def _create(self):
+ """
+ Fills up _field_definitions and _legacy_field_matchings dictionary with
+ the rules defined inside the configuration files.
+
+ It also resolve the includes present inside the configuration files and
+ recursively the ones in the other files.
+
+ This method should not be used (unless you really know what your are doing),
+ use instead :meth:`reparse`
+ """
+ already_included = [os.path.basename(f) for f in self.files]
+ for field_file in self.files:
+ parser = _create_record_field_parser()
+ field_descs = parser.parseFile(field_file, parseAll=True)
+ for include in field_descs.includes:
+ if include[0] in already_included:
+ continue
+ if not os.path.exists(include[0]):
+ raise FieldParserException("Can't find file: %s" % (include[0], ))
+ self.files.append(include[0])
+ for rule in field_descs.rules:
+ if rule.override:
+ self.__override_rules.append(rule)
+ elif rule.extend:
+ self.__extend_rules.append(rule)
+ elif rule.inherit_from:
+ self.__inherit_rules.append(rule)
+ else:
+ self._create_rule(rule)
+
+ self.__resolve_inherit_rules()
+ self.__resolve_override_rules()
+ self.__resolve_extend_rules()
+
+ def _create_rule(self, rule, override=False, extend=False):
+ """
+ Creates the field and legacy definitions.
+ The result looks like this::
+
+ {key: [key1, key2],
+ key1: {inherit_from: [],
+ override: True/False,
+ extend: True/False,
+ aliases: [],
+ persistent_identifier: num/None,
+ rules: {'master_format_1': [{rule1}, {rule2}, ...],
+ 'master_format_2': [....],
+ ......
+ 'calculated': [....],
+ 'derived': [...]}
+ }
+ }
+
+ Each of the rule (rule1, rule2, etc.) has the same content::
+
+ {'source_format' : [translation_rules]/None,
+ 'parse_first' : (parse_first_json_ids),
+ 'depends_on' : (depends_on_json_id),
+ 'only_if' : (only_if_boolean_expressions),
+ 'only_if_master_value': (only_if_master_value_boolean_expressions),
+ 'memoize' : time,
+ 'value' : value coming from master format
+ }
+
+ """
+ json_id = rule.json_id[0]
+ #Chech duplicate names
+ if json_id in self.__class__._field_definitions[self.__namespace] and not override and not extend:
+ raise FieldParserException("Name error: '%s' field name already defined"
+ % (rule.json_id[0],))
+ if not json_id in self.__class__._field_definitions[self.__namespace] and (override or extend):
+ raise FieldParserException("Name error: '%s' field name not defined"
+ % (rule.json_id[0],))
+
+ #Workaround to keep clean doctype files
+ #Just creates a dict entry with the main json field name and points it to
+ #the full one i.e.: 'authors' : ['authors[0]', 'authors[n]']
+ if '[0]' in json_id or '[n]' in json_id:
+ main_json_id = re.sub('(\[n\]|\[0\])', '', json_id)
+ if not main_json_id in self.__class__._field_definitions[self.__namespace]:
+ self.__class__._field_definitions[self.__namespace][main_json_id] = []
+ self.__class__._field_definitions[self.__namespace][main_json_id].append(json_id)
+
+ aliases = []
+ if rule.aliases:
+ aliases = rule.aliases.asList()
+
+ persistent_id = None
+ if rule.persistent_identifier:
+ persistent_id = int(rule.persistent_identifier[0][0])
+
+ inherit_from = None
+ if rule.inherit_from:
+ self.__unresolved_inheritence.append(json_id)
+ inherit_from = eval(rule.inherit_from[0])
+
+ if extend:
+ rules = self.__class__._field_definitions[self.__namespace][json_id]['rules']
+ else:
+ rules = {}
+
+ #TODO: check if pyparsing can handle this!
+ all_type_def = []
+ if rule.creator_def:
+ all_type_def = [r for r in rule.creator_def]
+ if all_type_def and rule.derived_calculated_def:
+ all_type_def.append(rule.derived_calculated_def)
+ elif rule.derived_calculated_def:
+ all_type_def = [rule.derived_calculated_def]
+
+ for r in all_type_def:
+ if r.source_format:
+ source = r.source_format[0]
+ source_tag = r.source_tag[0].split()
+ else:
+ source = rule.type_field[0]
+ source_tag = None
+
+ if source not in rules:
+ #Allow several tags point to the same json id
+ rules[source] = []
+ (depends_on, only_if, only_if_master_value,
+ parse_first, memoize) = self.__create_decorators_content(r)
+ self._create_legacy_rules(r.legacy, json_id, source)
+
+ rules[source].append({'source_tag' : source_tag,
+ 'parse_first' : parse_first,
+ 'depends_on' : depends_on,
+ 'only_if' : only_if,
+ 'only_if_master_value': only_if_master_value,
+ 'memoize' : memoize,
+ 'value' : compile(r.value[0].strip(), '', 'eval'),
+ })
+
+ if override:
+ self.__class__._field_definitions[self.__namespace][json_id]['override'] = override
+ self.__class__._field_definitions[self.__namespace][json_id]['rules'].update(rules)
+ self.__class__._field_definitions[self.__namespace][json_id]['aliases'] = \
+ aliases or self.__class__._field_definitions[self.__namespace][json_id]['aliases']
+ self.__class__._field_definitions[self.__namespace][json_id]['persistent_identifier'] = \
+ persistent_id or self.__class__._field_definitions[self.__namespace][json_id]['persistent_identifier']
+ self.__class__._field_definitions[self.__namespace][json_id]['inherit_from'] = \
+ inherit_from or self.__class__._field_definitions[self.__namespace][json_id]['inherit_from']
+ elif extend:
+ self.__class__._field_definitions[self.__namespace][json_id]['extend'] = extend
+ self.__class__._field_definitions[self.__namespace][json_id]['aliases'].extend(aliases)
+ else:
+ self.__class__._field_definitions[self.__namespace][json_id] = {'inherit_from' : inherit_from,
+ 'rules' : rules,
+ 'aliases' : aliases,
+ 'persistent_identifier': persistent_id,
+ 'override' : override,
+ 'extend' : extend,
+ }
+
+ self.__resolve_parser_extensions(rule)
+
+
+ def _create_legacy_rules(self, legacy_rules, json_id, source_format=None):
+ """
+ Creates the legacy rules dictionary::
+
+ {'100' : ['authors[0]'],
+ '100__' : ['authors[0]'],
+ '100__%': ['authors[0]'],
+ '100__a': ['auhtors[0].full_name'],
+ .......
+ }
+ """
+ if not legacy_rules:
+ return
+ for legacy_rule in legacy_rules:
+ legacy_rule = eval(legacy_rule[0])
+
+ if source_format in ('derived', 'calculated'):
+ inner_source_format = legacy_rule[0]
+ legacy_rule = legacy_rule[1]
+ else:
+ inner_source_format = source_format
+
+ if not inner_source_format in self.__class__._legacy_field_matchings:
+ self.__class__._legacy_field_matchings[inner_source_format] = {}
+
+ for field_legacy_rule in legacy_rule:
+ #Allow string and tuple in the config file
+ legacy_fields = isinstance(field_legacy_rule[0], six.string_types) and (field_legacy_rule[0], ) or field_legacy_rule[0]
+ json_field = json_id
+ if field_legacy_rule[-1]:
+ json_field = '.'.join((json_field, field_legacy_rule[-1]))
+ for legacy_field in legacy_fields:
+ if not legacy_field in self.__class__._legacy_field_matchings[inner_source_format]:
+ self.__class__._legacy_field_matchings[inner_source_format][legacy_field] = []
+ self.__class__._legacy_field_matchings[inner_source_format][legacy_field].append(json_field)
+
+ def __resolve_parser_extensions(self, rule):
+ """
+ For each of the extension available it tries to apply it in the incoming
+ rule
+ """
+ json_id = rule.json_id[0]
+ assert json_id in self.__class__._field_definitions[self.__namespace]
+ for parser_extension in parsers:
+ if getattr(rule, parser_extension.parser.__name__, None):
+ self.__class__._field_definitions[self.__namespace][json_id][parser_extension.parser.__name__] = \
+ parser_extension.parser.create_element(rule, self.__namespace)
+
+ #FIXME: it might be nice to have the decorators also extendibles
+ def __create_decorators_content(self, rule):
+ """
+ Extracts from the rule all the possible decorators.
+ """
+ depends_on = only_if = only_if_master_value = parse_first = memoize = None
+
+ if rule.depends_on:
+ depends_on = rule.depends_on[0]
+ if rule.only_if:
+ only_if = rule.only_if[0]
+ if rule.only_if_master_value:
+ only_if_master_value = rule.only_if_master_value[0]
+ if rule.parse_first:
+ parse_first = rule.parse_first[0]
+ if rule.memoize:
+ try:
+ memoize = int(rule.memoize[0][0])
+ except IndexError:
+ memoize = 300 # FIXME: Default value will be used
+
+ return (depends_on, only_if, only_if_master_value, parse_first, memoize)
+
+ def __resolve_inherit_rules(self):
+ """
+ Iterates over all the 'inherit' fields after all the normal field
+ creation to avoid problem when creating this rules.
+ """
+ def resolve_inheritance(json_id):
+ rule = self.__class__._field_definitions[self.__namespace][json_id]
+ inherit_from_list = self.__class__._field_definitions[self.__namespace][json_id]['inherit_from']
+ for inherit_json_id in inherit_from_list:
+ #Check if everithing is fine
+ if inherit_json_id == json_id:
+ raise FieldParserException("Inheritance from itself")
+ if inherit_json_id not in self.__class__._field_definitions[self.__namespace]:
+ raise FieldParserException("Unable to solve %s inheritance" % (inherit_json_id,))
+ if inherit_json_id in self.__unresolved_inheritence:
+ self._resolve_inheritance(inherit_json_id)
+ self.__unresolved_inheritence.remove(inherit_json_id)
+ inherit_rule = self.__class__._field_definitions[self.__namespace][inherit_json_id]
+ for format in inherit_rule['rules']:
+ if not format in rule['rules']:
+ rule['rules'][format] = []
+ rule['rules'][format].extend(inherit_rule['rules'][format])
+ # rule['checker'].extend(inherit_rule['checker'])
+
+ for rule in self.__inherit_rules:
+ self._create_rule(rule)
+
+ #Resolve inheritance
+ for i in xrange(len(self.__unresolved_inheritence) - 1, -1, -1):
+ resolve_inheritance(self.__unresolved_inheritence[i])
+ del self.__unresolved_inheritence[i]
+
+
+ def __resolve_override_rules(self):
+ """
+ Iterates over all the 'override' field to override the already created
+ fields.
+ """
+ for rule in self.__override_rules:
+ self._create_rule(rule, override=True)
+
+ def __resolve_extend_rules(self):
+ """
+ Iterates over all the 'extend' field to extend the rule definition of this
+ field.
+ """
+ for rule in self.__extend_rules:
+ self._create_rule(rule, extend=True)
+
+
+class ModelParser(object):
+ """Record model parser"""
+
+ _model_definitions = {}
+ """ """
+
+ def __init__(self, namespace):
+ #Autodiscover .cfg files
+ self.files = list(models_definitions(namespace))
+ self.__namespace = namespace
+
+ @classmethod
+ def model_definitions(cls, namespace):
+ if namespace not in cls._model_definitions:
+ cls.reparse(namespace)
+ return cls._model_definitions.get(namespace)
+
+ @classmethod
+ def reparse(cls, namespace):
+ cls._model_definitions[namespace] = {}
+ cls(namespace)._create()
+
+ def _create(self):
+ """
+ Fills up _model_definitions dictionary with what is written inside the
+ *.cfg model descriptions
+
+ It also resolve inheritance at creation time and name matching for the
+ field names present inside the model file
+
+ The result looks like this::
+
+ {'model': {'fields': {'name_for_fieldfield1': json_id1,
+ 'name_for_field2': json_id2,
+ ....
+ 'name_for_fieldN': fieldN },
+ 'inherit_from: [(inherit_from_list), ...]
+ },
+ ...
+ }
+
+ This method should not be used (unless you really know what your are doing),
+ use instead :meth:`reparse`
+
+ :raises: ModelParserException in case of missing model definition
+ (helpful if we use inheritance) or in case of unknown field name.
+ """
+ for model_file in self.files:
+ parser = _create_record_model_parser()
+ model_name = os.path.basename(model_file).split('.')[0]
+ if model_name in self.__class__._model_definitions[self.__namespace]:
+ raise ModelParserException("Already defined record model: %s" % (model_name,))
+ self.__class__._model_definitions[self.__namespace][model_name] = {'fields': {},
+ 'super': [],
+ }
+ model_definition = parser.parseFile(model_file, parseAll=True)
+
+ if not model_definition.fields:
+ raise ModelParserException("Field definition needed")
+ for field_def in model_definition.fields[0]:
+ if field_def.inherit_from:
+ self.__class__._model_definitions[self.__namespace][model_name]['super'].extend(eval(field_def[0]))
+ else:
+ if len(field_def) == 1:
+ json_id = field_def[0]
+ field_name = json_id
+ else:
+ field_name = field_def[0]
+ json_id = field_def[1]
+ if not json_id in FieldParser.field_definitions(self.__namespace):
+ raise ModelParserException("Unknown field name: %s" % (json_id,))
+
+ self.__class__._model_definitions[self.__namespace][model_name]['fields'][field_name] = json_id
+
+ self.__resolve_parser_extensions(model_name, model_definition)
+
+ for model, model_definition in self.__class__._model_definitions[self.__namespace].items():
+ model_definition['fields'] = self.__resolve_inheritance(model)
+
+ def __resolve_inheritance(self, model):
+ """
+ Resolves the inheritance
+
+ :param model: name of the super model
+ :type model: string
+
+ :return: List of new fields to be added to the son model
+ :raises: ModelParserException if the super model does not exist.
+ """
+ try:
+ model_definition = self.__class__._model_definitions[self.__namespace][model]
+ except KeyError:
+ raise ModelParserException("Missing model definition for %s" % (model,))
+ fields = {}
+ for super_model in model_definition['super']:
+ fields.update(self.__resolve_inheritance(super_model))
+ fields.update(model_definition['fields'])
+ return fields
+
+ def __resolve_parser_extensions(self, model_name, model_def):
+ """
+ For each of the extension available it tries to apply it in the incoming
+ rule
+ """
+ assert model_name in self.__class__._model_definitions[self.__namespace]
+ for parser_extension in parsers:
+ if getattr(model_def, parser_extension.parser.__name__, None):
+ self.__class__._model_definitions[self.__namespace][model_name][parser_extension.parser.__name__] = \
+ parser_extension.parser.create_element(model_def, self.__namespace)
+
+def guess_legacy_field_names(fields, master_format, namespace):
+ """
+ Using the legacy rules written in the config file (@legacy) tries to find
+ the equivalent json field for one or more legacy fields.
+
+ >>> guess_legacy_fields(('100__a', '245'), 'marc')
+ {'100__a':['authors[0].full_name'], '245':['title']}
+ """
+ res = {}
+ if isinstance(fields, six.string_types):
+ fields = (fields, )
+ for field in fields:
+ try:
+ res[field] = FieldParser.legacy_field_matchings(namespace)[master_format].get(field, [])
+ except:
+ res[field] = []
+ return res
+
+def get_producer_rules(field, code, namespace):
+ """docstring for get_producer_rules"""
+
+ rule = FieldParser.field_definitions(namespace)[field]
+ if isinstance(rule, list):
+ if len(rule) == 1:
+ # case field[n]
+ return [(rule[0].replace('[n]', ''), FieldParser.field_definitions(namespace)[rule[0]]['producer'].get(code, {}))]
+ else:
+ # case field[1], field[n]
+ rules = []
+ for new_field in rule:
+ rules.append((new_field.replace('[n]', '[1:]'), FieldParser.field_definitions(namespace)[new_field]['producer'].get(code, {})))
+ return rules
+ else:
+ return [(field, rule['producer'].get(code, {}))]
+
+#pylint: disable=R0921
+class BaseExtensionParser(object):
+ """Base class for the configuration file extensions"""
+ @classmethod
+ def parse_element(cls, indent_stack):
+ """
+ Using pyparsing defines a piece of the grammar to parse the
+ extension from configuration file
+
+ :return: pyparsing ParseElement
+ """
+ raise NotImplemented
+
+ @classmethod
+ def create_element(cls, rule, namespace):
+ """
+ Once the extension is parsed defines the actions that have to be taken
+ to store inside the field_definitions the information needed or useful.
+
+ :return: content of the key cls.__name__ inside the field_definitions
+ """
+ raise NotImplemented
+
+ @classmethod
+ def add_info_to_field(cls, json_id, info):
+ """
+ Optional method to define which information goes inside the
+ __meta_metadata__ dictionary and how.
+
+ :return: dictionary to update the existing one inside __meta_metadata__
+ """
+ return dict()
diff --git a/invenio/modules/jsonalchemy/reader.py b/invenio/modules/jsonalchemy/reader.py
new file mode 100644
index 000000000..c84d534d1
--- /dev/null
+++ b/invenio/modules/jsonalchemy/reader.py
@@ -0,0 +1,430 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.reader
+ ----------------------------------
+
+ Default reader, it could be considered as an interface to be implemented by
+ the other readers.
+"""
+import datetime
+import six
+
+from invenio.base.utils import try_to_eval
+from invenio.utils.datastructures import SmartDict
+
+from .errors import ReaderException
+from .parser import FieldParser, ModelParser
+from .registry import functions, parsers
+
+
+class Reader(object):
+ """Default reader"""
+
+ def __init__(self, blob=None, **kwargs):
+ """
+ :param blob:
+ """
+ self.blob = blob
+ self.json = None
+ self._additional_info = kwargs
+ self._additional_info['model'] = kwargs.get('model', '__default__')
+ self._additional_info['namespace'] = kwargs.get('namespace', None)
+
+ if self._additional_info['namespace'] is None:
+ raise ReaderException('A namespace is needed to instantiate a reader')
+
+ self._parsed = []
+
+ @staticmethod
+ def split_blob(blob, schema=None, **kwargs):
+ """
+ In case of several records inside the blob this method specify how to
+ split then and work one by one afterwards.
+ """
+ raise NotImplementedError()
+
+ @property
+ def field_definitions(self):
+ return FieldParser.field_definitions(self._additional_info['namespace'])
+
+ @property
+ def model_definitions(self):
+ return ModelParser.model_definitions(self._additional_info['namespace'])
+
+ @property
+ def functions(self):
+ return functions(self._additional_info['namespace'])
+
+ def translate(self):
+ """
+ It transforms the incoming blob into a json structure using the rules
+ described into the field and model definitions.
+ To apply this rules it takes into account the type of the reader, which
+ in fact means the type of the source format or `master_format`
+
+ :return: Json structure (typically a dictionary)
+ """
+ if not self.blob:
+ raise ReaderException("To perform a 'translate' operation a blob is needed")
+
+ # If we already have the json return it, use add or update to modify it
+ if self.json:
+ return self.json
+
+ self.json = {}
+ self.json['__meta_metadata__'] = {}
+ self.json['__meta_metadata__']['__additional_info__'] = self._additional_info
+ self.json['__meta_metadata__']['__aliases__'] = {}
+ self.json['__meta_metadata__']['__errors__'] = []
+ self.json['__meta_metadata__']['__continuable_errors__'] = []
+ if self._additional_info['model'] == '__default__' or \
+ self._additional_info['model'] not in self.model_definitions:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append("Warning - Using 'default' model for 'transalte', given model: '%s'" % (self._additional_info['model'], ))
+ fields = dict(zip(self.field_definitions.keys(), self.field_definitions.keys()))
+ else:
+ fields = self.model_definitions[self._additional_info['model']]['fields']
+
+ self.add(self.json, self.blob, fields)
+ return self.json._dict
+
+ def add(self, json, blob, fields):
+ """Adds the list of fields to the json structure"""
+ self.json = json if isinstance(json, SmartDict) else SmartDict(json)
+ self.blob = blob
+
+ if not self.blob or not self.json:
+ raise ReaderException("To perform an 'add' operation a json structure and a blob are needed")
+
+ if not isinstance(fields, dict):
+ if isinstance(fields, six.string_types):
+ fields = (fields, )
+ try:
+ model = self.json['__meta_metadata__']['__additional_info__']['model']
+ except KeyError as e:
+ raise ReaderException('The json structure must contain a model (%s)' % (e, ))
+
+ if model == '__default__' or model not in self.model_definitions:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append("Warning - Using 'default' model for 'add', given model: '%s'" % (model, ))
+ fields = dict(zip(fields, fields))
+ else:
+ fields = dict((field, self.model_definitions[model]['fields'].get(field, field))
+ for field in fields)
+
+ self._prepare_blob()
+
+ for field_name, json_id in fields.items():
+ self._unpack_rule(json_id, field_name)
+
+ self._post_process_json()
+
+
+ def set(self, json, field):
+ """
+
+ """
+ self.json = json if isinstance(json, SmartDict) else SmartDict(json)
+ try:
+ model = self.json['__meta_metadata__']['__additional_info__']['model']
+ except KeyError as e:
+ raise ReaderException('The json structure must contain a model (%s)' % (e, ))
+
+ if model == '__default__' or model not in self.model_definitions:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append("Warning - Using 'default' model for 'add', given model: '%s'" % (model, ))
+ json_id = field
+ else:
+ json_id = self.model_definitions[model]['fields'].get(field, field)
+
+ try:
+ rule = self.field_definitions[json_id]
+ except KeyError:
+ rule = {}
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append("Adding a new field '%s' without definition" % (field))
+
+ try:
+ if self.json['__meta_metadata__']['__additional_info__']['master_format'] in rule['rules']:
+ rule_def = rule['rules'][self.json['__meta_metadata__']['__additional_info__']['master_format']][0]
+ rule_type = 'creator'
+ elif 'derived' in rule['rules']:
+ rule_def = rule['rules']['derived'][0]
+ rule_type = 'derived'
+ elif 'calculated' in rule['rules']:
+ rule_def = rule['rules']['calculated'][0]
+ rule_type = 'calculated'
+ else:
+ rule_def = {}
+ rule_type = 'UNKNOWN'
+ except KeyError:
+ rule_def = {}
+ rule_type = 'UNKNOWN'
+
+ self.json['__meta_metadata__'][field] = self._find_meta_metadata(json_id, field, rule_type, rule, rule_def)
+
+ def update(self, json, blob, fields=None):
+ """
+ Tries to update the json structure with the fields given.
+ If no fields are given then it will try to update all the fields inside
+ the json structure.
+ """
+
+ if not blob or not blob:
+ raise ReaderException("To perform an 'add' operation a json structure and a blob are needed")
+
+ try:
+ model = json['__meta_metadata__']['__additional_info__']['model']
+ except KeyError as e:
+ raise ReaderException('The json structure must contain a model (%s)' % (e, ))
+
+ if not fields:
+ fields = dict(zip(json.keys(), json.keys()))
+ if model == '__default__' or model not in self.model_definitions:
+ json['__meta_metadata__']['__continuable_errors__']\
+ .append("Warning - Using 'default' model for 'update', given model: '%s'" % (model, ))
+ else:
+ fields = dict(fields, **self.model_definitions[model]['fields'])
+ elif not isinstance(fields, dict):
+ if isinstance(fields, six.string_types):
+ fields = (fields, )
+ if model == '__default__' or model not in self.model_definitions:
+ json['__meta_metadata__']['__continuable_errors__']\
+ .append("Warning - Using 'default' model for 'update', given model: '%s'" % (model, ))
+ fields = dict(zip(fields, fields))
+ else:
+ fields = dict((field, self.model_definitions[model]['fields'].get(field, field))
+ for field in fields)
+
+# for key in fields.keys():
+# del json['key']
+
+ self.add(json, blob, fields)
+
+
+ def validate(self, reset=True):
+ """docstring for validate"""
+ pass
+
+ def _prepare_blob(self, *args, **kwargs):
+ """
+ Responsible of doing any kind of transformation over the blob before the
+ translation begins
+ """
+ raise NotImplemented
+
+ def _get_elements_from_blob(self, regex_key):
+ """
+ Should handle 'entire_record' and '*'
+ Not an iterator!
+ """
+ raise NotImplemented
+
+
+ def _unpack_rule(self, json_id, field_name=None):
+ """From the field definitions extract the rules an tries to apply them"""
+ try:
+ rule_def = self.field_definitions[json_id]
+ except KeyError as e:
+ self.json['__meta_metadata__']['__continuable_errors__'].append("Error - Unable to find '%s' field definition" % (json_id, ))
+ return False
+
+ if not field_name:
+ model = self.json['__meta_metadata__']['__additional_info__']['model']
+ if model == '__default__' or model not in self.model_definitions:
+ field_name = json_id
+ else:
+ field_name = self.model_definitions[model].get(json_id, json_id)
+
+ # Undo the workaround for [0] and [n]
+ if isinstance(rule_def, list):
+ return all(map(self._unpack_rule, rule_def))
+
+ # Already parsed, avoid doing it again
+ if (json_id, field_name) in self._parsed:
+ return field_name in self.json
+
+ self._parsed.append((json_id, field_name))
+ return self._apply_rules(json_id, field_name, rule_def) or \
+ self._apply_virtual_rules(json_id, field_name, rule_def)
+
+ def _apply_rules(self, json_id, field_name, rule_def):
+ """Tries to apply a 'creator' rule"""
+ applied = False
+ for rule in rule_def['rules'].get(
+ self.json['__meta_metadata__']['__additional_info__']['master_format'], []):
+ elements = self._get_elements_from_blob(rule['source_tag'])
+ if not elements:
+ self._set_default_value(json_id, field_name)
+ return False
+ if not self._evaluate_decorators(rule):
+ return False
+ if 'entire_record' in rule['source_tag'] or '*' in rule['source_tag']:
+ try:
+ value = try_to_eval(rule['value'], self.functions, value=elements, self=self.json)
+ info = self._find_meta_metadata(json_id, field_name, 'creator', rule, rule_def)
+ if 'json_ext' in rule_def:
+ value = rule_def['json_ext']['dumps'](value)
+ self.json.set(field_name, value, extend=True)
+ self.json['__meta_metadata__.%s' % (SmartDict.main_key_pattern.sub('', field_name), )] = info
+ applied = True
+ except Exception as e:
+ self.json['__meta_metadata__']['__errors__']\
+ .append('Rule Error - Unable to apply rule for field %s - %s' % (field_name, str(e)),)
+ applied = False
+
+ else:
+ for element in elements:
+ if not isinstance(element, (list, tuple)):
+ element = (element, )
+ applied = False
+ for e in element:
+ if rule['only_if_master_value'] and \
+ not all(try_to_eval(rule['only_if_master_value'], self.functions, value=e, self=self.json)):
+ applied = applied or False
+ else:
+ try:
+ value = try_to_eval(rule['value'], self.functions, value=e, self=self.json)
+ info = self._find_meta_metadata(json_id, field_name, 'creator', rule, rule_def)
+ if 'json_ext' in rule_def:
+ value = rule_def['json_ext']['dumps'](value)
+ self.json.set(field_name, value, extend=True)
+ self.json['__meta_metadata__.%s' % (SmartDict.main_key_pattern.sub('', field_name), )] = info
+ applied = applied or True
+ except Exception as e:
+ self.json['__meta_metadata__']['__errors__']\
+ .append('Rule Error - Unable to apply rule for field %s - %s' % (field_name, str(e)),)
+ applied = applied or False
+
+ if field_name not in self.json or not applied:
+ self._set_default_value(json_id, field_name)
+ return applied
+
+ def _apply_virtual_rules(self, json_id, field_name, rule_def):
+ """Tries to apply either a 'derived' or 'calculated' rule"""
+ rules = []
+ rules.append(('calculated', rule_def['rules'].get('calculated', [])))
+ rules.append(('derived', rule_def['rules'].get('derived', [])))
+ for (rule_type, rrules) in rules:
+ for rule in rrules:
+ if not self._evaluate_decorators(rule):
+ return False
+ try:
+ info = self._find_meta_metadata(json_id, field_name, rule_type, rule, rule_def)
+ if rule_type == 'derived' or rule['memoize']:
+ value = try_to_eval(rule['value'], self.functions, self=self.json)
+ if 'json_ext' in rule_def:
+ value = rule_def['json_ext']['dumps'](value)
+ else:
+ value = None
+
+ self.json.set(field_name, value, extend=True)
+ self.json['__meta_metadata__.%s' % (SmartDict.main_key_pattern.sub('', field_name), )] = info
+ except Exception as e:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append('Virtual Rule CError - Unable to evaluate %s - %s' % (field_name, str(e)))
+ return False
+
+ if field_name not in self.json:
+ self._set_default_value(json_id, field_name)
+ return True
+
+ def _evaluate_decorators(self, rule):
+ """Evaluates all 'decorators' related with the current rule"""
+ if rule['parse_first']:
+ map(self._unpack_rule, try_to_eval(rule['parse_first']))
+ if rule['depends_on']:
+ for key in try_to_eval(rule['depends_on']):
+ if key in self.json:
+ continue
+ main_key = SmartDict.main_key_pattern.sub('', key)
+ if not self._unpack_rule(main_key):
+ return False
+ if rule['only_if'] and not all(try_to_eval(rule['only_if'], self.functions, self=self.json)):
+ return False
+ return True
+
+ def _find_meta_metadata(self, json_id, field_name, rule_type, rule, rule_def):
+ """Given one rule fills up the parallel dictionary with the needed meta-metadata"""
+ for alias in rule_def.get('aliases', []):
+ self.json['__meta_metadata__.__aliases__.%s' % (alias, )] = field_name
+ info = {}
+ info['timestamp'] = datetime.datetime.now().isoformat()
+ if rule_def.get('persistent_identifier', None) is not None:
+ info['pid'] = rule_def['persistent_identifier']
+ info['memoize'] = rule.get('memoize', None)
+ info['type'] = rule_type
+ if rule_type in ('calculated', 'derived'):
+ info['function'] = (json_id, 'rules', rule_type, 0, 'value')
+ elif rule_type == 'UNKNOWN':
+ info['function'] = 'UNKNOWN'
+ info['source_tag'] = 'UNKNOWN'
+ else:
+ info['source_tag'] = rule['source_tag']
+
+ #Check the extensions
+ for parser_extension in parsers:
+ info.update(parser_extension.parser.add_info_to_field(json_id, rule_def))
+
+ return info
+
+ def _set_default_value(self, json_id, field_name):
+ """
+ Finds the default value inside the schema, if any
+
+ :return: tuple containing if the value is required and the default value.
+ """
+ schema = self.field_definitions[json_id].get('schema', {}).get(json_id)
+ if schema and 'default' in schema:
+ try:
+ value = schema['default']()
+ try:
+ value = self.field_definitions[json_id]['json_ext']['dumps'](value)
+ except KeyError:
+ pass
+ self.json.set(field_name, value, extend=True)
+ except Exception, e:
+ self.json['__meta_metadata__']['__continuable_errors__']\
+ .append('Default Value CError - Unable to set default value for %s - %s' % (field_name, str(e)))
+
+
+ def _post_process_json(self):
+ """
+ Responsible of doing any kind of transformation over the json structure
+ after it is created, e.g. pruning the json to delete None values or
+ singletons.
+ """
+ def remove_none_values(obj):
+ if isinstance(obj, dict):
+ for key, value in obj.items():
+ if value is None:
+ del obj[key]
+ else:
+ remove_none_values(value)
+ if isinstance(obj, list):
+ for element in obj:
+ if element is None:
+ obj.remove(element)
+ else:
+ remove_none_values(element)
+
+ map(remove_none_values, [value for key, value in self.json.items() if not key == '__meta_metadata__'])
+
diff --git a/invenio/modules/jsonalchemy/registry.py b/invenio/modules/jsonalchemy/registry.py
new file mode 100644
index 000000000..d1bf7612b
--- /dev/null
+++ b/invenio/modules/jsonalchemy/registry.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from invenio.ext.registry import AutoDiscoverRegistry, AutoDiscoverSubRegistry, \
+ PkgResourcesDiscoverRegistry, RegistryProxy
+from invenio.utils.datastructures import LazyDict
+
+jsonext = lambda namespace: RegistryProxy(namespace, AutoDiscoverRegistry, namespace)
+
+fields_definitions = lambda namespace: RegistryProxy(
+ namespace + '.fields', PkgResourcesDiscoverRegistry, 'fields',
+ registry_namespace=jsonext(namespace))
+
+models_definitions = lambda namespace: RegistryProxy(
+ namespace + '.models', PkgResourcesDiscoverRegistry, 'models',
+ registry_namespace=jsonext(namespace))
+
+function_proxy = lambda namespace: RegistryProxy(
+ namespace + '.functions', AutoDiscoverSubRegistry, 'functions',
+ registry_namespace=jsonext(namespace))
+def functions(namespace):
+ funcs = dict((module.__name__.split('.')[-1],
+ getattr(module, module.__name__.split('.')[-1], ''))
+ for module in function_proxy('jsonext'))
+ funcs.update((module.__name__.split('.')[-1],
+ getattr(module, module.__name__.split('.')[-1], ''))
+ for module in function_proxy(namespace))
+ return funcs
+
+parsers = RegistryProxy('jsonext.parsers', AutoDiscoverSubRegistry,
+ 'parsers', registry_namespace=jsonext('jsonext'))
+
+producers_proxy = RegistryProxy('jsonext.producers', AutoDiscoverSubRegistry,
+ 'producers', registry_namespace=jsonext('jsonext'))
+producers = LazyDict(lambda: dict((module.__name__.split('.')[-1], module.produce)
+ for module in producers_proxy))
+
+readers_proxy = RegistryProxy('jsonext.readers', AutoDiscoverSubRegistry,
+ 'readers', registry_namespace=jsonext('jsonext'))
+readers = LazyDict(lambda: dict((module.reader.__master_format__, module.reader)
+ for module in readers_proxy))
diff --git a/invenio/utils/jsonalchemy/storage.py b/invenio/modules/jsonalchemy/storage.py
similarity index 68%
rename from invenio/utils/jsonalchemy/storage.py
rename to invenio/modules/jsonalchemy/storage.py
index a1e09b6b6..6b8081698 100644
--- a/invenio/utils/jsonalchemy/storage.py
+++ b/invenio/modules/jsonalchemy/storage.py
@@ -1,90 +1,98 @@
# -*- coding: utf-8 -*-
-
+##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
- invenio.utils.jsonalchemy.storage
- ---------------------------
+ invenio.modules.jsonalchemy.storage
+ -----------------------------------
- Record storage interface.
+ Json storage engine interface
"""
class Storage(object):
- """
- Defatul storage interface
- """
+ """Default storage engine interface"""
+
#TODO: set return values on success and error and setup a log function
+ #TODO: create a query class to mimic SQLAlchemy query object
def __init__(self, model, **kargs):
- """docstring for __init__"""
- raise NotImplementedError()
+ """
+ :param: model:
+ """
+ raise NotImplemented
def save_one(self, json, id=None):
"""Stores one json in the storage system"""
- raise NotImplementedError()
def save_many(self, jsons, ids=None):
"""
Stores many json in the storage system, as elements on the
iterable jsons.
"""
- raise NotImplementedError()
+ raise NotImplemented
def update_one(self, json, id=None):
"""
- Updates one json , if id is None a field representing the id is
+ Updates one json , if id is None a field representing the id is
expected inside the json object.
"""
- raise NotImplementedError()
+ raise NotImplemented
def update_many(self, jsons, ids=None):
"""Update many json objects following the same rule as update_one"""
- raise NotImplementedError()
+ raise NotImplemented
def get_one(self, id):
"""Returns the json matching the id"""
- raise NotImplementedError()
+ raise NotImplemented
def get_many(self, ids):
"""Returns an iterable of json objects which id is inside ids"""
- raise NotImplementedError()
+ raise NotImplemented
def get_field_values(ids, field, repetitive_values=True, count=False,
- include_recid=False, split_by=0):
+ include_recid=False, split_by=0):
"""
Returns a list of field values for field for the given ids.
:param ids: list (or iterable) of integers
:param repetitive_values: if set to True, returns all values even if
they are doubled. If set to False, then return unique values only.
:param count: in combination with repetitive_values=False, adds to the
result the number of occurrences of the field.
:param split: specifies the size of the output.
"""
- raise NotImplementedError()
+ raise NotImplemented
def get_fields_values(ids, fields, repetitive_values=True, count=False,
- include_recid=False, split_by=0):
+ include_recid=False, split_by=0):
"""
- As in :meth:`get_field_values` but in this case returns a dictionary with each
- of the fields and the list of field values.
+ As in :meth:`get_field_values` but in this case returns a dictionary
+ with each of the fields and the list of field values.
"""
- raise NotImplementedError()
-
+ raise NotImplemented
+ def search(query):
+ """
+ Retrieves all entries which match the query JSON prototype document.
+ This method should not be used on storage engines without native JSON
+ support (e.g., MySQL). Returns a cursor over the matched documents.
+ :param query: dictionary specifying the search prototype document
+ """
+ raise NotImplemented
diff --git a/invenio/core/__init__.py b/invenio/modules/jsonalchemy/testsuite/__init__.py
similarity index 83%
copy from invenio/core/__init__.py
copy to invenio/modules/jsonalchemy/testsuite/__init__.py
index eb537e874..0fef19ac6 100644
--- a/invenio/core/__init__.py
+++ b/invenio/modules/jsonalchemy/testsuite/__init__.py
@@ -1,18 +1,23 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
+## Copyright (C) 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+from .test_parser import *
+from .test_reader import *
+from .test_storage_engine import *
+from .test_wrapper import *
diff --git a/invenio/modules/jsonalchemy/testsuite/fields/authors.cfg b/invenio/modules/jsonalchemy/testsuite/fields/authors.cfg
new file mode 100644
index 000000000..c56766f06
--- /dev/null
+++ b/invenio/modules/jsonalchemy/testsuite/fields/authors.cfg
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+#Test for loops
+include "authors.cfg"
+
+authors[0], creator:
+ creator:
+ @legacy((("100", "100__", "100__%"), ""),
+ ("100__a", "first author name", "full_name"),
+ ("100__e", "relator_name"),
+ ("100__h", "CCID"),
+ ("100__i", "INSPIRE_number"),
+ ("100__u", "first author affiliation", "affiliation"))
+ marc, "100__", { 'full_name':value['a'], 'first_name':util_split(value['a'],',',1), 'last_name':util_split(value['a'],',',0), 'relator_name':value['e'], 'CCID':value['h'], 'INSPIRE_number':value['i'], 'affiliation':value['u'] }
+ checker:
+ check_field_existence(0,1)
+ check_field_type('str')
+ producer:
+ json_for_marc(), {"100__a": "full_name", "100__e": "relator_name", "100__h": "CCID", "100__i": "INSPIRE_number", "100__u": "affiliation"}
+ json_for_dc(), {"dc:creator": "full_name"}
+ description:
+ """Main Author"""
+
+authors[n], contributor:
+ creator:
+ @legacy((("700", "700__", "700__%"), ""),
+ ("700__a", "additional author name", "full_name"),
+ ("700__u", "additional author affiliation", "affiliation"))
+ marc, "700__", {'full_name': value['a'], 'first_name':util_split(value['a'],',',1), 'last_name':util_split(value['a'],',',0), 'relator_name':value['e'], 'CCID':value['h'], 'INSPIRE_number':value['i'], 'affiliation':value['u'] }
+ checker:
+ check_field_existence(0,'n')
+ check_field_type('str')
+ producer:
+ json_for_marc(), {"700__a": "full_name", "700__e": "relator_name", "700__h": "CCID", "700__i": "INSPIRE_number", "700__u": "affiliation"}
+ json_for_dc(), {"dc:contributor": "full_name"}
+ description:
+ """Authors"""
diff --git a/invenio/modules/jsonalchemy/testsuite/fields/fields.cfg b/invenio/modules/jsonalchemy/testsuite/fields/fields.cfg
new file mode 100644
index 000000000..0401f5cdf
--- /dev/null
+++ b/invenio/modules/jsonalchemy/testsuite/fields/fields.cfg
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+abstract:
+ creator:
+ @legacy((("520", "520__", "520__%"), "abstract", ""),
+ ("520__a", "abstract", "summary"),
+ ("520__b", "expansion"),
+ ("520__9", "number"))
+ marc, "520__", {'summary':value['a'], 'expansion':value['b'], 'number':value['9']}
+ producer:
+ json_for_marc(), {"520__a": "summary", "520__b": "expansion", "520__9": "number"}
+ json_for_dc(), {"dc:description":"summary"}
+
+collection:
+ creator:
+ @legacy((("980", "980__", "980__%"), ""),
+ ("980__%", "collection identifier", ""),
+ ("980__a", "primary"),
+ ("980__b", "secondary"),
+ ("980__c", "deleted"))
+ marc, "980__", { 'primary':value['a'], 'secondary':value['b'], 'deleted':value['c'] }
+ producer:
+ json_for_marc(), {"980__a":"primary", "980__b":"secondary", "980__c":"deleted"}
+
+@persistent_identifier(3)
+doi:
+ creator:
+ @legacy((("024", "0247_", "0247_%"), ""),
+ ("0247_a", ""))
+ marc, "0247_", get_doi(value)
+ checker:
+ check_field_existence(0,1)
+ producer:
+ json_for_marc(), {'0247_2': 'str("DOI")', '0247_a': ''}
+
+fft[n]:
+ creator:
+ @legacy(("FFT__a", "path"),
+ ("FFT__d", "description"),
+ ("FFT__f", "eformat"),
+ ("FFT__i", "temporary_id"),
+ ("FFT__m", "new_name"),
+ ("FFT__o", "flag"),
+ ("FFT__r", "restriction"),
+ ("FFT__s", "timestamp"),
+ ("FFT__t", "docfile_type"),
+ ("FFT__v", "version"),
+ ("FFT__x", "icon_path"),
+ ("FFT__z", "comment"),
+ ("FFT__w", "document_moreinfo"),
+ ("FFT__p", "version_moreinfo"),
+ ("FFT__b", "version_format_moreinfo"),
+ ("FFT__f", "format_moreinfo"))
+ marc, "FFT__", {'path': value['a'],
+ 'description': value['d'],
+ 'eformat': value['f'],
+ 'temporary_id': value['i'],
+ 'new_name': value['m'],
+ 'flag': value['o'],
+ 'restriction': value['r'],
+ 'timestamp': value['s'],
+ 'docfile_type': value['t'],
+ 'version': value['v'],
+ 'icon_path': value['x'],
+ 'comment': value['z'],
+ 'document_moreinfo': value['w'],
+ 'version_moreinfo': value['p'],
+ 'version_format_moreinfo': value['b'],
+ 'format_moreinfo': value['u']
+ }
+ @only_if_value((is_local_url(value['u']), ))
+ marc, "8564_", {'hots_name': value['a'],
+ 'access_number': value['b'],
+ 'compression_information': value['c'],
+ 'path':value['d'],
+ 'electronic_name': value['f'],
+ 'request_processor': value['h'],
+ 'institution': value['i'],
+ 'formart': value['q'],
+ 'settings': value['r'],
+ 'file_size': value['s'],
+ 'url': value['u'],
+ 'subformat':value['x'],
+ 'description':value['y'],
+ 'comment':value['z']}
+ producer:
+ json_for_marc(), {"FFT__a": "path", "FFT__d": "description", "FFT__f": "eformat", "FFT__i": "temporary_id", "FFT__m": "new_name", "FFT__o": "flag", "FFT__r": "restriction", "FFT__s": "timestamp", "FFT__t": "docfile_type", "FFT__v": "version", "FFT__x": "icon_path", "FFT__z": "comment", "FFT__w": "document_moreinfo", "FFT__p": "version_moreinfo", "FFT__b": "version_format_moreinfo", "FFT__f": "format_moreinfo"}
+
+isbn:
+ creator:
+ @legacy((("020", "020__", "020__%"), ""),
+ ("020__a", "isbn", "isbn"),
+ ("020__u", "medium"))
+ marc, "020__", {'isbn':value['a'], 'medium':value['u']}
+ checker:
+ check_field_type('isbn', 'isbn')
+ producer:
+ json_for_marc(), {"020__a": "isbn", "020__u": "medium"}
+
+keywords[n]:
+ creator:
+ @legacy((("653", "6531_", "6531_%"), ""),
+ ("6531_a", "keyword", "term"),
+ ("6531_9", "institute"))
+ marc, "6531_", { 'term': value['a'], 'institute': value['9'] }
+ checker:
+ check_field_existence(0,'n')
+ check_field_type('str')
+ producer:
+ json_for_marc(), {"6531_a": "term", "6531_9": "institute"}
+
+language:
+ creator:
+ @legacy((("041", "041__", "041__%"), ""),
+ ("041__a", ""))
+ marc, "041__", value['a']
+ producer:
+ json_for_marc(), {"041__a": ""}
+ json_for_dc(), {"dc:language": ""}
+
+modification_date, version_id:
+ schema:
+ {'modification_date': {'type': 'datetime', 'required': True, 'default': lambda: __import__('datetime').datetime.now()}}
+ creator:
+ @legacy(('005', ''),)
+ marc, '005', datetime.datetime(*(time.strptime(value, "%Y%m%d%H%M%S.0")[0:6]))
+ json:
+ dumps, lambda d: d.isoformat()
+ loads, lambda d: __import__('datetime').datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
+
+primary_report_number:
+ creator:
+ @legacy((("037", "037__", "037__%"), ""),
+ ("037__a", "primary report number", ""), )
+ marc, "037__", value['a']
+ producer:
+ json_for_marc(), {"037__a": ""}
+
+reference:
+ creator:
+ @legacy((("999", "999C5", "999C5%"), ""),
+ ("999C5", "reference", ""),
+ ("999C5a", "doi"),
+ ("999C5h", "authors"),
+ ("999C5m", "misc"),
+ ("999C5n", "issue_number"),
+ ("999C5o", "order_number"),
+ ("999C5p", "page"),
+ ("999C5r", "report_number"),
+ ("999C5s", "title"),
+ ("999C5u", "url"),
+ ("999C5v", "volume"),
+ ("999C5y", "year"),)
+ marc, "999C5", {'doi':value['a'], 'authors':value['h'], 'misc':value['m'], 'issue_number':value['n'], 'order_number':value['o'], 'page':value['p'], 'report_number':value['r'], 'title':value['s'], 'url':value['u'], 'volume':value['v'], 'year':value['y'],}
+ producer:
+ json_for_marc(), {"999C5a": "doi", "999C5h": "authors", "999C5m": "misc", "999C5n": "issue_number", "999C5o":"order_number", "999C5p":"page", "999C5r":"report_number", "999C5s":"title", "999C5u":"url", "999C5v":"volume", "999C5y": "year"}
+
+@persistent_identifier(2)
+system_control_number:
+ creator:
+ @legacy((("035", "035__", "035__%"), ""),
+ ("035__a", "system_control_number"),
+ ("035__9", "institute"))
+ marc, "035__", {'value': value['a'], 'canceled':value['z'], 'linkpage':value['6'], 'institute':value['9']}
+ producer:
+ json_for_marc(), {"035__a": "system_control_number", "035__9": "institute"}
+
+@persistent_identifier(1)
+system_number:
+ creator:
+ @legacy((("970", "970__", "970__%"), ""),
+ ("970__a", "sysno"),
+ ("970__d", "recid"))
+ marc, "970__", {'value':value['a'], 'recid':value['d']}
+ checker:
+ check_field_existence(0,1)
+ producer:
+ json_for_marc(), {"970__a": "sysno", "970__d": "recid"}
+
+#To be overwritten by test_tiltle.cfg
+title:
+ """Some useless documentation"""
+ creator:
+ marc, "245__", value['foo']
+
+url:
+ creator:
+ @legacy((("856", "8564_", "8564_%"), ""),
+ ("8564_a", "host_name"),
+ ("8564_b", "access_number"),
+ ("8564_c", "compression_information"),
+ ("8564_d", "path"),
+ ("8564_f", "electronic_name"),
+ ("8564_h", "request_processor"),
+ ("8564_i", "institution"),
+ ("8564_q", "eformat"),
+ ("8564_r", "settings"),
+ ("8564_s", "file_size"),
+ ("8564_u", "url", "url"),
+ ("8564_x", "subformat"),
+ ("8564_y", "caption", "description"),
+ ("8564_z", "comment"))
+ @only_if_value((not is_local_url(value['u']), ))
+ marc, "8564_", {'host_name': value['a'],
+ 'access_number': value['b'],
+ 'compression_information': value['c'],
+ 'path':value['d'],
+ 'electronic_name': value['f'],
+ 'request_processor': value['h'],
+ 'institution': value['i'],
+ 'eformart': value['q'],
+ 'settings': value['r'],
+ 'size': value['s'],
+ 'url': value['u'],
+ 'subformat':value['x'],
+ 'description':value['y'],
+ 'comment':value['z']}
+ producer:
+ json_for_marc(), {"8564_a": "host_name", "8564_b": "access_number", "8564_c": "compression_information", "8564_d": "path", "8564_f": "electronic_name", "8564_h": "request_processor", "8564_i": "institution", "8564_q": "eformat", "8564_r": "settings", "8564_s": "file_size", "8564_u": "url", "8564_x": "subformat", "8564_y": "description", "8564_z": "comment"}
+ json_for_dc(), {"dc:identifier": "url"}
+
+_persistent_identifiers_keys:
+ calculated:
+ @parse_first(('system_control_number', 'recid', 'doi', 'oai', 'system_number', '_id'))
+ @memoize()
+ get_persistent_identifiers_keys(self.keys())
+ description:
+ """
+ This field will tell you which fields among all are considered as
+ persistent identifiers (decorated with @persistent_identifier)
+ If a new persistent identifier field is added the cached version of this
+ field must be rebuild.
+
+ Note: If a new persistent idenfier is added the list of fields to parse
+ before this one should be updated
+ """
+
+_random:
+ """Checks on the fly fields"""
+ derived:
+ random.randint(0,100)
+
+@persistent_identifier(2)
+dummy:
+ """Dummy number of authors"""
+
+@extend
+dummy:
+ derived:
+ @depends_on(('number_of_authors',))
+ self.get('nunmber_of_authors', 0)
+ checker:
+ check_field_existence(0, 1)
+ check_field_type('num')
diff --git a/invenio/legacy/bibfield/functions/get_bibdoc.py b/invenio/modules/jsonalchemy/testsuite/fields/title.cfg
similarity index 56%
rename from invenio/legacy/bibfield/functions/get_bibdoc.py
rename to invenio/modules/jsonalchemy/testsuite/fields/title.cfg
index ed6efae7e..030ad4dc9 100644
--- a/invenio/legacy/bibfield/functions/get_bibdoc.py
+++ b/invenio/modules/jsonalchemy/testsuite/fields/title.cfg
@@ -1,35 +1,37 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
+## Copyright (C) 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-def get_bibdoc(recid):
- """
- Retrieves using BibDoc all the files related with a given record
+@override
+title:
+ """Overrided Title"""
+ creator:
+ @legacy((("245", "245__","245__%"), ""),
+ ("245__a", "title", "title"),
+ ("245__b", "subtitle"),
+ ("245__k", "form"))
+ marc, "245__", { 'title':value['a'], 'subtitle':value['b'], 'form':value['k'] }
+ checker:
+ check_field_existence(0,1)
+ check_field_type('str')
- @param recid
-
- @return BibDoc of the given record
- """
- if not recid or recid < 0:
- return None
-
- from invenio.legacy.bibdocfile.api import BibDoc, InvenioBibDocFileError
- try:
- return BibDoc(int(recid))
- except InvenioBibDocFileError:
- return None
+title_parallel:
+ creator:
+ @legacy(("246_1a", "title"),
+ ("246_1i", "text"))
+ marc, "246_1", { 'title':value['a'], 'text':value['i']}
diff --git a/invenio/core/__init__.py b/invenio/modules/jsonalchemy/testsuite/models/base.cfg
similarity index 93%
copy from invenio/core/__init__.py
copy to invenio/modules/jsonalchemy/testsuite/models/base.cfg
index eb537e874..63ebe3d9c 100644
--- a/invenio/core/__init__.py
+++ b/invenio/modules/jsonalchemy/testsuite/models/base.cfg
@@ -1,18 +1,25 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+Common fields
+"""
+
+fields:
+ modification_date
diff --git a/invenio/core/__init__.py b/invenio/modules/jsonalchemy/testsuite/models/test_model.cfg
similarity index 86%
rename from invenio/core/__init__.py
rename to invenio/modules/jsonalchemy/testsuite/models/test_model.cfg
index eb537e874..35952c547 100644
--- a/invenio/core/__init__.py
+++ b/invenio/modules/jsonalchemy/testsuite/models/test_model.cfg
@@ -1,18 +1,29 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+Test model
+"""
+
+fields:
+ @inherit_from(('base', ))
+ abstract
+ authors
+ keywords
+ title_article=title
diff --git a/invenio/modules/jsonalchemy/testsuite/test_parser.py b/invenio/modules/jsonalchemy/testsuite/test_parser.py
new file mode 100644
index 000000000..942661074
--- /dev/null
+++ b/invenio/modules/jsonalchemy/testsuite/test_parser.py
@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Unit tests for the parser engine."""
+
+__revision__ = \
+ "$Id$"
+
+import os
+import tempfile
+
+from invenio.base.wrappers import lazy_import
+from invenio.ext.registry import RegistryProxy, ImportPathRegistry, \
+ PkgResourcesDiscoverRegistry
+from invenio.testsuite import make_test_suite, run_test_suite, InvenioTestCase
+
+Field_parser = lazy_import('invenio.modules.jsonalchemy.parser:FieldParser')
+Model_parser = lazy_import('invenio.modules.jsonalchemy.parser:ModelParser')
+guess_legacy_field_names = lazy_import('invenio.modules.jsonalchemy.parser:guess_legacy_field_names')
+get_producer_rules = lazy_import('invenio.modules.jsonalchemy.parser:get_producer_rules')
+
+TEST_PACKAGE = 'invenio.modules.jsonalchemy.testsuite'
+
+test_registry = RegistryProxy('testsuite', ImportPathRegistry,
+ initial=[TEST_PACKAGE])
+
+field_definitions = lambda: PkgResourcesDiscoverRegistry(
+ 'fields', registry_namespace=test_registry)
+model_definitions = lambda: PkgResourcesDiscoverRegistry(
+ 'models', registry_namespace=test_registry)
+
+
+class TestParser(InvenioTestCase):
+
+ def setUp(self):
+ self.tmp_file_1 = tempfile.NamedTemporaryFile()
+ config_1 = '''
+@inherit_from(("authors[0]",))
+main_author:
+ """Just main author"""
+ '''
+ self.tmp_file_1.write(config_1)
+ self.tmp_file_1.flush()
+ self.tmp_file_2 = tempfile.NamedTemporaryFile()
+ config_2 = '''
+include "%s"
+
+authors[0], creator:
+ creator:
+ @legacy((("100", "100__", "100__%%"), ""),
+ ("100__a", "first author name", "full_name"),
+ ("100__e", "relator_name"),
+ ("100__h", "CCID"),
+ ("100__i", "INSPIRE_number"),
+ ("100__u", "first author affiliation", "affiliation"))
+ marc, "100__", { 'full_name':value['a'], 'first_name':util_split(value['a'],',',1), 'last_name':util_split(value['a'],',',0), 'relator_name':value['e'], 'CCID':value['h'], 'INSPIRE_number':value['i'], 'affiliation':value['u'] }
+ checker:
+ check_field_existence(0,1)
+ check_field_type('str')
+ producer:
+ json_for_marc(), {"100__a": "full_name", "100__e": "relator_name", "100__h": "CCID", "100__i": "INSPIRE_number", "100__u": "affiliation"}
+ json_for_dc(), {"dc:creator": "full_name"}
+ description:
+ """Main Author"""
+
+authors[n], contributor:
+ creator:
+ @legacy((("700", "700__", "700__%%"), ""),
+ ("700__a", "additional author name", "full_name"),
+ ("700__u", "additional author affiliation", "affiliation"))
+ marc, "700__", {'full_name': value['a'], 'first_name':util_split(value['a'],',',1), 'last_name':util_split(value['a'],',',0), 'relator_name':value['e'], 'CCID':value['h'], 'INSPIRE_number':value['i'], 'affiliation':value['u'] }
+ checker:
+ check_field_existence(0,'n')
+ check_field_type('str')
+ producer:
+ json_for_marc(), {"700__a": "full_name", "700__e": "relator_name", "700__h": "CCID", "700__i": "INSPIRE_number", "700__u": "affiliation"}
+ json_for_dc(), {"dc:contributor": "full_name"}
+ description:
+ """Authors"""
+
+ ''' % (self.tmp_file_1.name, )
+ self.tmp_file_2.write(config_2)
+ self.tmp_file_2.flush()
+
+ self.app.extensions['registry']['testsuite.fields'] = field_definitions()
+ for path in self.app.extensions['registry']['testsuite.fields'].registry:
+ if os.path.basename(path) == 'authors.cfg':
+ self.app.extensions['registry']['testsuite.fields'].registry.remove(path)
+ self.app.extensions['registry']['testsuite.fields'].register(self.tmp_file_2.name)
+ self.app.extensions['registry']['testsuite.models'] = model_definitions()
+
+ def tearDown(self):
+ del self.app.extensions['registry']['testsuite.fields']
+ del self.app.extensions['registry']['testsuite.models']
+ self.tmp_file_1.close()
+ self.tmp_file_2.close()
+
+ def test_field_rules(self):
+ """JsonAlchemy - field parser"""
+ self.assertTrue(len(Field_parser.field_definitions('testsuite')) >= 22)
+ #Check that all files are parsed
+ self.assertTrue('authors' in Field_parser.field_definitions('testsuite'))
+ self.assertTrue('title' in Field_parser.field_definitions('testsuite'))
+ #Check work arroung for [n] and [0]
+ self.assertTrue(len(Field_parser.field_definitions('testsuite')['authors']) == 2)
+ self.assertEqual(Field_parser.field_definitions('testsuite')['authors'], ['authors[0]', 'authors[n]'])
+ self.assertTrue('authors[0]' in Field_parser.field_definitions('testsuite'))
+ self.assertTrue('authors[n]' in Field_parser.field_definitions('testsuite'))
+ self.assertTrue(Field_parser.field_definitions('testsuite')['doi']['persistent_identifier'])
+ #Check if derived and calulated are well parserd
+ self.assertTrue('dummy' in Field_parser.field_definitions('testsuite'))
+ self.assertEquals(Field_parser.field_definitions('testsuite')['dummy']['persistent_identifier'], 2)
+ self.assertEquals(Field_parser.field_definitions('testsuite')['dummy']['rules'].keys(), ['derived'])
+ self.assertTrue(Field_parser.field_definitions('testsuite')['_random'])
+ #Check inheritance
+ self.assertTrue('main_author' in Field_parser.field_definitions('testsuite'))
+ self.assertEqual(Field_parser.field_definitions('testsuite')['main_author']['rules'],
+ Field_parser.field_definitions('testsuite')['authors[0]']['rules'])
+ #Check override
+ value = {'a':'a', 'b':'b', 'k':'k'}
+ self.assertEquals(eval(Field_parser.field_definitions('testsuite')['title']['rules']['marc'][0]['value']),
+ {'form': 'k', 'subtitle': 'b', 'title': 'a'})
+ #Check extras
+ self.assertTrue('json_ext' in Field_parser.field_definitions('testsuite')['modification_date'])
+
+ tmp = Field_parser.field_definitions('testsuite')
+ Field_parser.reparse('testsuite')
+ self.assertEquals(len(Field_parser.field_definitions('testsuite')), len(tmp))
+
+ def test_model_definitions(self):
+ """JsonAlchemy - model parser"""
+ self.assertTrue(len(Model_parser.model_definitions('testsuite')) >= 2)
+ self.assertTrue('base' in Model_parser.model_definitions('testsuite'))
+ tmp = Model_parser.model_definitions('testsuite')
+ Model_parser.reparse('testsuite')
+ self.assertEquals(len(Model_parser.model_definitions('testsuite')), len(tmp))
+
+ def test_guess_legacy_field_names(self):
+ """JsonAlchemy - check legacy field names"""
+ self.assertEquals(guess_legacy_field_names(('100__a', '245'), 'marc', 'testsuite'),
+ {'100__a':['authors[0].full_name'], '245':['title']})
+ self.assertEquals(guess_legacy_field_names('foo', 'bar', 'baz'), {'foo': []})
+
+ def test_get_producer_rules(self):
+ """JsonAlchemy - check producer rules"""
+ self.assertTrue(get_producer_rules('authors[0]', 'json_for_marc', 'testsuite')[0] in get_producer_rules('authors', 'json_for_marc', 'testsuite'))
+ self.assertTrue(len(get_producer_rules('keywords', 'json_for_marc', 'testsuite')) == 1)
+ self.assertRaises(KeyError, lambda: get_producer_rules('foo', 'json_for_marc', 'testsuite'))
+
+
+TEST_SUITE = make_test_suite(TestParser)
+
+if __name__ == '__main__':
+ run_test_suite(TEST_SUITE)
diff --git a/invenio/modules/jsonalchemy/testsuite/test_reader.py b/invenio/modules/jsonalchemy/testsuite/test_reader.py
new file mode 100644
index 000000000..07b0c0b9c
--- /dev/null
+++ b/invenio/modules/jsonalchemy/testsuite/test_reader.py
@@ -0,0 +1,424 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""Unit tests for the parser engine."""
+
+__revision__ = \
+ "$Id$"
+
+from invenio.base.wrappers import lazy_import
+from invenio.ext.registry import RegistryProxy, ImportPathRegistry, \
+ PkgResourcesDiscoverRegistry
+from invenio.testsuite import make_test_suite, run_test_suite, InvenioTestCase
+
+Field_parser = lazy_import('invenio.modules.jsonalchemy.parser:FieldParser')
+Model_parser = lazy_import('invenio.modules.jsonalchemy.parser:ModelParser')
+guess_legacy_field_names = lazy_import('invenio.modules.jsonalchemy.parser:guess_legacy_field_names')
+get_producer_rules = lazy_import('invenio.modules.jsonalchemy.parser:get_producer_rules')
+readers = lazy_import('invenio.modules.jsonalchemy.registry:readers')
+
+TEST_PACKAGE = 'invenio.modules.jsonalchemy.testsuite'
+
+test_registry = RegistryProxy('testsuite', ImportPathRegistry,
+ initial=[TEST_PACKAGE])
+
+field_definitions = lambda: PkgResourcesDiscoverRegistry(
+ 'fields', registry_namespace=test_registry)
+model_definitions = lambda: PkgResourcesDiscoverRegistry(
+ 'models', registry_namespace=test_registry)
+
+
+class TestReaders(InvenioTestCase):
+
+ def setUp(self):
+ self.app.extensions['registry']['testsuite.fields'] = field_definitions()
+ self.app.extensions['registry']['testsuite.models'] = model_definitions()
+
+ def tearDown(self):
+ del self.app.extensions['registry']['testsuite.fields']
+ del self.app.extensions['registry']['testsuite.models']
+
+ def test_marc_reader_translate(self):
+ """JsonAlchemy - Marc reader"""
+ xml = """
+
+
+ 8
+ SzGeCERN
+
+ astro-ph/9812226
+
+
+ eng
+
+
+ Efstathiou, G P
+ Cambridge University
+
+
+ Constraints on $\Omega_{\Lambda}$ and $\Omega_{m}$from Distant Type 1a Supernovae and Cosmic Microwave Background Anisotropies
+
+
+ 14 Dec 1998
+
+
+ 6 p
+
+
+ We perform a combined likelihood analysis of the latest cosmic microwave background anisotropy data and distant Type 1a Supernova data of Perlmutter etal (1998a). Our analysis is restricted tocosmological models where structure forms from adiabatic initial fluctuations characterised by a power-law spectrum with negligible tensor component. Marginalizing over other parameters, our bestfit solution gives Omega_m = 0.25 (+0.18, -0.12) and Omega_Lambda = 0.63 (+0.17, -0.23) (95 % confidence errors) for the cosmic densities contributed by matter and a cosmological constantrespectively. The results therefore strongly favour a nearly spatially flat Universe with a non-zero cosmological constant.
+
+
+ LANL EDS
+
+
+ SzGeCERN
+ Astrophysics and Astronomy
+
+
+ Lasenby, A N
+
+
+ Hobson, M P
+
+
+ Ellis, R S
+
+
+ Bridle, S L
+
+
+ George Efstathiou <gpe@ast.cam.ac.uk>
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.pdf
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.fig1.ps.gz
+ Additional
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.fig3.ps.gz
+ Additional
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.fig5.ps.gz
+ Additional
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.fig6.ps.gz
+ Additional
+
+
+ http://invenio-software.org/download/invenio-demo-site-files/9812226.fig7.ps.gz
+ Additional
+
+
+ 1998
+
+
+ 11
+
+
+ 1998-12-14
+ 50
+ 2001-04-07
+ BATCH
+
+
+ Mon. Not. R. Astron. Soc.
+
+
+ SLAC
+ 4162242
+
+
+ CER
+
+
+ n
+ 200231
+
+
+ PREPRINT
+
+
+ Bond, J.R. 1996, Theory and Observations of the Cosmic Background Radiation, in "Cosmology and Large Scale Structure", Les Houches Session LX, August 1993, eds. R. Schaeffer, J. Silk, M. Spiro and J. Zinn-Justin, Elsevier SciencePress, Amsterdam, p469
+
+
+ Bond J.R., Efstathiou G., Tegmark M., 1997
+ L33
+ Mon. Not. R. Astron. Soc.
+ 291
+ 1997
+ Mon. Not. R. Astron. Soc. 291 (1997) L33
+
+
+ Bond, J.R., Jaffe, A. 1997, in Proc. XXXI Rencontre de Moriond, ed. F. Bouchet, Edition Fronti eres, in press
+ astro-ph/9610091
+
+
+ Bond J.R., Jaffe A.H. and Knox L.E., 1998
+ astro-ph/9808264
+ Astrophys.J. 533 (2000) 19
+
+
+ Burles S., Tytler D., 1998a, to appear in the Proceedings of the Second Oak Ridge Symposium on Atomic & Nuclear Astrophysics, ed. A. Mezzacappa, Institute of Physics, Bristol
+ astro-ph/9803071
+
+
+ Burles S., Tytler D., 1998b, Astrophys. J.in press
+ astro-ph/9712109
+ Astrophys.J. 507 (1998) 732
+
+
+ Caldwell, R.R., Dave, R., Steinhardt P.J., 1998
+ 1582
+ Phys. Rev. Lett.
+ 80
+ 1998
+ Phys. Rev. Lett. 80 (1998) 1582
+
+
+ Carroll S.M., Press W.H., Turner E.L., 1992, Ann. Rev. Astr. Astrophys., 30, 499. Chaboyer B., 1998
+ astro-ph/9808200
+ Phys.Rept. 307 (1998) 23
+
+
+ Devlin M.J., De Oliveira-Costa A., Herbig T., Miller A.D., Netterfield C.B., Page L., Tegmark M., 1998, submitted to Astrophys. J
+ astro-ph/9808043
+ Astrophys. J. 509 (1998) L69-72
+
+
+ Efstathiou G. 1996, Observations of Large-Scale Structure in the Universe, in "Cosmology and Large Scale Structure", Les Houches Session LX, August 1993, eds. R. Schaeffer, J. Silk, M. Spiro and J. Zinn-Justin, Elsevier SciencePress, Amsterdam, p135. Efstathiou G., Bond J.R., Mon. Not. R. Astron. Soc.in press
+ astro-ph/9807130
+ Astrophys. J. 518 (1999) 2-23
+
+
+ Evrard G., 1998, submitted to Mon. Not. R. Astron. Soc
+ astro-ph/9701148
+ Mon.Not.Roy.Astron.Soc. 292 (1997) 289
+
+
+ Freedman J.B., Mould J.R., Kennicutt R.C., Madore B.F., 1998
+ astro-ph/9801090
+ Astrophys. J. 480 (1997) 705
+
+
+ Garnavich P.M. et al. 1998
+ astro-ph/9806396
+ Astrophys.J. 509 (1998) 74-79
+
+
+ Goobar A., Perlmutter S., 1995
+ 14
+ Astrophys. J.
+ 450
+ 1995
+ Astrophys. J. 450 (1995) 14
+
+
+ Hamuy M., Phillips M.M., Maza J., Suntzeff N.B., Schommer R.A., Aviles R. 1996
+ 2391
+ Astrophys. J.
+ 112
+ 1996
+ Astrophys. J. 112 (1996) 2391
+
+
+ Hancock S., Gutierrez C.M., Davies R.D., Lasenby A.N., Rocha G., Rebolo R., Watson R.A., Tegmark M., 1997
+ 505
+ Mon. Not. R. Astron. Soc.
+ 298
+ 1997
+ Mon. Not. R. Astron. Soc. 298 (1997) 505
+
+
+ Hancock S., Rocha G., Lasenby A.N., Gutierrez C.M., 1998
+ L1
+ Mon. Not. R. Astron. Soc.
+ 294
+ 1998
+ Mon. Not. R. Astron. Soc. 294 (1998) L1
+
+
+ Herbig T., De Oliveira-Costa A., Devlin M.J., Miller A.D., Page L., Tegmark M., 1998, submitted to Astrophys. J
+ astro-ph/9808044
+ Astrophys.J. 509 (1998) L73-76
+
+
+ Lineweaver C.H., 1998. Astrophys. J.505, L69. Lineweaver, C.H., Barbosa D., 1998a
+ 624
+ Astrophys. J.
+ 446
+ 1998
+ Astrophys. J. 446 (1998) 624
+
+
+ Lineweaver, C.H., Barbosa D., 1998b
+ 799
+ Astron. Astrophys.
+ 329
+ 1998
+ Astron. Astrophys. 329 (1998) 799
+
+
+ De Oliveira-Costa A., Devlin M.J., Herbig T., Miller A.D., Netterfield C.B. Page L., Tegmark M., 1998, submitted to Astrophys. J
+ astro-ph/9808045
+ Astrophys. J. 509 (1998) L77-80
+
+
+ Ostriker J.P., Steinhardt P.J., 1995
+ 600
+ Nature
+ 377
+ 1995
+ Nature 377 (1995) 600
+
+
+ Peebles P.J.E., 1993, Principles of Physical Cosmology, Princeton University Press, Princeton, New Jersey. Perlmutter S, et al., 1995, In Presentations at the NATO ASI in Aiguablava, Spain, LBL-38400; also published in Thermonuclear Supernova, P. Ruiz-Lapuente, R. Cana and J. Isern (eds), Dordrecht, Kluwer, 1997, p749. Perlmutter S, et al., 1997
+ 565
+ Astrophys. J.
+ 483
+ 1997
+ Astrophys. J. 483 (1997) 565
+
+
+ Perlmutter S. et al., 1998a, Astrophys. J.in press. (P98)
+ astro-ph/9812133
+ Astrophys. J. 517 (1999) 565-586
+
+
+ Perlmutter S. et al., 1998b, In Presentation at the January 1988 Meeting of the American Astronomical Society, Washington D.C., LBL-42230, available at www-supernova.lbl.gov; B.A.A.S., volume : 29 (1997) 1351Perlmutter S, et al., 1998c
+ 51
+ Nature
+ 391
+ 1998
+ Nature 391 (1998) 51
+
+
+ Ratra B., Peebles P.J.E., 1988
+ 3406
+ Phys. Rev., D
+ 37
+ 1988
+ Phys. Rev. D 37 (1988) 3406
+
+
+ Riess A. et al. 1998, Astrophys. J.in press
+ astro-ph/9805201
+ Astron. J. 116 (1998) 1009-1038
+
+
+ Seljak U., Zaldarriaga M. 1996
+ 437
+ Astrophys. J.
+ 469
+ 1996
+ Astrophys. J. 469 (1996) 437
+
+
+ Seljak U. & Zaldarriaga M., 1998
+ astro-ph/9811123
+ Phys. Rev. D60 (1999) 043504
+
+
+ Tegmark M., 1997
+ 3806
+ Phys. Rev. Lett.
+ 79
+ 1997
+ Phys. Rev. Lett. 79 (1997) 3806
+
+
+ Tegmark M. 1998, submitted to Astrophys. J
+ astro-ph/9809201
+ Astrophys. J. 514 (1999) L69-L72
+
+
+ Tegmark, M., Eisenstein D.J., Hu W., Kron R.G., 1998
+ astro-ph/9805117
+
+
+ Wambsganss J., Cen R., Ostriker J.P., 1998
+ 29
+ Astrophys. J.
+ 494
+ 1998
+ Astrophys. J. 494 (1998) 29
+
+
+ Webster M., Bridle S.L., Hobson M.P., Lasenby A.N., Lahav O., Rocha, G., 1998, Astrophys. J.in press
+ astro-ph/9802109
+
+
+ White M., 1998, Astrophys. J.in press
+ astro-ph/9802295
+ Astrophys. J. 506 (1998) 495
+
+
+ Zaldarriaga, M., Spergel D.N., Seljak U., 1997
+ 1
+ Astrophys. J.
+ 488
+ 1997
+ Astrophys. J. 488 (1997) 1
+
+
+
+ 33
+
+ eng
+
+
+
+ """
+ blob = list(readers['marc'].split_blob(xml, schema='foo'))
+ self.assertTrue(len(blob) == 0)
+ blob = list(readers['marc'].split_blob(xml))[0]
+ reader = readers['marc'](blob=blob, namespace='testsuite')
+ json = reader.translate()
+ self.assertIsNotNone(json)
+ self.assertTrue('__meta_metadata__' in json)
+ self.assertTrue(json['__meta_metadata__']['__additional_info__']['master_format'] == 'marc')
+ self.assertTrue('authors' in json)
+ self.assertTrue(json['authors'][0]['full_name'] == "Efstathiou, G P")
+ self.assertTrue(len(json['authors']) == 5)
+ self.assertTrue('title' in json)
+ self.assertTrue(json['title']['title'] == "Constraints on $\Omega_{\Lambda}$ and $\Omega_{m}$from Distant Type 1a Supernovae and Cosmic Microwave Background Anisotropies")
+ self.assertTrue('abstract' in json)
+ self.assertTrue(json['abstract']['summary'] == "We perform a combined likelihood analysis of the latest cosmic microwave background anisotropy data and distant Type 1a Supernova data of Perlmutter etal (1998a). Our analysis is restricted tocosmological models where structure forms from adiabatic initial fluctuations characterised by a power-law spectrum with negligible tensor component. Marginalizing over other parameters, our bestfit solution gives Omega_m = 0.25 (+0.18, -0.12) and Omega_Lambda = 0.63 (+0.17, -0.23) (95 % confidence errors) for the cosmic densities contributed by matter and a cosmological constantrespectively. The results therefore strongly favour a nearly spatially flat Universe with a non-zero cosmological constant.")
+ self.assertTrue('reference' in json)
+ self.assertTrue(len(json['reference']) == 36)
+
+ reader = readers['marc'](blob=blob, model='test_model', namespace='testsuite')
+ json = reader.translate()
+ self.assertTrue(json['__meta_metadata__']['__additional_info__']['model'] == 'test_model')
+ self.assertTrue(json['__meta_metadata__']['__additional_info__']['namespace'] == 'testsuite')
+ self.assertTrue('title_article' in json)
+
+
+ def test_json_reader(self):
+ """JsonAlchemy - Json reader"""
+ pass
+
+
+TEST_SUITE = make_test_suite(TestReaders)
+
+if __name__ == '__main__':
+ run_test_suite(TEST_SUITE)
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/testsuite/test_storage_engine.py
similarity index 100%
copy from invenio/legacy/bibfield/functions/__init__.py
copy to invenio/modules/jsonalchemy/testsuite/test_storage_engine.py
diff --git a/invenio/legacy/bibfield/functions/__init__.py b/invenio/modules/jsonalchemy/testsuite/test_wrapper.py
similarity index 100%
rename from invenio/legacy/bibfield/functions/__init__.py
rename to invenio/modules/jsonalchemy/testsuite/test_wrapper.py
diff --git a/invenio/modules/jsonalchemy/validator.py b/invenio/modules/jsonalchemy/validator.py
new file mode 100644
index 000000000..1c04402ad
--- /dev/null
+++ b/invenio/modules/jsonalchemy/validator.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.validator
+ -------------------------------------
+
+ See http://cerberus.readthedocs.org/en/latest
+"""
+
+from cerberus import Validator as ValidatorBase
+from cerberus import ValidationError, SchemaError
+from cerberus import errors
+
+from .reader import FieldParser, ModelParser
+
+
+class Validator(ValidatorBase):
+ """
+
+ """
+
+ def __init__(self, schema=None, transparent_schema_rules=True,
+ ignore_none_values=False, allow_unknown=True):
+ super(Validator, self).__init__(schema, transparent_schema_rules,
+ ignore_none_values, allow_unknown)
+
+ def _validate(self, document, schema=None, update=False):
+ self._errors = {}
+ self.update = update
+
+ if schema is not None:
+ self.schema = schema
+ elif self.schema is None:
+ raise SchemaError(errors.ERROR_SCHEMA_MISSING)
+ if not isinstance(self.schema, dict):
+ raise SchemaError(errors.ERROR_SCHEMA_FORMAT % str(self.schema))
+
+ if document is None:
+ raise ValidationError(errors.ERROR_DOCUMENT_MISSING)
+ if not hasattr(document, 'get'):
+ raise ValidationError(errors.ERROR_DOCUMENT_FORMAT % str(document))
+ self.document = document
+
+ special_rules = ["required", "nullable", "type"]
+ for field, value in self.document.items():
+
+ if self.ignore_none_values and value is None:
+ continue
+
+ definition = self.schema.get(field)
+ if definition:
+ if isinstance(definition, dict):
+
+ if definition.get("nullable", False) == True \
+ and value is None: # noqa
+ continue
+
+ if 'type' in definition:
+ self._validate_type(definition['type'], field, value)
+ if self.errors:
+ continue
+
+ definition_rules = [rule for rule in definition.keys()
+ if rule not in special_rules]
+ for rule in definition_rules:
+ validatorname = "_validate_" + rule.replace(" ", "_")
+ validator = getattr(self, validatorname, None)
+ if validator:
+ validator(definition[rule], field, value)
+ elif not self.transparent_schema_rules:
+ raise SchemaError(errors.ERROR_UNKNOWN_RULE %
+ (rule, field))
+ else:
+ raise SchemaError(errors.ERROR_DEFINITION_FORMAT % field)
+
+ else:
+ if not self.allow_unknown:
+ self._error(field, errors.ERROR_UNKNOWN_FIELD)
+
+ if not self.update:
+ self._validate_required_fields()
+
+ return len(self._errors) == 0
+
+ def _validate_type_objectid(self, field, value):
+ """
+ Enables validation for `objectid` schema attribute.
+
+ :param field: field name.
+ :param value: field value.
+ """
+ if not re.match('[a-f0-9]{24}', value):
+ self._error(field, ERROR_BAD_TYPE % 'ObjectId')
diff --git a/invenio/modules/jsonalchemy/wrappers.py b/invenio/modules/jsonalchemy/wrappers.py
new file mode 100644
index 000000000..c16b3cc60
--- /dev/null
+++ b/invenio/modules/jsonalchemy/wrappers.py
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.jsonalchemy.wrapper
+ -----------------------------------
+"""
+import datetime
+import six
+
+from werkzeug.utils import import_string
+
+from invenio.base.utils import try_to_eval
+from invenio.utils.datastructures import SmartDict
+
+from .parser import FieldParser, ModelParser
+from .registry import functions, readers, producers
+from .validator import Validator
+
+
+class SmartJson(SmartDict):
+ """Base class for Json structures"""
+
+ def __init__(self, json):
+ super(SmartJson, self).__init__(json)
+ self._dict_bson = SmartDict()
+ self._validator = None
+
+ if '__meta_metadata__.__additional_info__.model_meta_classes' in self:
+ meta_classes = [import_string(str_cls)
+ for str_cls in self['__meta_metadata__.__additional_info__.model_meta_classes']]
+ self.__class__ = type(self.__class__.__name__,
+ [self.__class__] + meta_classes, {})
+
+ def __getitem__(self, key):
+ """
+ Uses the load capabilities to output the information stored in the DB.
+ """
+ try:
+ return self._dict_bson[key]
+ except KeyError:
+ #We will try to find the key inside the json dict and load it
+ pass
+
+ main_key = SmartDict.main_key_pattern.sub('', key)
+ if main_key in self._dict['__meta_metadata__']['__aliases__']:
+ try:
+ rest_of_key = SmartDict.main_key_pattern.findall(key)[0]
+ except IndexError:
+ rest_of_key = ''
+ return self[self._dict['__meta_metadata__']['__aliases__'][main_key] + rest_of_key]
+ try:
+ if self._dict['__meta_metadata__'][main_key]['type'] == 'calculated':
+ self._load_precalculated_value(main_key)
+ else:
+ self._loads(main_key)
+ except KeyError:
+ self._loads(main_key)
+
+ return self._dict_bson[key]
+
+ def __setitem__(self, key, value):
+ """
+ Uses the dumps capabilities to set the items to store them in the DB
+ """
+ main_key = SmartDict.main_key_pattern.sub('', key)
+ if main_key in self:
+ self._dict_bson[key] = value
+ else:
+ reader = readers[self['__meta_metadata__']['__additional_info__']['master_format']]()
+ reader.set(self, main_key)
+ self._dict_bson[key] = value
+
+ self._dumps(main_key)
+
+ def items(self):
+ for key in self.keys():
+ yield (key, self[key])
+
+ @property
+ def validation_errors(self):
+ if self._validator is None:
+ self.validate()
+ return self._validator.errors
+
+ def dumps(self):
+ """ """
+ for key in self._dict_bson.keys():
+ if key == '__meta_metadata__':
+ continue
+ self._dumps(key)
+ return self._dict
+
+ def loads(self):
+ """ """
+ for key in self._dict.keys():
+ if key == '__meta_metadata__':
+ continue
+ self._loads(key)
+ return self._dict_bson._dict
+
+ def produce(self, output, fields=None):
+ return producers[output](self, fields=fields)
+
+ def validate(self):
+
+ def find_schema(json_id):
+ schema = FieldParser.field_definitions(self['__meta_metadata__']['__additional_info__']['namespace']).get(json_id, {})
+ if isinstance(schema, list):
+ for jjson_id in schema:
+ yield FieldParser.field_definitions(self['__meta_metadata__']['__additional_info__']['namespace']).get(jjson_id, {}).get('schema', {})
+ raise StopIteration()
+ yield schema.get('schema', {})
+
+ if self._validator is None:
+ schema = {}
+ model_fields = ModelParser.model_definitions(self['__meta_metadata__']['__additional_info__']['namespace']).get(fields, {})
+ if model_fields:
+ for field in self.document.keys():
+ if field not in model_fields:
+ model_fields[field] = field
+ model_field = [json_id for json_id in model_fields.values()]
+ else:
+ model_fields = self.document.keys()
+
+ for json_id in model_fields:
+ for schema in find_schema(json_id):
+ self.schema.update(schema)
+ self._validator = Validator(schema=shema)
+
+ return self._validator.validate(self)
+
+ def _dumps(self, field):
+ """ """
+ try:
+ self._dict[field] = reduce(lambda obj, key: obj[key], \
+ self._dict['__meta_metadata__'][field]['dumps'], \
+ FieldParser.field_definitions(self['__meta_metadata__']['__additional_info__']['namespace']))(self._dict_bson[field])
+ except (KeyError, IndexError):
+ if self['__meta_metadata__'][field]['memoize'] or \
+ self['__meta_metadata__'][field]['type'] in ('derived', 'creator', 'UNKNOW'):
+ self._dict[field] = self._dict_bson[field]
+
+ def _loads(self, field):
+ """ """
+ try:
+ self._dict_bson[field] = reduce(lambda obj, key: obj[key], \
+ self._dict['__meta_metadata__'][field]['loads'], \
+ FieldParser.field_definition(self['__meta_metadata__']['__additional_info__']['namespace']))(self._dict[field])
+ except (KeyError, IndexError):
+ self._dict_bson[field] = self._dict[field]
+
+ def _load_precalculated_value(self, field):
+ """
+
+ """
+ if self._dict['__meta_metadata__'][field]['memoize'] is None:
+ func = reduce(lambda obj, key: obj[key], \
+ self._dict['__meta_metadata__'][field]['function'], \
+ FieldParser.field_definitions(self['__meta_metadata__']['__additional_info__']['namespace']))
+ self._dict_bson[field] = try_to_eval(func, functions, self=self)
+ else:
+ live_time = datetime.timedelta(0, self._dict['__meta_metadata__'][field]['memoize'])
+ timestamp = datetime.datetime.strptime(self._dict['__meta_metadata__'][field]['timestamp'], "%Y-%m-%dT%H:%M:%S")
+ if datetime.datetime.now() > timestamp + live_time:
+ old_value = self._dict_bson[field]
+ func = reduce(lambda obj, key: obj[key], \
+ self._dict['__meta_metadata__'][field]['function'], \
+ FieldParser.field_definitions(self['__meta_metadata__']['__additional_info__']['namespace']))
+ self._dict_bson[field] = try_to_eval(func, functions, self=self)
+ if not old_value == self._dict_bson[field]:
+ #FIXME: trigger update in DB and fire signal to update others
+ pass
+
+ # Legacy methods, try not to use them as they are already deprecated
+
+ def legacy_export_as_marc(self):
+ """
+ It creates a valid marcxml using the legacy rules defined in the config
+ file
+ """
+ from collections import Iterable
+ def encode_for_marcxml(value):
+ from invenio.utils.text import encode_for_xml
+ if isinstance(value, unicode):
+ value = value.encode('utf8')
+ return encode_for_xml(str(value))
+
+ export = ''
+ marc_dicts = self.produce('json_for_marc')
+ for marc_dict in marc_dicts:
+ content = ''
+ tag = ''
+ ind1 = ''
+ ind2 = ''
+ for key, value in marc_dict.iteritems():
+ if isinstance(value, six.string_types) or not isinstance(value, Iterable):
+ value = [value]
+ for v in value:
+ if v is None:
+ continue
+ if key.startswith('00') and len(key) == 3:
+ # Control Field (No indicators no subfields)
+ export += '%s\n' % (key, encode_for_marcxml(v))
+ elif len(key) == 6:
+ if not (tag == key[:3] and ind1 == key[3].replace('_', '') and ind2 == key[4].replace('_', '')):
+ tag = key[:3]
+ ind1 = key[3].replace('_', '')
+ ind2 = key[4].replace('_', '')
+ if content:
+ export += '%s\n' % (tag, ind1, ind2, content)
+ content = ''
+ content += '%s' % (key[5], encode_for_marcxml(v))
+ else:
+ pass
+
+ if content:
+ export += '%s\n' % (tag, ind1, ind2, content)
+
+ export += ''
+ return export
+
+ def legacy_create_recstruct(self):
+ """
+ It creates the recstruct representation using the legacy rules defined in
+ the configuration file
+
+ #CHECK: it might be a bit overkilling
+ """
+ from invenio.legacy.bibrecord import create_record
+ return create_record(self.legacy_export_as_marc())[0]
+
diff --git a/invenio/modules/linkbacks/models.py b/invenio/modules/linkbacks/models.py
index 7dc1dd25c..38f5672d2 100644
--- a/invenio/modules/linkbacks/models.py
+++ b/invenio/modules/linkbacks/models.py
@@ -1,118 +1,118 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
WebLinkBack database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
# Create your models here.
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.accounts.models import User
class LnkADMINURL(db.Model):
"""Represents a LnkADMINURL record."""
__tablename__ = 'lnkADMINURL'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
nullable=False)
url = db.Column(db.String(100), nullable=False, unique=True)
list = db.Column(db.String(30), nullable=False, index=True)
class LnkENTRY(db.Model):
"""Represents a LnkENTRY record."""
__tablename__ = 'lnkENTRY'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True, nullable=False)
origin_url = db.Column(db.String(100), nullable=False)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=False)
additional_properties = db.Column(db.Binary)
type = db.Column(db.String(30), nullable=False, index=True)
status = db.Column(db.String(30), nullable=False, server_default='PENDING',
index=True)
insert_time = db.Column(db.DateTime, server_default='1900-01-01 00:00:00',
index=True)
@property
def title(self):
try:
return db.object_session(self).query(LnkENTRYURLTITLE).\
filter(db.and_(
LnkENTRYURLTITLE.url==self.origin_url,
LnkENTRYURLTITLE.title<>"",
LnkENTRYURLTITLE.broken==0)).first().title
except:
return self.origin_url
class LnkENTRYURLTITLE(db.Model):
"""Represents a LnkENTRYURLTITLE record."""
__tablename__ = 'lnkENTRYURLTITLE'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
nullable=False)
url = db.Column(db.String(100), nullable=False, unique=True)
title = db.Column(db.String(100), nullable=False, index=True)
manual_set = db.Column(db.TinyInteger(1), nullable=False,
server_default='0')
broken_count = db.Column(db.Integer(5), server_default='0')
broken = db.Column(db.TinyInteger(1), nullable=False, server_default='0')
class LnkLOG(db.Model):
"""Represents a LnkLOG record."""
__tablename__ = 'lnkLOG'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True, nullable=False)
id_user = db.Column(db.Integer(15, unsigned=True), db.ForeignKey(User.id))
action = db.Column(db.String(30), nullable=False, index=True)
log_time = db.Column(db.DateTime, server_default='1900-01-01 00:00:00',
index=True)
class LnkENTRYLOG(db.Model):
"""Represents a LnkENTRYLOG record."""
__tablename__ = 'lnkENTRYLOG'
id_lnkENTRY = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(LnkENTRY.id), nullable=False, primary_key=True)
id_lnkLOG = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(LnkLOG.id), nullable=False, primary_key=True)
class LnkADMINURLLOG(db.Model):
"""Represents a LnkADMINURLLOG record."""
__tablename__ = 'lnkADMINURLLOG'
id_lnkADMINURL = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(LnkADMINURL.id), primary_key=True, nullable=False)
id_lnkLOG = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(LnkLOG.id), primary_key=True, nullable=False)
__all__ = [ 'LnkADMINURL',
'LnkADMINURLLOG',
'LnkENTRY',
'LnkENTRYLOG',
'LnkENTRYURLTITLE',
'LnkLOG']
diff --git a/invenio/modules/oai_harvest/models.py b/invenio/modules/oai_harvest/models.py
index 91d3de8ee..ba6af2c05 100644
--- a/invenio/modules/oai_harvest/models.py
+++ b/invenio/modules/oai_harvest/models.py
@@ -1,119 +1,119 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
Oai harvest database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
# Create your models here.
from invenio.utils.serializers import deserialize_via_marshal
#from websearch_model import Collection
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.scheduler.models import SchTASK
class OaiHARVEST(db.Model):
"""Represents a OaiHARVEST record."""
__tablename__ = 'oaiHARVEST'
id = db.Column(db.MediumInteger(9, unsigned=True), nullable=False,
primary_key=True, autoincrement=True)
baseurl = db.Column(db.String(255), nullable=False, server_default='')
metadataprefix = db.Column(db.String(255), nullable=False,
server_default='oai_dc')
arguments = db.Column(db.LargeBinary, nullable=True)
comment = db.Column(db.Text, nullable=True)
name = db.Column(db.String(255), nullable=False)
lastrun = db.Column(db.DateTime, nullable=True)
frequency = db.Column(db.MediumInteger(12), nullable=False,
server_default='0')
postprocess = db.Column(db.String(20), nullable=False,
server_default='h')
setspecs = db.Column(db.Text, nullable=False)
def get_arguments(self):
return deserialize_via_marshal(self.arguments)
@classmethod
def get(cls, *criteria, **filters):
""" A wrapper for the filter and filter_by functions of sqlalchemy.
Define a dict with which columns should be filtered by which values.
look up also sqalchemy BaseQuery's filter and filter_by documentation
"""
return cls.query.filter(*criteria).filter_by(**filters)
class OaiREPOSITORY(db.Model):
"""Represents a OaiREPOSITORY record."""
__tablename__ = 'oaiREPOSITORY'
id = db.Column(db.MediumInteger(9, unsigned=True), nullable=False,
primary_key=True, autoincrement=True)
setName = db.Column(db.String(255), nullable=False,
server_default='')
setSpec = db.Column(db.String(255), nullable=False,
server_default='')
setCollection = db.Column(db.String(255), nullable=False,
server_default='')
setDescription = db.Column(db.Text, nullable=False)
setDefinition = db.Column(db.Text, nullable=False)
setRecList = db.Column(db.iLargeBinary, nullable=True)
last_updated = db.Column(db.DateTime, nullable=False,
server_default='1970-01-01 00:00:00')
p1 = db.Column(db.Text, nullable=False)
f1 = db.Column(db.Text, nullable=False)
m1 = db.Column(db.Text, nullable=False)
p2 = db.Column(db.Text, nullable=False)
f2 = db.Column(db.Text, nullable=False)
m2 = db.Column(db.Text, nullable=False)
p3 = db.Column(db.Text, nullable=False)
f3 = db.Column(db.Text, nullable=False)
m3 = db.Column(db.Text, nullable=False)
class OaiHARVESTLOG(db.Model):
"""Represents a OaiHARVESTLOG record."""
__tablename__ = 'oaiHARVESTLOG'
id_oaiHARVEST = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(OaiHARVEST.id), nullable=False)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=False, server_default='0')
bibupload_task_id = db.Column(db.Integer(11), db.ForeignKey(SchTASK.id),
nullable=False, server_default='0',
primary_key=True)
oai_id = db.Column(db.String(40), nullable=False, server_default='',
primary_key=True)
date_harvested = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00',
primary_key=True)
date_inserted = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
inserted_to_db = db.Column(db.Char(1), nullable=False,
server_default='P')
bibrec = db.relationship(Bibrec, backref='harvestlogs')
schtask = db.relationship(SchTASK)
__all__ = ['OaiHARVEST',
'OaiREPOSITORY',
'OaiHARVESTLOG']
diff --git a/invenio/modules/ranker/models.py b/invenio/modules/ranker/models.py
index ec32ede61..31f7e7efb 100644
--- a/invenio/modules/ranker/models.py
+++ b/invenio/modules/ranker/models.py
@@ -1,221 +1,222 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
BibRank database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
# Create your models here.
from invenio.modules.accounts.models import User
-from invenio.modules.record_editor.models import Bibrec, Bibdoc
+from invenio.modules.record_editor.models import Bibdoc
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.search.models import Collection
from flask import g
class RnkMETHOD(db.Model):
"""Represents a RnkMETHOD record."""
__tablename__ = 'rnkMETHOD'
id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True,
nullable=False)
name = db.Column(db.String(20), unique=True, nullable=False,
server_default='')
last_updated = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
def get_name_ln(self, ln=None):
"""Returns localized method name."""
try:
if ln is None:
ln = g.ln
return self.names.filter_by(ln=g.ln, type='ln').one().value
except:
return self.name
class RnkMETHODDATA(db.Model):
"""Represents a RnkMETHODDATA record."""
__tablename__ = 'rnkMETHODDATA'
id_rnkMETHOD = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(RnkMETHOD.id), primary_key=True)
relevance_data = db.Column(db.iLargeBinary, nullable=True)
class RnkMETHODNAME(db.Model):
"""Represents a RnkMETHODNAME record."""
__tablename__ = 'rnkMETHODNAME'
id_rnkMETHOD = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(RnkMETHOD.id), primary_key=True)
ln = db.Column(db.Char(5), primary_key=True, server_default='')
type = db.Column(db.Char(3), primary_key=True, server_default='sn')
value = db.Column(db.String(255), nullable=False)
method = db.relationship(RnkMETHOD, backref=db.backref('names',
lazy='dynamic'))
class RnkCITATIONDATA(db.Model):
"""Represents a RnkCITATIONDATA record."""
__tablename__ = 'rnkCITATIONDATA'
id = db.Column(db.MediumInteger(8, unsigned=True), primary_key=True)
object_name = db.Column(db.String(20), unique=True, nullable=False)
object_value = db.Column(db.iLargeBinary, nullable=True)
last_updated = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
class RnkCITATIONDATAERR(db.Model):
"""Represents a RnkCITATIONDATAERR record."""
__tablename__ = 'rnkCITATIONDATAERR'
type = db.Column(db.Enum('multiple-matches', 'not-well-formed'),
primary_key=True)
citinfo = db.Column(db.String(255), primary_key=True, server_default='')
class RnkCITATIONDATAEXT(db.Model):
"""Represents a RnkCITATIONDATAEXT record."""
__tablename__ = 'rnkCITATIONDATAEXT'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), autoincrement=False,
primary_key=True, nullable=False, server_default='0')
extcitepubinfo = db.Column(db.String(255), primary_key=True,
nullable=False, index=True)
class RnkAUTHORDATA(db.Model):
"""Represents a RnkAUTHORDATA record."""
__tablename__ = 'rnkAUTHORDATA'
aterm = db.Column(db.String(50), primary_key=True, nullable=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class RnkDOWNLOADS(db.Model):
"""Represents a RnkDOWNLOADS record."""
__tablename__ = 'rnkDOWNLOADS'
id = db.Column(db.Integer, primary_key=True, nullable=False,
autoincrement=True)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=True)
download_time = db.Column(db.DateTime, nullable=True,
server_default='1900-01-01 00:00:00')
client_host = db.Column(db.Integer(10, unsigned=True), nullable=True)
id_user = db.Column(db.Integer(15, unsigned=True), db.ForeignKey(User.id),
nullable=True)
id_bibdoc = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Bibdoc.id), nullable=True)
file_version = db.Column(db.SmallInteger(2, unsigned=True), nullable=True)
file_format = db.Column(db.String(50), nullable=True)
bibrec = db.relationship(Bibrec, backref='downloads')
bibdoc = db.relationship(Bibdoc, backref='downloads')
user = db.relationship(User, backref='downloads')
class RnkPAGEVIEWS(db.Model):
"""Represents a RnkPAGEVIEWS record."""
__tablename__ = 'rnkPAGEVIEWS'
id = db.Column(db.MediumInteger, primary_key=True, nullable=False,
autoincrement=True)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=True, primary_key=True)
id_user = db.Column(db.Integer(15, unsigned=True), db.ForeignKey(User.id),
server_default='0', primary_key=True)
client_host = db.Column(db.Integer(10, unsigned=True), nullable=True)
view_time = db.Column(db.DateTime, primary_key=True,
server_default='1900-01-01 00:00:00')
bibrec = db.relationship(Bibrec, backref='pageviews')
user = db.relationship(User, backref='pageviews')
class RnkWORD01F(db.Model):
"""Represents a RnkWORD01F record."""
__tablename__ = 'rnkWORD01F'
id = db.Column(db.MediumInteger(9, unsigned=True), nullable=False,
primary_key=True, autoincrement=True)
term = db.Column(db.String(50), nullable=True, unique=True)
hitlist = db.Column(db.iLargeBinary, nullable=True)
class RnkWORD01R(db.Model):
"""Represents a RnkWORD01R record."""
__tablename__ = 'rnkWORD01R'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=False, primary_key=True)
termlist = db.Column(db.LargeBinary, nullable=True)
type = db.Column(db.Enum('CURRENT', 'FUTURE', 'TEMPORARY'),
nullable=False, server_default='CURRENT', primary_key=True)
bibrec = db.relationship(Bibrec, backref='word01rs')
class RnkEXTENDEDAUTHORS(db.Model):
"""Represents a RnkEXTENDEDAUTHORS record."""
__tablename__ = 'rnkEXTENDEDAUTHORS'
id = db.Column(db.Integer(10, unsigned=True), primary_key=True,
nullable=False, autoincrement=False)
authorid = db.Column(db.BigInteger(10), primary_key=True, nullable=False,
autoincrement=False)
class RnkRECORDSCACHE(db.Model):
"""Represents a RnkRECORDSCACHE record."""
__tablename__ = 'rnkRECORDSCACHE'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=True, primary_key=True)
authorid = db.Column(db.BigInteger(10), primary_key=True, nullable=False)
class RnkSELFCITES(db.Model):
"""Represents a RnkSELFCITES record."""
__tablename__ = 'rnkSELFCITES'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=True, primary_key=True)
count = db.Column(db.Integer(10, unsigned=True), nullable=False)
references = db.Column(db.Text, nullable=False)
last_updated = db.Column(db.DateTime, nullable=False)
class CollectionRnkMETHOD(db.Model):
"""Represents a CollectionRnkMETHOD record."""
__tablename__ = 'collection_rnkMETHOD'
id_collection = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Collection.id), primary_key=True, nullable=False)
id_rnkMETHOD = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(RnkMETHOD.id), primary_key=True, nullable=False)
score = db.Column(db.TinyInteger(4, unsigned=True), nullable=False,
server_default='0')
collection = db.relationship(Collection, backref='rnkMETHODs')
rnkMETHOD = db.relationship(RnkMETHOD, backref='collections')
__all__ = ['RnkMETHOD',
'RnkMETHODDATA',
'RnkMETHODNAME',
'RnkCITATIONDATA',
'RnkCITATIONDATAERR',
'RnkCITATIONDATAEXT',
'RnkAUTHORDATA',
'RnkDOWNLOADS',
'RnkPAGEVIEWS',
'RnkWORD01F',
'RnkWORD01R',
'RnkEXTENDEDAUTHORS',
'RnkRECORDSCACHE',
'RnkSELFCITES',
'CollectionRnkMETHOD']
diff --git a/invenio/modules/record_editor/models.py b/invenio/modules/record_editor/models.py
index cd93e2fcf..aaf59be5b 100644
--- a/invenio/modules/record_editor/models.py
+++ b/invenio/modules/record_editor/models.py
@@ -1,3549 +1,3459 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
BibEdit database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
from flask import current_app
from werkzeug import cached_property
-# Create your models here.
-
+from invenio.modules.records.models import Record as Bibrec
-class Bibrec(db.Model):
- """Represents a Bibrec record."""
- def __init__(self):
- pass
- __tablename__ = 'bibrec'
- id = db.Column(db.MediumInteger(8, unsigned=True), primary_key=True,
- nullable=False,
- autoincrement=True)
- creation_date = db.Column(db.DateTime, nullable=False,
- server_default='1900-01-01 00:00:00',
- index=True)
- modification_date = db.Column(db.DateTime, nullable=False,
- server_default='1900-01-01 00:00:00',
- index=True)
- master_format = db.Column(db.String(16), nullable=False,
- server_default='marc')
-
- @property
- def deleted(self):
- """
- Return True if record is marked as deleted.
- """
- from invenio.legacy.bibrecord import get_fieldvalues
- # record exists; now check whether it isn't marked as deleted:
- dbcollids = get_fieldvalues(self.id, "980__%")
-
- return ("DELETED" in dbcollids) or \
- (current_app.config.get('CFG_CERN_SITE')
- and "DUMMY" in dbcollids)
-
- @staticmethod
- def _next_merged_recid(recid):
- """ Returns the ID of record merged with record with ID = recid """
- from invenio.legacy.bibrecord import get_fieldvalues
- merged_recid = None
- for val in get_fieldvalues(recid, "970__d"):
- try:
- merged_recid = int(val)
- break
- except ValueError:
- pass
-
- if not merged_recid:
- return None
- else:
- return merged_recid
-
- @cached_property
- def merged_recid(self):
- """ Return the record object with
- which the given record has been merged.
- @param recID: deleted record recID
- @type recID: int
- @return: merged record recID
- @rtype: int or None
- """
- return Bibrec._next_merged_recid(self.id)
-
- @property
- def merged_recid_final(self):
- """ Returns the last record from hierarchy of
- records merged with this one """
-
- cur_id = self.id
- next_id = Bibrec._next_merged_recid(cur_id)
-
- while next_id:
- cur_id = next_id
- next_id = Bibrec._next_merged_recid(cur_id)
-
- return cur_id
-
- @cached_property
- def is_restricted(self):
- """Returns True is record is restricted."""
- from invenio.legacy.search_engine import get_restricted_collections_for_recid
-
- if get_restricted_collections_for_recid(self.id,
- recreate_cache_if_needed=False):
- return True
- elif self.is_processed:
- return True
- return False
-
- @cached_property
- def is_processed(self):
- """Returns True is recods is processed (not in any collection)."""
- from invenio.legacy.search_engine import is_record_in_any_collection
- return not is_record_in_any_collection(self.id,
- recreate_cache_if_needed=False)
+# Create your models here.
class BibHOLDINGPEN(db.Model):
"""Represents a BibHOLDINGPEN record."""
def __init__(self):
pass
__tablename__ = 'bibHOLDINGPEN'
changeset_id = db.Column(db.Integer(11), primary_key=True,
autoincrement=True)
changeset_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00', index=True)
changeset_xml = db.Column(db.Text, nullable=False)
oai_id = db.Column(db.String(40), nullable=False,
server_default='')
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=False, server_default='0')
bibrec = db.relationship(Bibrec, backref='holdingpen')
class Bibdoc(db.Model):
"""Represents a Bibdoc record."""
__tablename__ = 'bibdoc'
id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True,
nullable=False, autoincrement=True)
status = db.Column(db.Text, nullable=False)
docname = db.Column(db.String(250), nullable=True, # collation='utf8_bin'
server_default='file', index=True)
creation_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00', index=True)
modification_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00', index=True)
text_extraction_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
doctype = db.Column(db.String(255))
class BibdocBibdoc(db.Model):
"""Represents a BibdocBibdoc record."""
__tablename__ = 'bibdoc_bibdoc'
id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True,
nullable=False, autoincrement=True)
id_bibdoc1 = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Bibdoc.id), nullable=True)
version1 = db.Column(db.TinyInteger(4, unsigned=True))
format1 = db.Column(db.String(50))
id_bibdoc2 = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Bibdoc.id), nullable=True)
version2 = db.Column(db.TinyInteger(4, unsigned=True))
format2 = db.Column(db.String(50))
rel_type = db.Column(db.String(255), nullable=True)
bibdoc1 = db.relationship(Bibdoc, backref='bibdoc2s',
primaryjoin=Bibdoc.id == id_bibdoc1)
bibdoc2 = db.relationship(Bibdoc, backref='bibdoc1s',
primaryjoin=Bibdoc.id == id_bibdoc2)
class BibrecBibdoc(db.Model):
"""Represents a BibrecBibdoc record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bibdoc'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), nullable=False,
server_default='0', primary_key=True)
id_bibdoc = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Bibdoc.id), nullable=False,
server_default='0', primary_key=True)
docname = db.Column(db.String(250), nullable=False, # collation='utf8_bin'
server_default='file', index=True)
type = db.Column(db.String(255), nullable=True)
bibrec = db.relationship(Bibrec, backref='bibdocs')
bibdoc = db.relationship(Bibdoc, backref='bibrecs')
class HstDOCUMENT(db.Model):
"""Represents a HstDOCUMENT record."""
def __init__(self):
pass
__tablename__ = 'hstDOCUMENT'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
nullable=False, autoincrement=True)
id_bibdoc = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Bibdoc.id), primary_key=True, nullable=False,
autoincrement=False)
docname = db.Column(db.String(250), nullable=False, index=True)
docformat = db.Column(db.String(50), nullable=False, index=True)
docversion = db.Column(db.TinyInteger(4, unsigned=True),
nullable=False)
docsize = db.Column(db.BigInteger(15, unsigned=True),
nullable=False)
docchecksum = db.Column(db.Char(32), nullable=False)
doctimestamp = db.Column(db.DateTime, nullable=False, index=True)
action = db.Column(db.String(50), nullable=False, index=True)
job_id = db.Column(db.MediumInteger(15, unsigned=True),
nullable=True, index=True)
job_name = db.Column(db.String(255), nullable=True, index=True)
job_person = db.Column(db.String(255), nullable=True, index=True)
job_date = db.Column(db.DateTime, nullable=True, index=True)
job_details = db.Column(db.iBinary, nullable=True)
class HstRECORD(db.Model):
"""Represents a HstRECORD record."""
def __init__(self):
pass
__tablename__ = 'hstRECORD'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
nullable=False, autoincrement=True)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id), autoincrement=False,
nullable=False, primary_key=True)
marcxml = db.Column(db.iBinary, nullable=False)
job_id = db.Column(db.MediumInteger(15, unsigned=True),
nullable=False, index=True)
job_name = db.Column(db.String(255), nullable=False, index=True)
job_person = db.Column(db.String(255), nullable=False, index=True)
job_date = db.Column(db.DateTime, nullable=False, index=True)
job_details = db.Column(db.iBinary, nullable=False)
affected_fields = db.Column(db.Text, nullable=True)
# GENERATED
class Bib00x(db.Model):
"""Represents a Bib00x record."""
def __init__(self):
pass
__tablename__ = 'bib00x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib00x(db.Model):
"""Represents a BibrecBib00x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib00x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib00x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib00xs')
bibxxx = db.relationship(Bib00x, backref='bibrecs')
class Bib01x(db.Model):
"""Represents a Bib01x record."""
def __init__(self):
pass
__tablename__ = 'bib01x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib01x(db.Model):
"""Represents a BibrecBib01x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib01x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib01x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib01xs')
bibxxx = db.relationship(Bib01x, backref='bibrecs')
class Bib02x(db.Model):
"""Represents a Bib02x record."""
def __init__(self):
pass
__tablename__ = 'bib02x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib02x(db.Model):
"""Represents a BibrecBib02x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib02x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib02x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib02xs')
bibxxx = db.relationship(Bib02x, backref='bibrecs')
class Bib03x(db.Model):
"""Represents a Bib03x record."""
def __init__(self):
pass
__tablename__ = 'bib03x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib03x(db.Model):
"""Represents a BibrecBib03x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib03x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib03x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib03xs')
bibxxx = db.relationship(Bib03x, backref='bibrecs')
class Bib04x(db.Model):
"""Represents a Bib04x record."""
def __init__(self):
pass
__tablename__ = 'bib04x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib04x(db.Model):
"""Represents a BibrecBib04x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib04x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib04x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib04xs')
bibxxx = db.relationship(Bib04x, backref='bibrecs')
class Bib05x(db.Model):
"""Represents a Bib05x record."""
def __init__(self):
pass
__tablename__ = 'bib05x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib05x(db.Model):
"""Represents a BibrecBib05x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib05x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib05x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib05xs')
bibxxx = db.relationship(Bib05x, backref='bibrecs')
class Bib06x(db.Model):
"""Represents a Bib06x record."""
def __init__(self):
pass
__tablename__ = 'bib06x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib06x(db.Model):
"""Represents a BibrecBib06x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib06x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib06x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib06xs')
bibxxx = db.relationship(Bib06x, backref='bibrecs')
class Bib07x(db.Model):
"""Represents a Bib07x record."""
def __init__(self):
pass
__tablename__ = 'bib07x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib07x(db.Model):
"""Represents a BibrecBib07x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib07x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib07x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib07xs')
bibxxx = db.relationship(Bib07x, backref='bibrecs')
class Bib08x(db.Model):
"""Represents a Bib08x record."""
def __init__(self):
pass
__tablename__ = 'bib08x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib08x(db.Model):
"""Represents a BibrecBib08x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib08x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib08x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib08xs')
bibxxx = db.relationship(Bib08x, backref='bibrecs')
class Bib09x(db.Model):
"""Represents a Bib09x record."""
def __init__(self):
pass
__tablename__ = 'bib09x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib09x(db.Model):
"""Represents a BibrecBib09x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib09x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib09x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib09xs')
bibxxx = db.relationship(Bib09x, backref='bibrecs')
class Bib10x(db.Model):
"""Represents a Bib10x record."""
def __init__(self):
pass
__tablename__ = 'bib10x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib10x(db.Model):
"""Represents a BibrecBib10x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib10x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib10x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib10xs')
bibxxx = db.relationship(Bib10x, backref='bibrecs')
class Bib11x(db.Model):
"""Represents a Bib11x record."""
def __init__(self):
pass
__tablename__ = 'bib11x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib11x(db.Model):
"""Represents a BibrecBib11x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib11x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib11x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib11xs')
bibxxx = db.relationship(Bib11x, backref='bibrecs')
class Bib12x(db.Model):
"""Represents a Bib12x record."""
def __init__(self):
pass
__tablename__ = 'bib12x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib12x(db.Model):
"""Represents a BibrecBib12x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib12x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib12x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib12xs')
bibxxx = db.relationship(Bib12x, backref='bibrecs')
class Bib13x(db.Model):
"""Represents a Bib13x record."""
def __init__(self):
pass
__tablename__ = 'bib13x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib13x(db.Model):
"""Represents a BibrecBib13x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib13x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib13x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib13xs')
bibxxx = db.relationship(Bib13x, backref='bibrecs')
class Bib14x(db.Model):
"""Represents a Bib14x record."""
def __init__(self):
pass
__tablename__ = 'bib14x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib14x(db.Model):
"""Represents a BibrecBib14x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib14x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib14x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib14xs')
bibxxx = db.relationship(Bib14x, backref='bibrecs')
class Bib15x(db.Model):
"""Represents a Bib15x record."""
def __init__(self):
pass
__tablename__ = 'bib15x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib15x(db.Model):
"""Represents a BibrecBib15x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib15x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib15x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib15xs')
bibxxx = db.relationship(Bib15x, backref='bibrecs')
class Bib16x(db.Model):
"""Represents a Bib16x record."""
def __init__(self):
pass
__tablename__ = 'bib16x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib16x(db.Model):
"""Represents a BibrecBib16x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib16x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib16x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib16xs')
bibxxx = db.relationship(Bib16x, backref='bibrecs')
class Bib17x(db.Model):
"""Represents a Bib17x record."""
def __init__(self):
pass
__tablename__ = 'bib17x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib17x(db.Model):
"""Represents a BibrecBib17x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib17x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib17x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib17xs')
bibxxx = db.relationship(Bib17x, backref='bibrecs')
class Bib18x(db.Model):
"""Represents a Bib18x record."""
def __init__(self):
pass
__tablename__ = 'bib18x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib18x(db.Model):
"""Represents a BibrecBib18x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib18x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib18x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib18xs')
bibxxx = db.relationship(Bib18x, backref='bibrecs')
class Bib19x(db.Model):
"""Represents a Bib19x record."""
def __init__(self):
pass
__tablename__ = 'bib19x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib19x(db.Model):
"""Represents a BibrecBib19x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib19x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib19x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib19xs')
bibxxx = db.relationship(Bib19x, backref='bibrecs')
class Bib20x(db.Model):
"""Represents a Bib20x record."""
def __init__(self):
pass
__tablename__ = 'bib20x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib20x(db.Model):
"""Represents a BibrecBib20x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib20x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib20x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib20xs')
bibxxx = db.relationship(Bib20x, backref='bibrecs')
class Bib21x(db.Model):
"""Represents a Bib21x record."""
def __init__(self):
pass
__tablename__ = 'bib21x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib21x(db.Model):
"""Represents a BibrecBib21x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib21x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib21x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib21xs')
bibxxx = db.relationship(Bib21x, backref='bibrecs')
class Bib22x(db.Model):
"""Represents a Bib22x record."""
def __init__(self):
pass
__tablename__ = 'bib22x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib22x(db.Model):
"""Represents a BibrecBib22x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib22x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib22x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib22xs')
bibxxx = db.relationship(Bib22x, backref='bibrecs')
class Bib23x(db.Model):
"""Represents a Bib23x record."""
def __init__(self):
pass
__tablename__ = 'bib23x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib23x(db.Model):
"""Represents a BibrecBib23x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib23x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib23x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib23xs')
bibxxx = db.relationship(Bib23x, backref='bibrecs')
class Bib24x(db.Model):
"""Represents a Bib24x record."""
def __init__(self):
pass
__tablename__ = 'bib24x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib24x(db.Model):
"""Represents a BibrecBib24x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib24x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib24x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib24xs')
bibxxx = db.relationship(Bib24x, backref='bibrecs')
class Bib25x(db.Model):
"""Represents a Bib25x record."""
def __init__(self):
pass
__tablename__ = 'bib25x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib25x(db.Model):
"""Represents a BibrecBib25x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib25x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib25x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib25xs')
bibxxx = db.relationship(Bib25x, backref='bibrecs')
class Bib26x(db.Model):
"""Represents a Bib26x record."""
def __init__(self):
pass
__tablename__ = 'bib26x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib26x(db.Model):
"""Represents a BibrecBib26x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib26x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib26x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib26xs')
bibxxx = db.relationship(Bib26x, backref='bibrecs')
class Bib27x(db.Model):
"""Represents a Bib27x record."""
def __init__(self):
pass
__tablename__ = 'bib27x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib27x(db.Model):
"""Represents a BibrecBib27x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib27x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib27x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib27xs')
bibxxx = db.relationship(Bib27x, backref='bibrecs')
class Bib28x(db.Model):
"""Represents a Bib28x record."""
def __init__(self):
pass
__tablename__ = 'bib28x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib28x(db.Model):
"""Represents a BibrecBib28x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib28x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib28x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib28xs')
bibxxx = db.relationship(Bib28x, backref='bibrecs')
class Bib29x(db.Model):
"""Represents a Bib29x record."""
def __init__(self):
pass
__tablename__ = 'bib29x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib29x(db.Model):
"""Represents a BibrecBib29x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib29x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib29x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib29xs')
bibxxx = db.relationship(Bib29x, backref='bibrecs')
class Bib30x(db.Model):
"""Represents a Bib30x record."""
def __init__(self):
pass
__tablename__ = 'bib30x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib30x(db.Model):
"""Represents a BibrecBib30x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib30x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib30x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib30xs')
bibxxx = db.relationship(Bib30x, backref='bibrecs')
class Bib31x(db.Model):
"""Represents a Bib31x record."""
def __init__(self):
pass
__tablename__ = 'bib31x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib31x(db.Model):
"""Represents a BibrecBib31x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib31x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib31x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib31xs')
bibxxx = db.relationship(Bib31x, backref='bibrecs')
class Bib32x(db.Model):
"""Represents a Bib32x record."""
def __init__(self):
pass
__tablename__ = 'bib32x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib32x(db.Model):
"""Represents a BibrecBib32x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib32x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib32x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib32xs')
bibxxx = db.relationship(Bib32x, backref='bibrecs')
class Bib33x(db.Model):
"""Represents a Bib33x record."""
def __init__(self):
pass
__tablename__ = 'bib33x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib33x(db.Model):
"""Represents a BibrecBib33x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib33x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib33x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib33xs')
bibxxx = db.relationship(Bib33x, backref='bibrecs')
class Bib34x(db.Model):
"""Represents a Bib34x record."""
def __init__(self):
pass
__tablename__ = 'bib34x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib34x(db.Model):
"""Represents a BibrecBib34x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib34x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib34x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib34xs')
bibxxx = db.relationship(Bib34x, backref='bibrecs')
class Bib35x(db.Model):
"""Represents a Bib35x record."""
def __init__(self):
pass
__tablename__ = 'bib35x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib35x(db.Model):
"""Represents a BibrecBib35x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib35x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib35x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib35xs')
bibxxx = db.relationship(Bib35x, backref='bibrecs')
class Bib36x(db.Model):
"""Represents a Bib36x record."""
def __init__(self):
pass
__tablename__ = 'bib36x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib36x(db.Model):
"""Represents a BibrecBib36x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib36x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib36x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib36xs')
bibxxx = db.relationship(Bib36x, backref='bibrecs')
class Bib37x(db.Model):
"""Represents a Bib37x record."""
def __init__(self):
pass
__tablename__ = 'bib37x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib37x(db.Model):
"""Represents a BibrecBib37x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib37x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib37x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib37xs')
bibxxx = db.relationship(Bib37x, backref='bibrecs')
class Bib38x(db.Model):
"""Represents a Bib38x record."""
def __init__(self):
pass
__tablename__ = 'bib38x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib38x(db.Model):
"""Represents a BibrecBib38x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib38x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib38x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib38xs')
bibxxx = db.relationship(Bib38x, backref='bibrecs')
class Bib39x(db.Model):
"""Represents a Bib39x record."""
def __init__(self):
pass
__tablename__ = 'bib39x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib39x(db.Model):
"""Represents a BibrecBib39x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib39x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib39x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib39xs')
bibxxx = db.relationship(Bib39x, backref='bibrecs')
class Bib40x(db.Model):
"""Represents a Bib40x record."""
def __init__(self):
pass
__tablename__ = 'bib40x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib40x(db.Model):
"""Represents a BibrecBib40x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib40x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib40x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib40xs')
bibxxx = db.relationship(Bib40x, backref='bibrecs')
class Bib41x(db.Model):
"""Represents a Bib41x record."""
def __init__(self):
pass
__tablename__ = 'bib41x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib41x(db.Model):
"""Represents a BibrecBib41x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib41x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib41x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib41xs')
bibxxx = db.relationship(Bib41x, backref='bibrecs')
class Bib42x(db.Model):
"""Represents a Bib42x record."""
def __init__(self):
pass
__tablename__ = 'bib42x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib42x(db.Model):
"""Represents a BibrecBib42x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib42x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib42x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib42xs')
bibxxx = db.relationship(Bib42x, backref='bibrecs')
class Bib43x(db.Model):
"""Represents a Bib43x record."""
def __init__(self):
pass
__tablename__ = 'bib43x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib43x(db.Model):
"""Represents a BibrecBib43x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib43x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib43x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib43xs')
bibxxx = db.relationship(Bib43x, backref='bibrecs')
class Bib44x(db.Model):
"""Represents a Bib44x record."""
def __init__(self):
pass
__tablename__ = 'bib44x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib44x(db.Model):
"""Represents a BibrecBib44x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib44x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib44x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib44xs')
bibxxx = db.relationship(Bib44x, backref='bibrecs')
class Bib45x(db.Model):
"""Represents a Bib45x record."""
def __init__(self):
pass
__tablename__ = 'bib45x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib45x(db.Model):
"""Represents a BibrecBib45x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib45x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib45x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib45xs')
bibxxx = db.relationship(Bib45x, backref='bibrecs')
class Bib46x(db.Model):
"""Represents a Bib46x record."""
def __init__(self):
pass
__tablename__ = 'bib46x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib46x(db.Model):
"""Represents a BibrecBib46x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib46x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib46x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib46xs')
bibxxx = db.relationship(Bib46x, backref='bibrecs')
class Bib47x(db.Model):
"""Represents a Bib47x record."""
def __init__(self):
pass
__tablename__ = 'bib47x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib47x(db.Model):
"""Represents a BibrecBib47x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib47x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib47x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib47xs')
bibxxx = db.relationship(Bib47x, backref='bibrecs')
class Bib48x(db.Model):
"""Represents a Bib48x record."""
def __init__(self):
pass
__tablename__ = 'bib48x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib48x(db.Model):
"""Represents a BibrecBib48x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib48x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib48x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib48xs')
bibxxx = db.relationship(Bib48x, backref='bibrecs')
class Bib49x(db.Model):
"""Represents a Bib49x record."""
def __init__(self):
pass
__tablename__ = 'bib49x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib49x(db.Model):
"""Represents a BibrecBib49x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib49x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib49x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib49xs')
bibxxx = db.relationship(Bib49x, backref='bibrecs')
class Bib50x(db.Model):
"""Represents a Bib50x record."""
def __init__(self):
pass
__tablename__ = 'bib50x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib50x(db.Model):
"""Represents a BibrecBib50x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib50x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib50x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib50xs')
bibxxx = db.relationship(Bib50x, backref='bibrecs')
class Bib51x(db.Model):
"""Represents a Bib51x record."""
def __init__(self):
pass
__tablename__ = 'bib51x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib51x(db.Model):
"""Represents a BibrecBib51x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib51x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib51x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib51xs')
bibxxx = db.relationship(Bib51x, backref='bibrecs')
class Bib52x(db.Model):
"""Represents a Bib52x record."""
def __init__(self):
pass
__tablename__ = 'bib52x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib52x(db.Model):
"""Represents a BibrecBib52x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib52x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib52x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib52xs')
bibxxx = db.relationship(Bib52x, backref='bibrecs')
class Bib53x(db.Model):
"""Represents a Bib53x record."""
def __init__(self):
pass
__tablename__ = 'bib53x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib53x(db.Model):
"""Represents a BibrecBib53x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib53x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib53x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib53xs')
bibxxx = db.relationship(Bib53x, backref='bibrecs')
class Bib54x(db.Model):
"""Represents a Bib54x record."""
def __init__(self):
pass
__tablename__ = 'bib54x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib54x(db.Model):
"""Represents a BibrecBib54x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib54x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib54x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib54xs')
bibxxx = db.relationship(Bib54x, backref='bibrecs')
class Bib55x(db.Model):
"""Represents a Bib55x record."""
def __init__(self):
pass
__tablename__ = 'bib55x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib55x(db.Model):
"""Represents a BibrecBib55x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib55x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib55x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib55xs')
bibxxx = db.relationship(Bib55x, backref='bibrecs')
class Bib56x(db.Model):
"""Represents a Bib56x record."""
def __init__(self):
pass
__tablename__ = 'bib56x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib56x(db.Model):
"""Represents a BibrecBib56x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib56x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib56x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib56xs')
bibxxx = db.relationship(Bib56x, backref='bibrecs')
class Bib57x(db.Model):
"""Represents a Bib57x record."""
def __init__(self):
pass
__tablename__ = 'bib57x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib57x(db.Model):
"""Represents a BibrecBib57x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib57x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib57x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib57xs')
bibxxx = db.relationship(Bib57x, backref='bibrecs')
class Bib58x(db.Model):
"""Represents a Bib58x record."""
def __init__(self):
pass
__tablename__ = 'bib58x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib58x(db.Model):
"""Represents a BibrecBib58x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib58x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib58x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib58xs')
bibxxx = db.relationship(Bib58x, backref='bibrecs')
class Bib59x(db.Model):
"""Represents a Bib59x record."""
def __init__(self):
pass
__tablename__ = 'bib59x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib59x(db.Model):
"""Represents a BibrecBib59x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib59x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib59x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib59xs')
bibxxx = db.relationship(Bib59x, backref='bibrecs')
class Bib60x(db.Model):
"""Represents a Bib60x record."""
def __init__(self):
pass
__tablename__ = 'bib60x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib60x(db.Model):
"""Represents a BibrecBib60x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib60x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib60x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib60xs')
bibxxx = db.relationship(Bib60x, backref='bibrecs')
class Bib61x(db.Model):
"""Represents a Bib61x record."""
def __init__(self):
pass
__tablename__ = 'bib61x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib61x(db.Model):
"""Represents a BibrecBib61x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib61x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib61x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib61xs')
bibxxx = db.relationship(Bib61x, backref='bibrecs')
class Bib62x(db.Model):
"""Represents a Bib62x record."""
def __init__(self):
pass
__tablename__ = 'bib62x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib62x(db.Model):
"""Represents a BibrecBib62x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib62x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib62x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib62xs')
bibxxx = db.relationship(Bib62x, backref='bibrecs')
class Bib63x(db.Model):
"""Represents a Bib63x record."""
def __init__(self):
pass
__tablename__ = 'bib63x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib63x(db.Model):
"""Represents a BibrecBib63x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib63x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib63x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib63xs')
bibxxx = db.relationship(Bib63x, backref='bibrecs')
class Bib64x(db.Model):
"""Represents a Bib64x record."""
def __init__(self):
pass
__tablename__ = 'bib64x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib64x(db.Model):
"""Represents a BibrecBib64x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib64x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib64x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib64xs')
bibxxx = db.relationship(Bib64x, backref='bibrecs')
class Bib65x(db.Model):
"""Represents a Bib65x record."""
def __init__(self):
pass
__tablename__ = 'bib65x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib65x(db.Model):
"""Represents a BibrecBib65x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib65x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib65x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib65xs')
bibxxx = db.relationship(Bib65x, backref='bibrecs')
class Bib66x(db.Model):
"""Represents a Bib66x record."""
def __init__(self):
pass
__tablename__ = 'bib66x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib66x(db.Model):
"""Represents a BibrecBib66x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib66x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib66x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib66xs')
bibxxx = db.relationship(Bib66x, backref='bibrecs')
class Bib67x(db.Model):
"""Represents a Bib67x record."""
def __init__(self):
pass
__tablename__ = 'bib67x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib67x(db.Model):
"""Represents a BibrecBib67x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib67x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib67x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib67xs')
bibxxx = db.relationship(Bib67x, backref='bibrecs')
class Bib68x(db.Model):
"""Represents a Bib68x record."""
def __init__(self):
pass
__tablename__ = 'bib68x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib68x(db.Model):
"""Represents a BibrecBib68x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib68x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib68x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib68xs')
bibxxx = db.relationship(Bib68x, backref='bibrecs')
class Bib69x(db.Model):
"""Represents a Bib69x record."""
def __init__(self):
pass
__tablename__ = 'bib69x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib69x(db.Model):
"""Represents a BibrecBib69x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib69x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib69x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib69xs')
bibxxx = db.relationship(Bib69x, backref='bibrecs')
class Bib70x(db.Model):
"""Represents a Bib70x record."""
def __init__(self):
pass
__tablename__ = 'bib70x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib70x(db.Model):
"""Represents a BibrecBib70x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib70x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib70x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib70xs')
bibxxx = db.relationship(Bib70x, backref='bibrecs')
class Bib71x(db.Model):
"""Represents a Bib71x record."""
def __init__(self):
pass
__tablename__ = 'bib71x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib71x(db.Model):
"""Represents a BibrecBib71x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib71x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib71x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib71xs')
bibxxx = db.relationship(Bib71x, backref='bibrecs')
class Bib72x(db.Model):
"""Represents a Bib72x record."""
def __init__(self):
pass
__tablename__ = 'bib72x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib72x(db.Model):
"""Represents a BibrecBib72x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib72x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib72x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib72xs')
bibxxx = db.relationship(Bib72x, backref='bibrecs')
class Bib73x(db.Model):
"""Represents a Bib73x record."""
def __init__(self):
pass
__tablename__ = 'bib73x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib73x(db.Model):
"""Represents a BibrecBib73x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib73x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib73x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib73xs')
bibxxx = db.relationship(Bib73x, backref='bibrecs')
class Bib74x(db.Model):
"""Represents a Bib74x record."""
def __init__(self):
pass
__tablename__ = 'bib74x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib74x(db.Model):
"""Represents a BibrecBib74x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib74x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib74x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib74xs')
bibxxx = db.relationship(Bib74x, backref='bibrecs')
class Bib75x(db.Model):
"""Represents a Bib75x record."""
def __init__(self):
pass
__tablename__ = 'bib75x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib75x(db.Model):
"""Represents a BibrecBib75x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib75x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib75x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib75xs')
bibxxx = db.relationship(Bib75x, backref='bibrecs')
class Bib76x(db.Model):
"""Represents a Bib76x record."""
def __init__(self):
pass
__tablename__ = 'bib76x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib76x(db.Model):
"""Represents a BibrecBib76x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib76x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib76x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib76xs')
bibxxx = db.relationship(Bib76x, backref='bibrecs')
class Bib77x(db.Model):
"""Represents a Bib77x record."""
def __init__(self):
pass
__tablename__ = 'bib77x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib77x(db.Model):
"""Represents a BibrecBib77x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib77x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib77x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib77xs')
bibxxx = db.relationship(Bib77x, backref='bibrecs')
class Bib78x(db.Model):
"""Represents a Bib78x record."""
def __init__(self):
pass
__tablename__ = 'bib78x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib78x(db.Model):
"""Represents a BibrecBib78x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib78x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib78x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib78xs')
bibxxx = db.relationship(Bib78x, backref='bibrecs')
class Bib79x(db.Model):
"""Represents a Bib79x record."""
def __init__(self):
pass
__tablename__ = 'bib79x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib79x(db.Model):
"""Represents a BibrecBib79x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib79x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib79x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib79xs')
bibxxx = db.relationship(Bib79x, backref='bibrecs')
class Bib80x(db.Model):
"""Represents a Bib80x record."""
def __init__(self):
pass
__tablename__ = 'bib80x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib80x(db.Model):
"""Represents a BibrecBib80x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib80x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib80x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib80xs')
bibxxx = db.relationship(Bib80x, backref='bibrecs')
class Bib81x(db.Model):
"""Represents a Bib81x record."""
def __init__(self):
pass
__tablename__ = 'bib81x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib81x(db.Model):
"""Represents a BibrecBib81x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib81x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib81x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib81xs')
bibxxx = db.relationship(Bib81x, backref='bibrecs')
class Bib82x(db.Model):
"""Represents a Bib82x record."""
def __init__(self):
pass
__tablename__ = 'bib82x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib82x(db.Model):
"""Represents a BibrecBib82x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib82x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib82x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib82xs')
bibxxx = db.relationship(Bib82x, backref='bibrecs')
class Bib83x(db.Model):
"""Represents a Bib83x record."""
def __init__(self):
pass
__tablename__ = 'bib83x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib83x(db.Model):
"""Represents a BibrecBib83x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib83x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib83x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib83xs')
bibxxx = db.relationship(Bib83x, backref='bibrecs')
class Bib84x(db.Model):
"""Represents a Bib84x record."""
def __init__(self):
pass
__tablename__ = 'bib84x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib84x(db.Model):
"""Represents a BibrecBib84x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib84x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib84x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib84xs')
bibxxx = db.relationship(Bib84x, backref='bibrecs')
class Bib85x(db.Model):
"""Represents a Bib85x record."""
def __init__(self):
pass
__tablename__ = 'bib85x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib85x(db.Model):
"""Represents a BibrecBib85x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib85x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib85x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib85xs')
bibxxx = db.relationship(Bib85x, backref='bibrecs')
class Bib86x(db.Model):
"""Represents a Bib86x record."""
def __init__(self):
pass
__tablename__ = 'bib86x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib86x(db.Model):
"""Represents a BibrecBib86x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib86x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib86x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib86xs')
bibxxx = db.relationship(Bib86x, backref='bibrecs')
class Bib87x(db.Model):
"""Represents a Bib87x record."""
def __init__(self):
pass
__tablename__ = 'bib87x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib87x(db.Model):
"""Represents a BibrecBib87x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib87x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib87x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib87xs')
bibxxx = db.relationship(Bib87x, backref='bibrecs')
class Bib88x(db.Model):
"""Represents a Bib88x record."""
def __init__(self):
pass
__tablename__ = 'bib88x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib88x(db.Model):
"""Represents a BibrecBib88x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib88x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib88x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib88xs')
bibxxx = db.relationship(Bib88x, backref='bibrecs')
class Bib89x(db.Model):
"""Represents a Bib89x record."""
def __init__(self):
pass
__tablename__ = 'bib89x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib89x(db.Model):
"""Represents a BibrecBib89x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib89x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib89x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib89xs')
bibxxx = db.relationship(Bib89x, backref='bibrecs')
class Bib90x(db.Model):
"""Represents a Bib90x record."""
def __init__(self):
pass
__tablename__ = 'bib90x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib90x(db.Model):
"""Represents a BibrecBib90x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib90x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib90x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib90xs')
bibxxx = db.relationship(Bib90x, backref='bibrecs')
class Bib91x(db.Model):
"""Represents a Bib91x record."""
def __init__(self):
pass
__tablename__ = 'bib91x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib91x(db.Model):
"""Represents a BibrecBib91x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib91x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib91x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib91xs')
bibxxx = db.relationship(Bib91x, backref='bibrecs')
class Bib92x(db.Model):
"""Represents a Bib92x record."""
def __init__(self):
pass
__tablename__ = 'bib92x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib92x(db.Model):
"""Represents a BibrecBib92x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib92x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib92x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib92xs')
bibxxx = db.relationship(Bib92x, backref='bibrecs')
class Bib93x(db.Model):
"""Represents a Bib93x record."""
def __init__(self):
pass
__tablename__ = 'bib93x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib93x(db.Model):
"""Represents a BibrecBib93x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib93x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib93x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib93xs')
bibxxx = db.relationship(Bib93x, backref='bibrecs')
class Bib94x(db.Model):
"""Represents a Bib94x record."""
def __init__(self):
pass
__tablename__ = 'bib94x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib94x(db.Model):
"""Represents a BibrecBib94x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib94x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib94x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib94xs')
bibxxx = db.relationship(Bib94x, backref='bibrecs')
class Bib95x(db.Model):
"""Represents a Bib95x record."""
def __init__(self):
pass
__tablename__ = 'bib95x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib95x(db.Model):
"""Represents a BibrecBib95x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib95x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib95x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib95xs')
bibxxx = db.relationship(Bib95x, backref='bibrecs')
class Bib96x(db.Model):
"""Represents a Bib96x record."""
def __init__(self):
pass
__tablename__ = 'bib96x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib96x(db.Model):
"""Represents a BibrecBib96x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib96x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib96x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib96xs')
bibxxx = db.relationship(Bib96x, backref='bibrecs')
class Bib97x(db.Model):
"""Represents a Bib97x record."""
def __init__(self):
pass
__tablename__ = 'bib97x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib97x(db.Model):
"""Represents a BibrecBib97x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib97x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib97x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib97xs')
bibxxx = db.relationship(Bib97x, backref='bibrecs')
class Bib98x(db.Model):
"""Represents a Bib98x record."""
def __init__(self):
pass
__tablename__ = 'bib98x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib98x(db.Model):
"""Represents a BibrecBib98x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib98x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib98x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib98xs')
bibxxx = db.relationship(Bib98x, backref='bibrecs')
class Bib99x(db.Model):
"""Represents a Bib99x record."""
def __init__(self):
pass
__tablename__ = 'bib99x'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
autoincrement=True)
tag = db.Column(db.String(6), nullable=False, index=True,
server_default='')
value = db.Column(db.Text(35), nullable=False,
index=True)
class BibrecBib99x(db.Model):
"""Represents a BibrecBib99x record."""
def __init__(self):
pass
__tablename__ = 'bibrec_bib99x'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False, primary_key=True, index=True,
server_default='0')
id_bibxxx = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bib99x.id),
nullable=False, primary_key=True, index=True,
server_default='0')
field_number = db.Column(db.SmallInteger(5, unsigned=True),
primary_key=True)
bibrec = db.relationship(Bibrec, backref='bib99xs')
bibxxx = db.relationship(Bib99x, backref='bibrecs')
__all__ = ['Bibrec',
'Bibfmt',
'BibHOLDINGPEN',
'Bibdoc',
'BibdocBibdoc',
'BibrecBibdoc',
'HstDOCUMENT',
'HstRECORD',
'Bib00x',
'BibrecBib00x',
'Bib01x',
'BibrecBib01x',
'Bib02x',
'BibrecBib02x',
'Bib03x',
'BibrecBib03x',
'Bib04x',
'BibrecBib04x',
'Bib05x',
'BibrecBib05x',
'Bib06x',
'BibrecBib06x',
'Bib07x',
'BibrecBib07x',
'Bib08x',
'BibrecBib08x',
'Bib09x',
'BibrecBib09x',
'Bib10x',
'BibrecBib10x',
'Bib11x',
'BibrecBib11x',
'Bib12x',
'BibrecBib12x',
'Bib13x',
'BibrecBib13x',
'Bib14x',
'BibrecBib14x',
'Bib15x',
'BibrecBib15x',
'Bib16x',
'BibrecBib16x',
'Bib17x',
'BibrecBib17x',
'Bib18x',
'BibrecBib18x',
'Bib19x',
'BibrecBib19x',
'Bib20x',
'BibrecBib20x',
'Bib21x',
'BibrecBib21x',
'Bib22x',
'BibrecBib22x',
'Bib23x',
'BibrecBib23x',
'Bib24x',
'BibrecBib24x',
'Bib25x',
'BibrecBib25x',
'Bib26x',
'BibrecBib26x',
'Bib27x',
'BibrecBib27x',
'Bib28x',
'BibrecBib28x',
'Bib29x',
'BibrecBib29x',
'Bib30x',
'BibrecBib30x',
'Bib31x',
'BibrecBib31x',
'Bib32x',
'BibrecBib32x',
'Bib33x',
'BibrecBib33x',
'Bib34x',
'BibrecBib34x',
'Bib35x',
'BibrecBib35x',
'Bib36x',
'BibrecBib36x',
'Bib37x',
'BibrecBib37x',
'Bib38x',
'BibrecBib38x',
'Bib39x',
'BibrecBib39x',
'Bib40x',
'BibrecBib40x',
'Bib41x',
'BibrecBib41x',
'Bib42x',
'BibrecBib42x',
'Bib43x',
'BibrecBib43x',
'Bib44x',
'BibrecBib44x',
'Bib45x',
'BibrecBib45x',
'Bib46x',
'BibrecBib46x',
'Bib47x',
'BibrecBib47x',
'Bib48x',
'BibrecBib48x',
'Bib49x',
'BibrecBib49x',
'Bib50x',
'BibrecBib50x',
'Bib51x',
'BibrecBib51x',
'Bib52x',
'BibrecBib52x',
'Bib53x',
'BibrecBib53x',
'Bib54x',
'BibrecBib54x',
'Bib55x',
'BibrecBib55x',
'Bib56x',
'BibrecBib56x',
'Bib57x',
'BibrecBib57x',
'Bib58x',
'BibrecBib58x',
'Bib59x',
'BibrecBib59x',
'Bib60x',
'BibrecBib60x',
'Bib61x',
'BibrecBib61x',
'Bib62x',
'BibrecBib62x',
'Bib63x',
'BibrecBib63x',
'Bib64x',
'BibrecBib64x',
'Bib65x',
'BibrecBib65x',
'Bib66x',
'BibrecBib66x',
'Bib67x',
'BibrecBib67x',
'Bib68x',
'BibrecBib68x',
'Bib69x',
'BibrecBib69x',
'Bib70x',
'BibrecBib70x',
'Bib71x',
'BibrecBib71x',
'Bib72x',
'BibrecBib72x',
'Bib73x',
'BibrecBib73x',
'Bib74x',
'BibrecBib74x',
'Bib75x',
'BibrecBib75x',
'Bib76x',
'BibrecBib76x',
'Bib77x',
'BibrecBib77x',
'Bib78x',
'BibrecBib78x',
'Bib79x',
'BibrecBib79x',
'Bib80x',
'BibrecBib80x',
'Bib81x',
'BibrecBib81x',
'Bib82x',
'BibrecBib82x',
'Bib83x',
'BibrecBib83x',
'Bib84x',
'BibrecBib84x',
'Bib85x',
'BibrecBib85x',
'Bib86x',
'BibrecBib86x',
'Bib87x',
'BibrecBib87x',
'Bib88x',
'BibrecBib88x',
'Bib89x',
'BibrecBib89x',
'Bib90x',
'BibrecBib90x',
'Bib91x',
'BibrecBib91x',
'Bib92x',
'BibrecBib92x',
'Bib93x',
'BibrecBib93x',
'Bib94x',
'BibrecBib94x',
'Bib95x',
'BibrecBib95x',
'Bib96x',
'BibrecBib96x',
'Bib97x',
'BibrecBib97x',
'Bib98x',
'BibrecBib98x',
'Bib99x',
'BibrecBib99x']
diff --git a/invenio/modules/records/api.py b/invenio/modules/records/api.py
new file mode 100644
index 000000000..975b8b44b
--- /dev/null
+++ b/invenio/modules/records/api.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.records.api
+ ---------------------------
+
+"""
+
+from invenio.modules.jsonalchemy.jsonext.engines.sqlalchemy import SQLAlchemyStorage
+from invenio.modules.jsonalchemy.parser import FieldParser
+from invenio.modules.jsonalchemy.registry import readers
+from invenio.modules.jsonalchemy.wrappers import SmartJson
+
+from .models import RecordMetadata as RecordMetadataModel
+from .models import Record as RecordModel
+
+
+class Record(SmartJson):
+ """
+ Default/Base record class
+ """
+
+ storage_engine = SQLAlchemyStorage(RecordMetadataModel)
+
+ @classmethod
+ def create(cls, blob, master_format, **kwargs):
+ reader = readers[master_format](blob, namespace='recordext', **kwargs)
+ return cls(reader.translate())
+
+ @classmethod
+ def get_record(cls, recid):
+ try:
+ json = cls.storage_engine.get_one(recid)
+ return Record(json)
+ except:
+ # try to retrieve the record from the master format if any
+ # this might be deprecated in the near future as soon as json will
+ # become the master format, until then ...
+ blob = cls.get_blob(recid)
+ record_model = RecordModel.query.get(recid)
+ if record_model is None or blob is None:
+ return None
+ additional_info = record_model.additional_info \
+ if record_model.additional_info else {'master_format': 'marc'}
+ record = cls.create(blob, record_model.master_format, **additional_info)
+ record._save()
+ record_model.additional_info = \
+ record.get('__meta_metadata__.__additional_info__', {'master_format': 'marc'})
+ record_model.commit()
+ return record
+
+ @staticmethod
+ def get_blob(recid):
+ #FIXME: start using bibarchive or bibingest for this
+ from invenio.modules.formatter.models import Bibfmt
+ from zlib import decompress
+
+ record_blob = Bibfmt.query.get((recid, 'xm'))
+ if record_blob is None:
+ return None
+ return decompress(record_blob.value)
+
+ def _save(self):
+ self.storage_engine.save_one(self.dumps())
+
+# Functional interface
+create_record = Record.create
+get_record = Record.get_record
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/invenio/modules/records/models.py b/invenio/modules/records/models.py
new file mode 100644
index 000000000..8802c687c
--- /dev/null
+++ b/invenio/modules/records/models.py
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2014 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+"""
+ invenio.modules.record.models
+ -----------------------------
+"""
+
+from invenio.ext.sqlalchemy import db
+from flask import current_app
+from werkzeug import cached_property
+
+class Record(db.Model):
+ """Represents a record object inside the SQL database"""
+
+ __tablename__ = 'bibrec'
+
+ id = db.Column(db.MediumInteger(8, unsigned=True), primary_key=True,
+ nullable=False, autoincrement=True)
+ creation_date = db.Column(db.DateTime, nullable=False,
+ server_default='1900-01-01 00:00:00',
+ index=True)
+ modification_date = db.Column(db.DateTime, nullable=False,
+ server_default='1900-01-01 00:00:00',
+ index=True)
+ master_format = db.Column(db.String(16), nullable=False,
+ server_default='marc')
+ additional_info = db.Column(db.JSON)
+
+ #FIXME: remove this from the model and add them to the record class, all?
+
+ @property
+ def deleted(self):
+ """
+ Return True if record is marked as deleted.
+ """
+ from invenio.legacy.bibrecord import get_fieldvalues
+ # record exists; now check whether it isn't marked as deleted:
+ dbcollids = get_fieldvalues(self.id, "980__%")
+
+ return ("DELETED" in dbcollids) or \
+ (current_app.config.get('CFG_CERN_SITE')
+ and "DUMMY" in dbcollids)
+
+ @staticmethod
+ def _next_merged_recid(recid):
+ """ Returns the ID of record merged with record with ID = recid """
+ from invenio.legacy.bibrecord import get_fieldvalues
+ merged_recid = None
+ for val in get_fieldvalues(recid, "970__d"):
+ try:
+ merged_recid = int(val)
+ break
+ except ValueError:
+ pass
+
+ if not merged_recid:
+ return None
+ else:
+ return merged_recid
+
+ @cached_property
+ def merged_recid(self):
+ """ Return the record object with
+ which the given record has been merged.
+ @param recID: deleted record recID
+ @type recID: int
+ @return: merged record recID
+ @rtype: int or None
+ """
+ return Record._next_merged_recid(self.id)
+
+ @property
+ def merged_recid_final(self):
+ """ Returns the last record from hierarchy of
+ records merged with this one """
+
+ cur_id = self.id
+ next_id = Record._next_merged_recid(cur_id)
+
+ while next_id:
+ cur_id = next_id
+ next_id = Record._next_merged_recid(cur_id)
+
+ return cur_id
+
+ @cached_property
+ def is_restricted(self):
+ """Returns True is record is restricted."""
+ from invenio.legacy.search_engine import get_restricted_collections_for_recid
+
+ if get_restricted_collections_for_recid(self.id,
+ recreate_cache_if_needed=False):
+ return True
+ elif self.is_processed:
+ return True
+ return False
+
+ @cached_property
+ def is_processed(self):
+ """Returns True is recods is processed (not in any collection)."""
+ from invenio.legacy.search_engine import is_record_in_any_collection
+ return not is_record_in_any_collection(self.id,
+ recreate_cache_if_needed=False)
+
+
+class RecordMetadata(db.Model):
+ """Represents a json record inside the SQL database"""
+
+ __tablename__ = 'record_metadata'
+
+ id = db.Column(db.MediumInteger(8, unsigned=True),
+ db.ForeignKey(Record.id),
+ primary_key=True,
+ nullable=False,
+ autoincrement=True)
+ json = db.Column(db.MarshalBinary(default_value={}, force_type=dict),
+ nullable=False)
+
+ record = db.relationship(Record, backref='record_metadata')
+
+__all__ = ['Record',
+ 'RecordMetadata',
+ ]
diff --git a/invenio/core/record/backends/__init__.py b/invenio/modules/records/recordext/__init__.py
similarity index 100%
rename from invenio/core/record/backends/__init__.py
rename to invenio/modules/records/recordext/__init__.py
diff --git a/invenio/modules/records/recordext/fields/base.cfg b/invenio/modules/records/recordext/fields/base.cfg
new file mode 100644
index 000000000..91bd19cf9
--- /dev/null
+++ b/invenio/modules/records/recordext/fields/base.cfg
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+##
+## This file is part of Invenio.
+## Copyright (C) 2013 CERN.
+##
+## Invenio is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License as
+## published by the Free Software Foundation; either version 2 of the
+## License, or (at your option) any later version.
+##
+## Invenio is distributed in the hope that it will be useful, but
+## WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+## General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with Invenio; if not, write to the Free Software Foundation, Inc.,
+## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+@persistent_identifier(0)
+_id:
+ """
+ This is the main persistent identifier of th record and will be
+ used internally as this, therefore the pid weight must always be '0'.
+
+ """
+ schema:
+ {'_id': {'type':'integer', 'min': 1}}
+ creator:
+ @legacy(('001', ''), )
+ marc, '001', int(value)
+ producer:
+ json_for_marc(), {'001': ''}
+
+@persistent_identifier(0)
+recid:
+ """
+ Hard link to ```_id```
+ """
+ derived:
+ @depends_on(('_id', ))
+ self['_id']
+
+modification_date:
+ """ Modification date """
+ schema:
+ {'modification_date': {'type': 'datetime', 'required': True, 'default': lambda: __import__('datetime').datetime.now()}}
+ creator:
+ @legacy(('005', ''),)
+ marc, '005', datetime.datetime(*(time.strptime(value, "%Y%m%d%H%M%S.0")[0:6]))
+ json:
+ dumps, lambda d: d.isoformat()
+ loads, lambda d: __import__('datetime').datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
+
+creation_date:
+ """ Creation date """
+ schema:
+ {'creation_date': {'type': 'datetime', 'required': True, 'default': lambda: __import__('datetime').datetime.now()}}
+ json:
+ dumps, lambda d: d.isoformat()
+ loads, lambda d: __import__('datetime').datetime.strptime(d, "%Y-%m-%dT%H:%M:%S")
diff --git a/invenio/core/record/recordext/__init__.py b/invenio/modules/records/recordext/functions/__init__.py
similarity index 96%
rename from invenio/core/record/recordext/__init__.py
rename to invenio/modules/records/recordext/functions/__init__.py
index eb537e874..0eab0f888 100644
--- a/invenio/core/record/recordext/__init__.py
+++ b/invenio/modules/records/recordext/functions/__init__.py
@@ -1,18 +1,18 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
-## Copyright (C) 2013 CERN.
+## Copyright (C) 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
diff --git a/invenio/core/record/recordext/models/common.cfg b/invenio/modules/records/recordext/models/base.cfg
similarity index 84%
rename from invenio/core/record/recordext/models/common.cfg
rename to invenio/modules/records/recordext/models/base.cfg
index 3df2b52be..86379c1e6 100644
--- a/invenio/core/record/recordext/models/common.cfg
+++ b/invenio/modules/records/recordext/models/base.cfg
@@ -1,32 +1,31 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+"""
+ invenio.modules.records.model.record_base
+ -----------------------------------------
+
+ Common fields in all the records inside any Invenio installation.
+"""
+
fields:
recid
_id
modification_date
- _modification_date
creation_date
-documentation:
- """
- Common fields in all the records inside any Invenio installation
-
- See: invenio.core.record.recordext.corefields for a better explanation about
- the purpose of this fields.
- """
diff --git a/invenio/modules/records/testsuite/deprecated_test_bibfield_config_engine.py b/invenio/modules/records/testsuite/deprecated_test_bibfield_config_engine.py
deleted file mode 100644
index 44e02efaf..000000000
--- a/invenio/modules/records/testsuite/deprecated_test_bibfield_config_engine.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# -*- coding: utf-8 -*-
-##
-## This file is part of Invenio.
-## Copyright (C) 2012, 2013 CERN.
-##
-## Invenio is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License as
-## published by the Free Software Foundation; either version 2 of the
-## License, or (at your option) any later version.
-##
-## Invenio is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
-"""
-BibFieldParser Unit tests.
-"""
-#FIXME: Not working now! A new version of the config parser its out there!
-
-from invenio.testsuite import make_test_suite, run_test_suite, InvenioTestCase
-
-
-class BibFieldParserUnitTests(InvenioTestCase):
- """
- Test to verify the correct creation of bibfield_config.py from the rules and
- doctypes files.
- """
- def setUp(self):
- """Loads bibfield configuration test files"""
- super(BibFieldParserUnitTests, self).setUp()
- from invenio.legacy.bibfield.config_engine import BibFieldParser
- parser = BibFieldParser(main_config_file="test_bibfield.cfg")
- self.config_rules = parser.config_rules
-
- def test_bibfield_rules_parser(self):
- """BibField - configuration rules building process"""
- self.assertTrue(len(self.config_rules) >= 20)
- #Check imports
- self.assertTrue('authors' in self.config_rules)
- self.assertTrue('title' in self.config_rules)
- #Check work arroung for [n] and [0]
- self.assertTrue(len(self.config_rules['authors']) == 2)
- self.assertEqual(self.config_rules['authors'], ['authors[0]', 'authors[n]'])
- self.assertTrue('authors[0]' in self.config_rules)
- self.assertTrue('authors[n]' in self.config_rules)
- self.assertTrue(self.config_rules['doi']['persistent_identifier'])
- #Check if derived and calulated are well parserd
- self.assertTrue('dummy' in self.config_rules)
- self.assertTrue(self.config_rules['dummy']['type'] == 'derived')
- self.assertTrue(self.config_rules['dummy']['persistent_identifier'])
- self.assertTrue(self.config_rules['_number_of_copies']['type'] == 'calculated')
- self.assertTrue(self.config_rules['authors[0]']['type'] == 'real')
- self.assertTrue(self.config_rules['_random']['rules']['do_not_cache'])
- self.assertFalse(self.config_rules['_number_of_copies']['rules']['do_not_cache'])
- #Check inheritance
- self.assertTrue('main_author' in self.config_rules)
- self.assertEqual(self.config_rules['main_author']['rules'],
- self.config_rules['authors[0]']['rules'])
-
- def test_bibfield_docytpes_parser(self):
- #TODO: next iteration will come with this
- pass
-
- def test_writing_bibfield_config_file(self):
- #TODO: tests
- pass
-
-
-TEST_SUITE = make_test_suite(BibFieldParserUnitTests)
-
-if __name__ == "__main__":
- run_test_suite(TEST_SUITE)
diff --git a/invenio/modules/records/testsuite/deprecated_test_bibfield_utils.py b/invenio/modules/records/testsuite/deprecated_test_bibfield_utils.py
deleted file mode 100644
index 5268d3da6..000000000
--- a/invenio/modules/records/testsuite/deprecated_test_bibfield_utils.py
+++ /dev/null
@@ -1,190 +0,0 @@
-# -*- coding: utf-8 -*-
-##
-## This file is part of Invenio.
-## Copyright (C) 2004, 2005, 2006, 2007, 2008, 2010, 2011, 2013 CERN.
-##
-## Invenio is free software; you can redistribute it and/or
-## modify it under the terms of the GNU General Public License as
-## published by the Free Software Foundation; either version 2 of the
-## License, or (at your option) any later version.
-##
-## Invenio is distributed in the hope that it will be useful, but
-## WITHOUT ANY WARRANTY; without even the implied warranty of
-## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-## General Public License for more details.
-##
-## You should have received a copy of the GNU General Public License
-## along with Invenio; if not, write to the Free Software Foundation, Inc.,
-## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
-
-"""
-BibFieldUtils Unit tests.
-"""
-
-from invenio.testsuite import make_test_suite, run_test_suite, InvenioTestCase
-
-
-class BibFieldCoolListDictUnitTests(InvenioTestCase):
- """
- Test class to verify the correct behaviour of the classes involved into the
- intermediate structure
- """
-
- def test_cool_list(self):
- """Bibfield Utils, CoolList - Unit tests"""
- from invenio.legacy.bibfield.utils import CoolList
- ll = CoolList()
- ll.append(1)
- ll.append(2)
- ll.append(3)
- self.assertFalse(ll.consumed)
- ll[1]
- self.assertEqual(ll._consumed, [False, True, False])
- self.assertFalse(ll.consumed)
- [i for i in ll]
- self.assertTrue(ll.consumed)
- ll[1] = [4, 5, 6]
- self.assertFalse(ll.consumed)
- self.assertEqual(ll._consumed, [True, [False, False, False], True])
- [i for i in ll]
- self.assertFalse(ll.consumed)
- self.assertEqual(ll._consumed, [True, [False, False, False], True])
- ll[1]
- self.assertFalse(ll.consumed)
- [i for i in ll[1]]
- self.assertTrue(ll.consumed)
-
- def test_cool_dict(self):
- """Bibfield Utils, CoolDict - Unit tests"""
- from invenio.legacy.bibfield.utils import CoolDict, CoolList
- d = CoolDict()
- d['a'] = 1
- d['b'] = 2
- d['c'] = 3
- self.assertFalse(d.consumed)
- d['a']
- self.assertFalse(d.consumed)
- [v for dummy_k, v in d.iteritems()]
- self.assertTrue(d.consumed)
- d['b'] = {'d': 1}
- self.assertFalse(d.consumed)
- d['b']
- self.assertFalse(d.consumed)
- [v for dummy_k, v in d['b'].iteritems()]
- self.assertTrue(d.consumed)
- d.extend('a', 11)
- self.assertFalse(d.consumed)
- self.assertTrue(isinstance(d['a'], CoolList))
- [i for i in d['a']]
- self.assertTrue(d.consumed)
-
- def test_cool_list_and_dict(self):
- """Bibfield Utils, CoolList and CoolDict - Unit tests"""
- from invenio.legacy.bibfield.utils import CoolDict, CoolList
- d = CoolDict()
- l = CoolList()
- d['a'] = l
- self.assertTrue(d.consumed)
- l.append(1)
- l.append(2)
- d['a'] = l
- self.assertFalse(d.consumed)
- d['b'] = CoolList([{'a': 1}, {'a': 2}])
- self.assertFalse(d.consumed)
- [v for dummy_k, v in d.iteritems()]
- self.assertFalse(d.consumed)
- [i for i in d['a']]
- [v for i in d['b'] for dummy_k, v in i.iteritems()]
- self.assertTrue(d.consumed)
-
-
-class BibFieldUtilsUnitTests(InvenioTestCase):
- """
- Test class for bibfield utilities
- """
-
- def test_prepare_field_keys(self):
- """BibField Utils, prepare_field_keys - Unit Test"""
- from invenio.legacy.bibfield.utils import prepare_field_keys
- key = 'authors'
- self.assertEqual(prepare_field_keys(key), ['["authors"]'])
- self.assertEqual(prepare_field_keys(key, write=True), ['["authors"]'])
- key = 'authors[0]'
- self.assertEqual(prepare_field_keys(key), ['["authors"][0]'])
- self.assertEqual(prepare_field_keys(key, True), ['["authors"]', '[0]'])
- key = 'authors[n]'
- self.assertEqual(prepare_field_keys(key), ['["authors"][-1]'])
- self.assertEqual(prepare_field_keys(key, True), ['["authors"]', '[-1]'])
- key = 'authors.ln'
- self.assertEqual(prepare_field_keys(key), ['["authors"]', '["ln"]'])
- self.assertEqual(prepare_field_keys(key, True), ['["authors"]', '["ln"]'])
-
- key = 'a[1].b[0].c.d[n]'
- self.assertEqual(prepare_field_keys(key), ['["a"][1]', '["b"][0]', '["c"]', '["d"][-1]'])
- self.assertEqual(prepare_field_keys(key, True), ['["a"]', '[1]', '["b"]', '[0]', '["c"]', '["d"]', '[-1]'])
-
- def test_build_data_structure(self):
- """BibField Utils, build_data_structure - Unit Test"""
- from invenio.legacy.bibfield.utils import build_data_structure
- d = dict()
- build_data_structure(d, 'authors')
- self.assertEqual(d, {'authors': None})
- build_data_structure(d, 'authors[0]')
- self.assertEqual(d, {'authors': [None]})
- build_data_structure(d, 'authors[n]')
- self.assertEqual(d, {'authors': [None, None]})
-
- d = dict()
- build_data_structure(d, 'a[0].b[n].c.d[n]')
- self.assertEqual(d, {'a': [{'b': [{'c': {'d': [None]}}]}]})
-
-
-class BibFieldDictUnitTest(InvenioTestCase):
- """
- Test class for bibfield base dictionary
- """
-
- def test_bibfielddict(self):
- """BibFieldDict - Unit Test"""
- import random
- from invenio.legacy.bibfield.utils import BibFieldDict
- d = BibFieldDict()
- d['foo'] = {'a': 'world', 'b': 'hello'}
- d['a'] = [{'b': 1}, {'b': 2}, {'b': 3}]
- d['_c'] = 1
- d['_cc'] = random.random()
-
- d['__do_not_cache'].append('_cc')
- d['__calculated_functions']['_c'] = "random.random()"
- d['__calculated_functions']['_cc'] = "random.random()"
- d['__aliases']['aa'] = 'a'
-
- self.assertTrue(len(d.keys()) == 7)
- self.assertTrue('foo' in d)
- self.assertTrue('a.b' in d)
-
- self.assertEqual(d['foo'], {'a': 'world', 'b': 'hello'})
- self.assertEqual(d['a'], d.get('a'))
- self.assertEqual(d['a[-1].b'], 3)
-
- self.assertEqual(d['a'], d['aa'])
- self.assertEqual(d['a[1].b'], d['aa[1].b'])
-
- self.assertEqual(d['_c'], 1)
- self.assertNotEqual(d['_c'], d.get('_c', reset_cache=True))
-
- self.assertNotEqual(d['_cc'], 1)
- self.assertNotEqual(d['_cc'], d.get('_cc'))
-
- #Python 2.5 or higher
- #self.assertEqual('hello world!', d.get('foo', formatstring="{0[b]} {0[a]}!"))
-
- def dummy(s):
- return s.upper()
- self.assertEqual('HELLO', d.get('foo.b', formatfunction=dummy))
-
-
-TEST_SUITE = make_test_suite(BibFieldCoolListDictUnitTests, BibFieldUtilsUnitTests, BibFieldDictUnitTest)
-
-if __name__ == "__main__":
- run_test_suite(TEST_SUITE)
diff --git a/invenio/modules/records/views.py b/invenio/modules/records/views.py
index 33ba35186..285cacfd1 100644
--- a/invenio/modules/records/views.py
+++ b/invenio/modules/records/views.py
@@ -1,232 +1,232 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""WebSearch Flask Blueprint"""
from functools import wraps
from flask import g, render_template, request, flash, redirect, url_for, \
current_app, abort, Blueprint
from flask.ext.login import current_user
from invenio.base.decorators import wash_arguments
from invenio.base.globals import cfg
from invenio.config import CFG_SITE_RECORD
from invenio.ext.template.context_processor import \
register_template_context_processor
from invenio.modules.search.models import Collection
from invenio.modules.search.signals import record_viewed
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.base.i18n import _
from invenio.utils import apache
from flask.ext.breadcrumbs import default_breadcrumb_root
blueprint = Blueprint('record', __name__, url_prefix="/" + CFG_SITE_RECORD,
static_url_path='/record', template_folder='templates',
static_folder='static')
default_breadcrumb_root(blueprint, '.')
def request_record(f):
@wraps(f)
def decorated(recid, *args, **kwargs):
from invenio.modules.access.mailcookie import \
mail_cookie_create_authorize_action
from invenio.modules.access.local_config import VIEWRESTRCOLL
from invenio.legacy.search_engine import guess_primary_collection_of_a_record, \
check_user_can_view_record
from invenio.legacy.websearch.adminlib import get_detailed_page_tabs,\
get_detailed_page_tabs_counts
# ensure recid to be integer
recid = int(recid)
g.collection = collection = Collection.query.filter(
Collection.name == guess_primary_collection_of_a_record(recid)).\
one()
(auth_code, auth_msg) = check_user_can_view_record(current_user, recid)
# only superadmins can use verbose parameter for obtaining debug information
if not current_user.is_super_admin and 'verbose' in kwargs:
kwargs['verbose'] = 0
if auth_code and current_user.is_guest:
cookie = mail_cookie_create_authorize_action(VIEWRESTRCOLL, {
'collection': g.collection.name})
url_args = {'action': cookie, 'ln': g.ln, 'referer': request.url}
flash(_("Authorization failure"), 'error')
return redirect(url_for('webaccount.login', **url_args))
elif auth_code:
flash(auth_msg, 'error')
abort(apache.HTTP_UNAUTHORIZED)
- from invenio.legacy.bibfield import get_record
+ from invenio.modules.records.api import get_record
from invenio.legacy.search_engine import record_exists, get_merged_recid
# check if the current record has been deleted
# and has been merged, case in which the deleted record
# will be redirect to the new one
record_status = record_exists(recid)
merged_recid = get_merged_recid(recid)
if record_status == -1 and merged_recid:
return redirect(url_for('record.metadata', recid=merged_recid))
elif record_status == -1:
abort(apache.HTTP_GONE) # The record is gone!
g.bibrec = Bibrec.query.get(recid)
record = get_record(recid)
if record is None:
return render_template('404.html')
title = record.get('title.title', '')
# b = [(_('Home'), '')] + collection.breadcrumbs()[1:]
# b += [(title, 'record.metadata', dict(recid=recid))]
# current_app.config['breadcrumbs_map'][request.endpoint] = b
g.record_tab_keys = []
tabs = []
counts = get_detailed_page_tabs_counts(recid)
for k, v in get_detailed_page_tabs(collection.id, recid,
g.ln).iteritems():
t = {}
b = 'record'
if k == '':
k = 'metadata'
if k == 'comments' or k == 'reviews':
b = 'comments'
if k == 'linkbacks':
b = 'weblinkback'
k = 'index'
t['key'] = b + '.' + k
t['count'] = counts.get(k.capitalize(), -1)
t.update(v)
tabs.append(t)
if v['visible']:
g.record_tab_keys.append(b+'.'+k)
if cfg.get('CFG_WEBLINKBACK_TRACKBACK_ENABLED'):
@register_template_context_processor
def trackback_context():
from invenio.legacy.weblinkback.templates import get_trackback_auto_discovery_tag
return dict(headerLinkbackTrackbackLink=get_trackback_auto_discovery_tag(recid))
def _format_record(recid, of='hd', user_info=current_user, *args, **kwargs):
from invenio.legacy.search_engine import print_record
return print_record(recid, format=of, user_info=user_info, *args, **kwargs)
@register_template_context_processor
def record_context():
from invenio.modules.comments.api import get_mini_reviews
return dict(recid=recid,
record=record,
tabs=tabs,
title=title,
get_mini_reviews=lambda *args, **kwargs:
get_mini_reviews(*args, **kwargs).decode('utf8'),
collection=collection,
format_record=_format_record
)
return f(recid, *args, **kwargs)
return decorated
@blueprint.route('//metadata', methods=['GET', 'POST'])
@blueprint.route('//', methods=['GET', 'POST'])
@blueprint.route('/', methods=['GET', 'POST'])
@wash_arguments({'of': (unicode, 'hd')})
@request_record
def metadata(recid, of='hd'):
from invenio.legacy.bibrank.downloads_similarity import register_page_view_event
from invenio.modules.formatter import get_output_format_content_type
register_page_view_event(recid, current_user.get_id(), str(request.remote_addr))
if get_output_format_content_type(of) != 'text/html':
return redirect('/%s/%d/export/%s' % (CFG_SITE_RECORD, recid, of))
# Send the signal 'document viewed'
record_viewed.send(
current_app._get_current_object(),
recid=recid,
id_user=current_user.get_id(),
request=request)
return render_template('records/metadata.html', of=of)
@blueprint.route('//references', methods=['GET', 'POST'])
@request_record
def references(recid):
return render_template('records/references.html')
@blueprint.route('//files', methods=['GET', 'POST'])
@request_record
def files(recid):
def get_files():
from invenio.legacy.bibdocfile.api import BibRecDocs
for bibdoc in BibRecDocs(recid).list_bibdocs():
for file in bibdoc.list_all_files():
yield file.get_url()
return render_template('records/files.html', files=list(get_files()))
@blueprint.route('//citations', methods=['GET', 'POST'])
@request_record
def citations(recid):
from invenio.legacy.bibrank.citation_searcher import calculate_cited_by_list,\
get_self_cited_by, calculate_co_cited_with_list
citations = dict(
citinglist=calculate_cited_by_list(recid),
selfcited=get_self_cited_by(recid),
co_cited=calculate_co_cited_with_list(recid)
)
return render_template('records/citations.html',
citations=citations)
@blueprint.route('//keywords', methods=['GET', 'POST'])
@request_record
def keywords(recid):
from invenio.legacy.bibclassify.webinterface import record_get_keywords
found, keywords, record = record_get_keywords(recid)
return render_template('records/keywords.html',
found=found,
keywords=keywords)
@blueprint.route('//usage', methods=['GET', 'POST'])
@request_record
def usage(recid):
from invenio.legacy.bibrank.downloads_similarity import calculate_reading_similarity_list
from invenio.legacy.bibrank.downloads_grapher import create_download_history_graph_and_box
viewsimilarity = calculate_reading_similarity_list(recid, "pageviews")
downloadsimilarity = calculate_reading_similarity_list(recid, "downloads")
downloadgraph = create_download_history_graph_and_box(recid)
return render_template('records/usage.html',
viewsimilarity=viewsimilarity,
downloadsimilarity=downloadsimilarity,
downloadgraph=downloadgraph)
@blueprint.route('/', methods=['GET', 'POST'])
def no_recid():
return redirect("/")
diff --git a/invenio/modules/sword/models.py b/invenio/modules/sword/models.py
index 569bc3117..115a416ab 100644
--- a/invenio/modules/sword/models.py
+++ b/invenio/modules/sword/models.py
@@ -1,96 +1,96 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2011, 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02D111-1307, USA.
"""
bibsword database models.
"""
# General imports.
from invenio.ext.sqlalchemy import db
# Create your models here.
from invenio.modules.accounts.models import User
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
class SwrREMOTESERVER(db.Model):
"""Represents a SwrREMOTESERVER record."""
def __init__(self):
pass
__tablename__ = 'swrREMOTESERVER'
id = db.Column(db.Integer(15, unsigned=True), nullable=False,
primary_key=True, autoincrement=True)
name = db.Column(db.String(50), nullable=False, unique=True)
host = db.Column(db.String(50), nullable=False)
username = db.Column(db.String(50), nullable=False)
password = db.Column(db.String(50), nullable=False)
email = db.Column(db.String(50), nullable=False)
realm = db.Column(db.String(50), nullable=False)
url_base_record = db.Column(db.String(50), nullable=False)
url_servicedocument = db.Column(db.String(80),
nullable=False)
xml_servicedocument = db.Column(db.LargeBinary,
nullable=True)
last_update = db.Column(db.Integer(15, unsigned=True),
nullable=False)
class SwrCLIENTDATA(db.Model):
"""Represents a SwrCLIENTDATA record."""
def __init__(self):
pass
__tablename__ = 'swrCLIENTDATA'
id = db.Column(db.Integer(15, unsigned=True), nullable=False,
primary_key=True,
autoincrement=True)
id_swrREMOTESERVER = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(SwrREMOTESERVER.id),
nullable=False)
id_record = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False)
report_no = db.Column(db.String(50), nullable=False)
id_remote = db.Column(db.String(50), nullable=False)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False)
user_name = db.Column(db.String(100), nullable=False)
user_email = db.Column(db.String(100), nullable=False)
xml_media_deposit = db.Column(db.LargeBinary,
nullable=False)
xml_metadata_submit = db.Column(db.LargeBinary,
nullable=False)
submission_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
publication_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
removal_date = db.Column(db.DateTime, nullable=False,
server_default='1900-01-01 00:00:00')
link_medias = db.Column(db.String(150), nullable=False)
link_metadata = db.Column(db.String(150), nullable=False)
link_status = db.Column(db.String(150), nullable=False)
status = db.Column(db.String(150), nullable=False,
server_default='submitted')
last_update = db.Column(db.DateTime, nullable=False)
remoteserver = db.relationship(SwrREMOTESERVER,
backref='clientdata')
user = db.relationship(User, backref='clientdata')
bibrec = db.relationship(Bibrec)
__all__ = ['SwrREMOTESERVER',
'SwrCLIENTDATA']
diff --git a/invenio/modules/tags/forms.py b/invenio/modules/tags/forms.py
index fc09c8108..856c97658 100644
--- a/invenio/modules/tags/forms.py
+++ b/invenio/modules/tags/forms.py
@@ -1,277 +1,277 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""WebTag Forms"""
from invenio.base.i18n import _
from invenio.base.globals import cfg
from invenio.utils.forms import InvenioBaseForm
from flask.ext.login import current_user
from wtforms import \
IntegerField, \
BooleanField, \
SelectField, \
HiddenField, \
TextField, \
SelectMultipleField, \
SelectField, \
validators
#Models
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.accounts.models import User, Usergroup, UserUsergroup
# Internal
from invenio.ext.sqlalchemy import db
from .models import \
WtgTAG, \
WtgTAGRecord, \
wash_tag_silent, \
wash_tag_blocking
def validate_tag_name(dummy_form, field):
""" Check validity of tag name """
max_len = cfg['CFG_TAGS_NAME_MAX_LENGTH']
max_char = cfg['CFG_TAGS_MAX_CHARACTER']
if field.data:
suggested_silent = wash_tag_silent(field.data)
suggested = wash_tag_blocking(suggested_silent)
field.data = suggested_silent
if suggested != suggested_silent:
raise validators.ValidationError(
_('Forbidden characters. Try ') + suggested + '.')
if len(suggested) <= 0:
raise validators.ValidationError(
_('The name must contain valid characters.'))
if len(suggested_silent) > max_len:
raise validators.ValidationError( _('The name cannot exeed ')
+ str(max_len) + _(' characters.'))
if max(ord(letter) for letter in suggested_silent) > max_char:
raise validators.ValidationError( _('Forbidden character.'))
def validate_name_available(dummy_form, field):
""" Check if the user already has tag named this way """
if field.data:
uid = current_user.get_id()
copy_count = db.session.query(WtgTAG).\
filter_by(id_user=uid, name=field.data).count()
if copy_count > 0:
raise validators.ValidationError(
_('Tag with that name already exists.'))
def validate_tag_exists(dummy_form, field):
""" Check if id_tag matches a tag in database """
if field.data:
try:
field.data = int(field.data)
except ValueError:
raise validators.ValidationError(_('Tag ID must be an integer.'))
if not db.session.query(WtgTAG).get(field.data):
raise validators.ValidationError(_('Tag does not exist.'))
def validate_user_owns_tag(dummy_form, field):
""" Check if id_tag matches a tag in database """
if field.data:
tag = db.session.query(WtgTAG).get(field.data)
if tag and tag.id_user != current_user.get_id():
raise validators.ValidationError(
_('You are not the owner of this tag.'))
def validate_bibrec_exists(dummy_form, field):
""" Check if id_bibrec matches a bibrec in database """
if field.data:
try:
field.data = int(field.data)
except ValueError:
raise validators.ValidationError(_('Bibrec ID must be an integer.'))
record = db.session.query(Bibrec).get(field.data)
if (not record):
raise validators.ValidationError(_('Bibrec does not exist.'))
# Switch to merged record if present
merged_id = record.merged_recid_final
if merged_id != record.id:
record = db.session.query(Bibrec).get(merged_id)
field.data = merged_id
if record.deleted:
raise validators.ValidationError(_('Bibrec has been deleted.'))
def validate_user_can_see_bibrec(dummy_form, field):
""" Check if user has rights to view bibrec """
if field.data:
from invenio.legacy.search_engine import check_user_can_view_record
(auth_code, msg) = check_user_can_view_record(current_user, field.data)
if auth_code > 0:
raise validators.ValidationError(
_('Unauthorized to view record: ')+msg)
def validate_not_already_attached(form, dummy_field):
""" Check if the pair (tag, bibrec) is already connected """
if form:
if ('id_tag' in form.data) and ('id_bibrec' in form.data):
tag_record = db.session.query(WtgTAGRecord)\
.get((form.data['id_tag'], form.data['id_bibrec']))
if tag_record is not None:
raise validators.ValidationError(_('Tag already attached.'))
def validate_already_attached(form, dummy_field):
""" Check if the pair (tag, bibrec) is already connected """
if form:
if ('id_tag' in form.data) and ('id_bibrec' in form.data):
tag_record = db.session.query(WtgTAGRecord)\
.get((form.data['id_tag'], form.data['id_bibrec']))
if tag_record is None:
raise validators.ValidationError(_('Tag not attached.'))
class CreateTagForm(InvenioBaseForm):
"""Defines form for creating a new tag."""
name = TextField(_('Name'), [validators.Required(),
validate_tag_name,
validate_name_available])
# Ajax requests only:
# Send a record ID if the tag should be attached to the record
# right after creation
id_bibrec = HiddenField('Tagged record',
[validate_bibrec_exists,
validate_user_can_see_bibrec])
class DeleteTagForm(InvenioBaseForm):
"""Defines form for deleting a tag."""
id_tag = SelectMultipleField('Tag ID',
[validators.Required(),
validate_tag_exists,
validate_user_owns_tag])
class AttachTagForm(InvenioBaseForm):
"""Defines a form validating attaching a tag to record"""
# Ajax requests only:
id_tag = IntegerField('Tag ID',
[validators.Required(),
validate_tag_exists,
validate_not_already_attached,
validate_user_owns_tag])
# validate user rights on tag
id_bibrec = IntegerField('Record ID',
[validate_bibrec_exists,
validate_user_can_see_bibrec])
class DetachTagForm(InvenioBaseForm):
"""Defines a form validating detaching a tag from record"""
# Ajax requests only:
id_tag = IntegerField('Tag ID',
[validators.Required(),
validate_tag_exists,
validate_already_attached,
validate_user_owns_tag])
# validate user rights on tag
id_bibrec = IntegerField('Record ID',
[validators.Required(),
validate_bibrec_exists,
validate_user_can_see_bibrec])
class TagAnnotationForm(InvenioBaseForm):
"""Defines a form validating attaching a tag to record"""
# Ajax requests only:
id_tag = IntegerField('Tag ID',
[validators.Required(),
validate_tag_exists,
validate_already_attached,
validate_user_owns_tag])
# validate user rights on tag
id_bibrec = IntegerField('Record ID',
[validate_bibrec_exists,
validate_user_can_see_bibrec])
annotation_value = TextField('Annotation')
class GetGroupOptions(object):
def __iter__(self):
id_user = current_user.get_id()
options = [('0', _('Private'))]
options += db.session.query(Usergroup.id, Usergroup.name)\
.join(UserUsergroup)\
.filter(UserUsergroup.id_user == id_user)\
.all()
for (gid, name) in options:
yield (str(gid), name)
class EditTagForm(InvenioBaseForm):
"""Defines form for editing an existing tag."""
name = TextField(_('Name'), [validators.Required(),
validate_tag_name])
id_usergroup = SelectField(
_('Group sharing options'),
choices=GetGroupOptions())
group_access_rights = SelectField(
_('Group access rights'),
choices=[
(str(WtgTAG.ACCESS_LEVELS['View']), 'View'),
(str(WtgTAG.ACCESS_LEVELS['Add and remove']), 'Attach to documents')
])
class WebTagUserSettingsForm(InvenioBaseForm):
"""User's personal settings influencing WebTag module"""
display_tags = BooleanField(_('Display tags with records'))
display_tags_group = BooleanField(_('Show group tags'))
display_tags_public = BooleanField(_('Show public tags'))
diff --git a/invenio/modules/tags/models.py b/invenio/modules/tags/models.py
index 682ab68f5..8104b4f28 100644
--- a/invenio/modules/tags/models.py
+++ b/invenio/modules/tags/models.py
@@ -1,346 +1,346 @@
# -*- coding: utf-8 -*-
#
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
""" WebTag database models. """
# Database
from invenio.ext.sqlalchemy import db
from sqlalchemy.ext.associationproxy import association_proxy
# Related models
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.accounts.models import User, Usergroup
# Functions
from invenio.base.globals import cfg
from werkzeug import cached_property
from invenio.utils.text import wash_for_xml
from datetime import datetime, date
import re
class Serializable(object):
"""Class which can present its fields as dict for json serialization"""
# Set of fields which are to be serialized
__public__ = set([])
def _serialize_field(self, value):
""" Converts value of a field to format suitable for json """
if type(value) in (datetime, date):
return value.isoformat()
elif hasattr(value, '__iter__'):
result = []
for element in value:
result.append(self._serialize_field(element))
return result
elif Serializable in value.__class__.__bases__:
return value.get_public()
else:
return value
def serializable_fields(self, fields=None):
"""Returns model's fields (__public__) or
intersection(fields, __public__) for jsonify"""
data = {}
keys = self._sa_instance_state.attrs.items()
public = set()
if self.__public__:
public = public.union(self.__public__)
if fields:
public = public.intersection(fields)
for key, field in keys:
if key in public:
value = self._serialize_field(field.value)
if value:
data[key] = value
return data
#
# TAG
#
class WtgTAG(db.Model, Serializable):
""" Represents a Tag """
__tablename__ = 'wtgTAG'
__public__ = set(['id', 'name', 'id_owner'])
#
# Access Rights
#
ACCESS_NAMES = {
0: 'Nothing',
10: 'View',
20: 'Add',
30: 'Add and remove',
40: 'Manage',
}
ACCESS_LEVELS = \
dict((v, k) for (k, v) in ACCESS_NAMES.iteritems())
ACCESS_RIGHTS = {
0: [],
10: ['view'],
20: ['view', 'add'],
30: ['view', 'add', 'remove'],
40: ['view', 'add', 'remove', 'edit'],
}
ACCESS_OWNER_DEFAULT = ACCESS_LEVELS['Manage']
ACCESS_GROUP_DEFAULT = ACCESS_LEVELS['View']
# Primary key
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True,
nullable=False,
autoincrement=True)
# Name
name = db.Column(db.String(255),
nullable=False,
server_default='',
index=True)
# Owner
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
server_default='0')
# Access rights of owner
user_access_rights = db.Column(db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_OWNER_DEFAULT)
# Group
# equal to 0 for private tags
id_usergroup = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey(Usergroup.id),
server_default='0')
# Group access rights
group_access_rights = db.Column(
db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_GROUP_DEFAULT)
# Access rights of everyone
public_access_rights = db.Column(db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_LEVELS['Nothing'])
# Visibility in document description
show_in_description = db.Column(db.Boolean,
nullable=False,
default=True)
# Relationships
user = db.relationship(User,
backref=db.backref('tags', cascade='all'))
user_query = db.relationship(User,
backref=db.backref('tags_query',
cascade='all',
lazy='dynamic'))
usergroup = db.relationship(
Usergroup,
backref=db.backref('tags', cascade='all'))
# association proxy of "user_keywords" collection
# to "keyword" attribute
records = association_proxy('records_association', 'bibrec')
#Calculated fields
@db.hybrid_property
def record_count(self):
return self.records_association_query.count()
@record_count.expression
def record_count(cls):
return db.select([db.func.count(WtgTAGRecord.id_bibrec)]).\
where(WtgTAGRecord.id_tag == cls.id).\
label('record_count')
@db.validates('user_access_rights')
@db.validates('group_access_rights')
@db.validates('public_access_rights')
def validate_user_access_rights(self, key, value):
""" Check if the value is among defined levels """
assert value in WtgTAG.ACCESS_NAMES
return value
#
# TAG - RECORD
#
class WtgTAGRecord(db.Model, Serializable):
""" Represents a connection between Tag and Record """
__tablename__ = 'wtgTAG_bibrec'
__public__ = set(['id_tag', 'id_bibrec', 'date_added'])
# tagTAG.id
id_tag = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(WtgTAG.id),
nullable=False,
primary_key=True)
# Bibrec.id
id_bibrec = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False,
primary_key=True)
# Annotation
annotation = db.Column(
db.Text(convert_unicode=True),
default='')
# Creation date
date_added = db.Column(db.DateTime,
default=datetime.now)
# Relationships
tag = db.relationship(WtgTAG,
backref=db.backref('records_association',
cascade='all'))
tag_query = db.relationship(WtgTAG,
backref=db.backref('records_association_query',
cascade='all',
lazy='dynamic'))
bibrec = db.relationship(Bibrec,
backref=db.backref('tags_association',
cascade='all'))
bibrec_query = db.relationship(Bibrec,
backref=db.backref('tags_association_query',
cascade='all',
lazy='dynamic'))
# Constructor
def __init__(self, bibrec=None, **kwargs):
super(WtgTAGRecord, self).__init__(**kwargs)
self.bibrec = bibrec
# Compiling once should improve regexp speed
class ReplacementList(object):
def __init__(self, config_name):
self.config_name = config_name
@cached_property
def replacements(self):
return cfg.get(self.config_name, [])
@cached_property
def compiled(self):
return [(re.compile(exp), repl)
for (exp, repl) in self.replacements]
def apply(self, text):
"""Applies a list of regular expression replacements to a string.
:param replacements: list of pairs (compiled_expression, replacement)
"""
for (reg_exp, replacement) in self.compiled:
text = re.sub(reg_exp, replacement, text)
return text
COMPILED_REPLACEMENTS_SILENT = \
ReplacementList('CFG_TAGS_NAME_REPLACEMENTS_SILENT')
COMPILED_REPLACEMENTS_BLOCKING = \
ReplacementList('CFG_TAGS_NAME_REPLACEMENTS_BLOCKING')
def wash_tag_silent(tag_name):
"""
Whitespace and character cleanup.
:param tag_name: Single tag.
:return: Tag Unicode string with all whitespace characters replaced with
Unicode single space (' '), no whitespace at the start and end of the tags,
no duplicate whitespace, and only characters valid in XML 1.0.
Also applies list of replacements from CFG_WEBTAG_REPLACEMENTS_SILENT.
Examples:
>>> print(_tag_cleanup('Well formatted string: Should not be changed'))
Well formatted string: Should not be changed
>>> print(_tag_cleanup('double space characters'))
double space characters
>>> print(_tag_cleanup('All\\tthe\\ndifferent\\x0bwhitespace\\x0cin\\rone go'))
All the different whitespace in one go
>>> print(_tag_cleanup(' Preceding whitespace'))
Preceding whitespace
>>> print(_tag_cleanup('Trailing whitespace '))
Trailing whitespace
>>> print(_tag_cleanup(' Preceding and trailing double whitespace '))
Preceding and trailing double whitespace
>>> _tag_cleanup(unichr(CFG_WEBTAG_LAST_MYSQL_CHARACTER))
u''
>>> from string import whitespace
>>> _tag_cleanup(whitespace)
''
"""
if tag_name is None:
return None
# convert to string
if type(tag_name) == unicode:
tag_name = tag_name.encode('utf-8')
# wash_for_xml
tag_name = wash_for_xml(tag_name)
# replacements
tag_name = COMPILED_REPLACEMENTS_SILENT.apply(tag_name)
return tag_name
def wash_tag_blocking(tag_name):
""" Applies list of replacements from CFG_WEBTAG_REPLACEMENTS_BLOCKING """
if tag_name is None:
return None
# replacements
tag_name = COMPILED_REPLACEMENTS_BLOCKING.apply(tag_name)
return tag_name
def wash_tag(tag_name):
""" Applies all washing procedures in order """
return wash_tag_blocking(wash_tag_silent(tag_name))
diff --git a/invenio/modules/tags/user_settings.py b/invenio/modules/tags/user_settings.py
index 63c2b689e..d983e0912 100644
--- a/invenio/modules/tags/user_settings.py
+++ b/invenio/modules/tags/user_settings.py
@@ -1,80 +1,80 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""WebTag User Settings"""
# Flask
from flask import url_for
from invenio.ext.template import render_template_to_string
from invenio.base.i18n import _
from flask.ext.login import current_user
from invenio.modules.dashboard.settings import \
Settings, \
UserSettingsAttributeStorage
# Related models
from invenio.modules.accounts.models import User
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
# Internal
from .models import WtgTAG, WtgTAGRecord
from .forms import WebTagUserSettingsForm
class WebTagSettings(Settings):
keys = \
[
'display_tags',
'display_tags_group',
'display_tags_public',
]
form_builder = WebTagUserSettingsForm
storage_builder = UserSettingsAttributeStorage('webtag')
def __init__(self):
super(WebTagSettings, self).__init__()
self.icon = 'tags'
self.title = _('Tags')
self.view = url_for('webtag.display_cloud')
self.edit = url_for('webaccount.edit', name=self.name)
def widget(self):
user = User.query.get(current_user.get_id())
tag_count = user.tags_query.count()
record_count = Bibrec.query.join(WtgTAGRecord)\
.join(WtgTAG)\
.filter(WtgTAG.user == user).count()
return render_template_to_string(
'tags/user_settings.html',
tag_count=tag_count,
record_count=record_count)
widget.size = 4
@property
def is_authorized(self):
return current_user.is_authenticated()
# and current_user.is_authorized('usebaskets')
## Compulsory plugin interface
settings = WebTagSettings
diff --git a/invenio/modules/tags/views.py b/invenio/modules/tags/views.py
index df2d5bb6a..93026d978 100644
--- a/invenio/modules/tags/views.py
+++ b/invenio/modules/tags/views.py
@@ -1,478 +1,478 @@
# -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
invenio.modules.tags.blueprint
------------------------------
Tagging interface.
"""
# Flask
from werkzeug import LocalProxy
from flask import render_template, request, flash, redirect, url_for, \
jsonify, Blueprint
from invenio.base.i18n import _
from invenio.base.decorators import wash_arguments, templated
from flask.ext.login import current_user, login_required
from invenio.base.globals import cfg
# External imports
from invenio.modules.accounts.models import User
-from invenio.modules.record_editor.models import Bibrec
+from invenio.modules.records.models import Record as Bibrec
from invenio.modules.search.models import Collection
from invenio.modules.search.views.search import response_formated_records
from flask.ext.menu import register_menu
from flask.ext.breadcrumbs import default_breadcrumb_root, register_breadcrumb
from invenio.ext.sqlalchemy import db
# Internal imports
from .models import \
WtgTAG, \
WtgTAGRecord, \
wash_tag
# Forms
from .forms import \
CreateTagForm, \
AttachTagForm, \
DetachTagForm, \
EditTagForm, \
TagAnnotationForm, \
validate_tag_exists, \
validate_user_owns_tag, \
validators
# Uset settings
user_settings = LocalProxy(lambda:
current_user['settings'].get('webtag', cfg['CFG_WEBTAG_DEFAULT_USER_SETTINGS']))
blueprint = Blueprint('webtag', __name__, url_prefix='/yourtags',
template_folder='templates', static_folder='static')
default_breadcrumb_root(blueprint, '.webaccount.tags')
@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/display', methods=['GET', 'POST'])
@blueprint.route('/display/cloud', methods=['GET', 'POST'])
@login_required
@templated('tags/display_cloud.html')
@register_menu(blueprint, 'personalize.tags', _('Your Tags'))
@register_breadcrumb(blueprint, '.', _('Your Tags'))
def display_cloud():
""" List of user's private/group/public tags """
user = User.query.get(current_user.get_id())
tags = user.tags_query.order_by(WtgTAG.name).all()
# Calculate document count for each tag
min_count = 0
max_count = 0
for tag in tags:
if tag.record_count > max_count:
max_count = tag.record_count
if tag.record_count < min_count:
min_count = tag.record_count
difference = float(max_count - min_count)
if not difference:
difference = 1.0
# Assign sizes
min_size = 1.0
max_size = 2.0
for tag in tags:
size = min_size + \
float(max_size - min_size) * \
float(tag.record_count - min_count) / difference
tag.css_size = str(size*100)
return dict(user_tags=tags,
display_mode='cloud')
@blueprint.route('/display/list', methods=['GET', 'POST'])
@login_required
@templated('tags/display_list.html')
@wash_arguments({'sort_by': (unicode, 'name'),
'order': (unicode, '')})
@register_breadcrumb(blueprint, '.list', _('Display as List'))
def display_list(sort_by, order):
""" List of user's private/group/public tags """
tags = User.query.get(current_user.get_id()).tags_query
sort_by = str(sort_by)
order = str(order)
if sort_by == 'record_count':
tags = tags.order_by(WtgTAG.record_count)
else:
tags = tags.order_by(WtgTAG.name)
tags = tags.all()
if order == 'desc':
tags.reverse()
return dict(user_tags=tags,
display_mode='list')
@blueprint.route('/tag//records', methods=['GET', 'POST'])
@login_required
@register_breadcrumb(blueprint, '.tags_details', _('Associated Records'))
def tag_details(id_tag):
""" List of documents attached to this tag """
if not id_tag:
flash(_('Invalid tag id'), "error")
return redirect(url_for('.display_cloud'))
tag = WtgTAG.query.get(id_tag)
if not tag:
flash(_('Invalid tag id'), "error")
return redirect(url_for('.display_cloud'))
if tag.id_user != current_user.get_id():
flash(_('You are not authorized to view this tag'), "error")
return redirect(url_for('.display_cloud'))
if not tag.records:
flash(_('There are no documents tagged with ') + tag.name)
return redirect(url_for('.display_cloud'))
return response_formated_records([bibrec.id for bibrec in tag.records],
Collection.query.get(1),
'hb')
@blueprint.route('/tag//edit', methods=['GET', 'POST'])
@login_required
@templated('tags/record_tags_test.html') # FIXME: tag editor template missing?
@register_breadcrumb(blueprint, '.tag_edit', _('Edit tag'))
def tag_edit(id_tag):
""" List of documents attached to this tag """
id_user = current_user.get_id()
tag = WtgTAG.query.get(id_tag)
if not tag:
flash(_('Invalid tag id'), "error")
return redirect(url_for('.display_cloud'))
if tag.id_user != id_user:
flash(_('You are not authorized to view this tag'), "error")
return redirect(url_for('.display_cloud'))
form = EditTagForm(request.values, csrf_enabled=False, obj=tag)
if form.validate_on_submit():
form.populate_obj(tag)
name_count = db.session.query(WtgTAG).\
filter_by(id_user=id_user, name=tag.name).count()
if name_count == 1:
db.session.add(tag)
db.session.commit()
flash(_('Tag Successfully edited.'), 'success')
else:
flash(_('Tag name') + ' ' + tag.name + ' ' + _('is already in use.'), 'error')
return dict(tag=tag, form=form)
@blueprint.route('/tag//annotations/', methods=['GET', 'POST'])
@login_required
def update_annotation(id_tag, id_bibrec):
""" Change the annotation on relationship between record and tag """
from werkzeug.datastructures import MultiDict
values = MultiDict(request.values)
values['id_tag'] = id_tag
values['id_bibrec'] = id_bibrec
form = TagAnnotationForm(values, csrf_enabled=False)
if form.validate():
relation = db.session.query(WtgTAGRecord)\
.filter(WtgTAGRecord.id_tag == id_tag)\
.filter(WtgTAGRecord.id_bibrec == id_bibrec)\
.one()
relation.annotation = form.data.get('annotation_value', '')
db.session.add(relation)
db.session.commit()
return relation.annotation
else:
return str(form.errors)
@blueprint.route('/record//tags', methods=['GET', 'POST'])
@login_required
@templated('tags/record_tags_test.html')
@register_breadcrumb(blueprint, '.tags_details', _('Tag list'))
def record_tags(id_bibrec):
""" List of documents attached to this tag """
from .template_context_functions.tfn_webtag_record_tags \
import template_context_function
from invenio.ext.template import render_template_to_string
return dict(
tag_list = template_context_function(id_bibrec, current_user.get_id()))
@blueprint.route('/tokenize/', methods=['GET', 'POST'])
@login_required
@wash_arguments({'q': (unicode, '')})
def tokenize(id_bibrec, q):
""" Data for tokeninput """
id_user = current_user.get_id()
# Output only tags unattached to this record
record = Bibrec.query.get(id_bibrec)
tags = WtgTAG.query\
.filter_by(id_user=id_user)\
.filter(WtgTAG.name.like('%' + q + '%'))\
.filter(db.not_(WtgTAG.records.contains(record)))\
.order_by(WtgTAG.name)
# If a tag with searched name does not exist, lets suggest creating it
# Clean the name
new_name = wash_tag(q)
add_new_name = True
response_tags = []
for tag in tags.all():
tag_json = tag.serializable_fields(set(['id', 'name']))
response_tags.append(tag_json)
# Check if it matches the search name
if tag_json['name'] == new_name:
add_new_name = False
#If the name was not found
if add_new_name:
# Check if a tag with this name is already attached
already_attached = WtgTAG.query\
.join(WtgTAGRecord)\
.filter(WtgTAG.name == new_name)\
.filter(WtgTAGRecord.id_bibrec == id_bibrec)\
.count()
if not already_attached:
tag_json = {'id': 0, 'name': new_name}
response_tags.append(tag_json)
return jsonify(dict(results=response_tags, query=q))
@blueprint.route('/record//edit', methods=['GET', 'POST'])
@login_required
def editor(id_bibrec):
"""Edits your tags for `id_bibrec`.
:param id_bibrec: record identifier"""
user = db.session.query(User).get(current_user.get_id())
record = db.session.query(Bibrec).get(id_bibrec)
tags = db.session.query(WtgTAG)\
.filter_by(user=user)\
.filter(WtgTAG.records.contains(record))
tags_json = []
for tag in tags.all():
fields = tag.serializable_fields(set(['id', 'name']))
fields['can_remove'] = True
tags_json.append(fields)
# invenio_templated cannot be used,
# because this view is requested using AJAX
return render_template('tags/record_editor.html', id_bibrec=id_bibrec,
record_tags=tags_json)
#Temporary solution to call validators, we need a better one
class Field(object):
def __init__(self, attr, value):
setattr(self, attr, value)
@blueprint.route('/delete', methods=['GET', 'POST'])
@login_required
def delete():
""" Delete a tag """
response = {}
response['action'] = 'delete'
id_tags = request.values.getlist('id_tag', type=int)
# Validate
for id_tag in id_tags:
try:
field = Field('data', id_tag)
validate_tag_exists(None, field)
validate_user_owns_tag(None, field)
except validators.ValidationError, ex:
flash(ex.message, 'error')
for id_tag in id_tags:
tag = WtgTAG.query.get(id_tag)
db.session.delete(tag)
db.session.commit()
#WtgTAG.query\
# .filter(WtgTAG.id.in_(id_tags))\
# .delete(synchronize_session=False)
flash(_('Successfully deleted tags.'), 'success')
return redirect(url_for('.display_list'))
# AJAX
# reposnse template:
# action = name of action 'create' 'attach' 'detach'
# success = True / False
#
# if success:
# id_tag = id of tag participating in process
# id_bibrec = id of bibrec (if present in action)
# items = created or deleted model objects
#
# else:
# errors = dict of errors from form
@blueprint.route('/create', methods=['GET', 'POST'])
@login_required
@register_breadcrumb(blueprint, '.create', _('New tag'))
@templated('tags/create.html')
def create():
""" Create a new tag """
response = {}
response['action'] = 'create'
user = User.query.get(current_user.get_id())
form = CreateTagForm(request.values, csrf_enabled=False)
if form.validate_on_submit() or\
(request.is_xhr and form.validate()):
new_tag = WtgTAG()
form.populate_obj(new_tag)
new_tag.user = user
db.session.add(new_tag)
db.session.flush()
db.session.refresh(new_tag)
if 'id_bibrec' in form.data and form.data['id_bibrec']:
record = db.session.query(Bibrec).get(form.data['id_bibrec'])
new_tag.records.append(record)
db.session.add(new_tag)
response['id_bibrec'] = form.data['id_bibrec']
db.session.commit()
db.session.refresh(new_tag)
response['success'] = True
response['id_tag'] = new_tag.id
response['items'] = [new_tag.serializable_fields()]
if request.is_xhr:
return jsonify(response)
else:
return redirect(url_for('.display_list'))
else:
if request.is_xhr:
response['success'] = False
response['errors'] = form.errors
return jsonify(response)
else:
return dict(form=form)
@blueprint.route('/attach', methods=['GET', 'POST'])
@login_required
def attach():
""" Attach a tag to a record """
response = {}
response['action'] = 'attach'
# Ajax - disable csrf
form = AttachTagForm(request.values, csrf_enabled=False)
if form.validate():
association = WtgTAGRecord()
form.populate_obj(association)
db.session.add(association)
db.session.commit()
response['success'] = True
response['id_tag'] = association.id_tag
response['id_bibrec'] = association.id_bibrec
response['items'] = [association.serializable_fields()]
else:
response['success'] = False
response['errors'] = form.errors
return jsonify(response)
@blueprint.route('/detach', methods=['GET', 'POST'])
@login_required
def detach():
""" Detach a tag from a record """
response = {}
response['action'] = 'detach'
# Ajax - disable csrf
form = DetachTagForm(request.values, csrf_enabled=False)
if form.validate():
association = db.session.query(WtgTAGRecord)\
.filter_by(id_tag = form.data['id_tag'],
id_bibrec = form.data['id_bibrec']).first()
if association:
db.session.delete(association)
db.session.commit()
response['success'] = True
response['id_tag'] = association.id_tag
response['id_bibrec'] = association.id_bibrec
response['items'] = [association.serializable_fields()]
else:
response['success'] = False
response['errors'] = form.errors
return jsonify(response)
diff --git a/invenio/modules/workflows/models.py b/invenio/modules/workflows/models.py
index 273479283..2cff16bd3 100644
--- a/invenio/modules/workflows/models.py
+++ b/invenio/modules/workflows/models.py
@@ -1,559 +1,558 @@
# -*- coding: utf-8 -*-
## This file is part of Invenio.
## Copyright (C) 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
import os
import tempfile
import cPickle
import base64
import logging
import six
from datetime import datetime
from sqlalchemy import desc
from sqlalchemy.orm.exc import NoResultFound
from invenio.ext.sqlalchemy import db
from invenio.base.globals import cfg
from .config import CFG_OBJECT_VERSION
from .utils import redis_create_search_entry
from .logger import (get_logger,
BibWorkflowLogHandler)
def get_default_data():
""" Returns the base64 representation of the data default value """
data_default = {}
return base64.b64encode(cPickle.dumps(data_default))
def get_default_extra_data():
""" Returns the base64 representation of the extra_data default value """
extra_data_default = {"tasks_results": {},
"owner": {},
"task_counter": {},
"error_msg": "",
"last_task_name": "",
"latest_object": -1,
"widget": None,
"redis_search": {}}
return base64.b64encode(cPickle.dumps(extra_data_default))
class Workflow(db.Model):
__tablename__ = "bwlWORKFLOW"
uuid = db.Column(db.String(36), primary_key=True, nullable=False)
name = db.Column(db.String(255), default="Default workflow",
nullable=False)
created = db.Column(db.DateTime, default=datetime.now, nullable=False)
modified = db.Column(db.DateTime, default=datetime.now,
onupdate=datetime.now, nullable=False)
id_user = db.Column(db.Integer, default=0, nullable=False)
_extra_data = db.Column(db.LargeBinary, nullable=False, default=get_default_extra_data())
status = db.Column(db.Integer, default=0, nullable=False)
current_object = db.Column(db.Integer, default="0", nullable=False)
objects = db.relationship("BibWorkflowObject", backref="bwlWORKFLOW")
counter_initial = db.Column(db.Integer, default=0, nullable=False)
counter_halted = db.Column(db.Integer, default=0, nullable=False)
counter_error = db.Column(db.Integer, default=0, nullable=False)
counter_finished = db.Column(db.Integer, default=0, nullable=False)
module_name = db.Column(db.String(64), nullable=False)
def __repr__(self):
return "" % \
(str(self.name),
str(self.module_name),
str(self.created),
str(self.modified),
str(self.id_user),
str(self.status))
def __str__(self):
return """Workflow:
Uuid: %s
Name: %s
User id: %s
Module name: %s
Created: %s
Modified: %s
Status: %s
Current object: %s
Counters: initial=%s, halted=%s, error=%s, finished=%s
Extra data: %s""" % (str(self.uuid),
str(self.name),
str(self.id_user),
str(self.module_name),
str(self.created),
str(self.modified),
str(self.status),
str(self.current_object),
str(self.counter_initial),
str(self.counter_halted),
str(self.counter_error),
str(self.counter_finished),
str(self._extra_data),)
@classmethod
def get(cls, *criteria, **filters):
""" A wrapper for the filter and filter_by functions of sqlalchemy.
Define a dict with which columns should be filtered by which values.
e.g. Workflow.get(uuid=uuid)
Workflow.get(Workflow.uuid != uuid)
The function supports also "hybrid" arguments.
e.g. Workflow.get(Workflow.module_name != 'i_hate_this_module',
user_id=user_id)
look up also sqalchemy BaseQuery's filter and filter_by documentation
"""
return cls.query.filter(*criteria).filter_by(**filters)
@classmethod
def get_status(cls, uuid=None):
""" Returns the status of the workflow """
return cls.get(Workflow.uuid == uuid).one().status
@classmethod
def get_most_recent(cls, *criteria, **filters):
""" Returns the most recently modified workflow. """
most_recent = cls.get(*criteria, **filters).\
order_by(desc(Workflow.modified)).first()
if most_recent is None:
raise NoResultFound
else:
return most_recent
@classmethod
def get_objects(cls, uuid=None):
""" Returns the objects of the workflow """
return cls.get(Workflow.uuid == uuid).one().objects
def get_extra_data(self, user_id=0, uuid=None, key=None, getter=None):
"""Returns a json of the column extra_data or
if any of the other arguments are defined,
a specific value.
You can define either the key or the getter function.
@param key: the key to access the desirable value
@param getter: a callable that takes a dict as param and returns a
value
"""
extra_data = Workflow.get(Workflow.id_user == self.id_user,
Workflow.uuid == self.uuid).one()._extra_data
extra_data = cPickle.loads(base64.b64decode(extra_data))
if key is not None:
return extra_data[key]
elif callable(getter):
return getter(extra_data)
def set_extra_data(self, user_id=0, uuid=None,
key=None, value=None, setter=None):
"""Modifies the json of the column extra_data or
if any of the other arguments are defined,
a specific value.
You can define either the key, value or the setter function.
@param key: the key to access the desirable value
@param value: the new value
@param setter: a callable that takes a dict as param and modifies it
"""
extra_data = Workflow.get(Workflow.id_user == user_id,
Workflow.uuid == uuid).one()._extra_data
extra_data = cPickle.loads(base64.b64decode(extra_data))
if key is not None and value is not None:
extra_data[key] = value
elif callable(setter):
setter(extra_data)
Workflow.get(Workflow.uuid == self.uuid).update({'_extra_data': base64.b64encode(cPickle.dumps(extra_data))})
@classmethod
def delete(cls, uuid=None):
cls.get(Workflow.uuid == uuid).delete()
db.session.commit()
class BibWorkflowObject(db.Model):
# db table definition
__tablename__ = "bwlOBJECT"
id = db.Column(db.Integer, primary_key=True)
# Our internal data column. Default is encoded dict.
_data = db.Column(db.LargeBinary, nullable=False, default=get_default_data())
_extra_data = db.Column(db.LargeBinary, nullable=False, default=get_default_extra_data())
id_workflow = db.Column(db.String(36),
db.ForeignKey("bwlWORKFLOW.uuid"), nullable=True)
version = db.Column(db.Integer(3),
default=CFG_OBJECT_VERSION.RUNNING, nullable=False)
id_parent = db.Column(db.Integer, db.ForeignKey("bwlOBJECT.id"),
default=None)
child_objects = db.relationship("BibWorkflowObject",
remote_side=[id_parent])
created = db.Column(db.DateTime, default=datetime.now, nullable=False)
modified = db.Column(db.DateTime, default=datetime.now,
onupdate=datetime.now, nullable=False)
status = db.Column(db.String(255), default="", nullable=False)
data_type = db.Column(db.String(150), default="",
nullable=True)
uri = db.Column(db.String(500), default="")
id_user = db.Column(db.Integer, default=0, nullable=False)
child_logs = db.relationship("BibWorkflowObjectLog")
workflow = db.relationship(
Workflow, foreign_keys=[id_workflow], remote_side=Workflow.uuid
)
_log = None
@property
def log(self):
if not self._log:
db_handler_obj = BibWorkflowLogHandler(BibWorkflowObjectLog, "id")
self._log = get_logger(logger_name="object.%s_%s" % (self.id_workflow, self.id),
db_handler_obj=db_handler_obj,
loglevel=logging.DEBUG,
obj=self)
return self._log
def get_data(self):
"""
Main method to retrieve data saved to the object.
"""
return cPickle.loads(base64.b64decode(self._data))
def set_data(self, value):
"""
Main method to update data saved to the object.
"""
self._data = base64.b64encode(cPickle.dumps(value))
def get_extra_data(self):
"""
Main method to retrieve data saved to the object.
"""
return cPickle.loads(base64.b64decode(self._extra_data))
def set_extra_data(self, value):
"""
Main method to update data saved to the object.
"""
self._extra_data = base64.b64encode(cPickle.dumps(value))
def _create_db_obj(self):
db.session.add(self)
db.session.commit()
def __repr__(self):
return "" % \
(self.id, self.id_object, self.message, self.created)
@classmethod
def get(cls, *criteria, **filters):
""" A wrapper for the filter and filter_by functions of sqlalchemy.
Define a dict with which columns should be filtered by which values.
look up also sqalchemy BaseQuery's filter and filter_by documentation
"""
return cls.query.filter(*criteria).filter_by(**filters)
@classmethod
def get_most_recent(cls, *criteria, **filters):
""" Returns the most recently created log. """
most_recent = cls.get(*criteria, **filters).\
order_by(desc(BibWorkflowObjectLog.created)).first()
if most_recent is None:
raise NoResultFound
else:
return most_recent
@classmethod
def delete(cls, id=None):
cls.get(BibWorkflowObjectLog.id == id).delete()
db.session.commit()
class BibWorkflowEngineLog(db.Model):
__tablename__ = "bwlWORKFLOWLOGGING"
id = db.Column(db.Integer, primary_key=True)
id_object = db.Column(db.String(255), nullable=False)
log_type = db.Column(db.Integer, default=0, nullable=False)
created = db.Column(db.DateTime, default=datetime.now)
message = db.Column(db.TEXT, default="", nullable=False)
def __repr__(self):
return "" % \
(self.id, self.id_object, self.message, self.created)
@classmethod
def get(cls, *criteria, **filters):
""" A wrapper for the filter and filter_by functions of sqlalchemy.
Define a dict with which columns should be filtered by which values.
look up also sqalchemy BaseQuery's filter and filter_by documentation
"""
return cls.query.filter(*criteria).filter_by(**filters)
@classmethod
def get_most_recent(cls, *criteria, **filters):
""" Returns the most recently created log. """
most_recent = cls.get(*criteria, **filters).\
order_by(desc(BibWorkflowEngineLog.created)).first()
if most_recent is None:
raise NoResultFound
else:
return most_recent
@classmethod
def delete(cls, uuid=None):
cls.get(BibWorkflowEngineLog.id == uuid).delete()
db.session.commit()
__all__ = ['Workflow', 'BibWorkflowObject', 'BibWorkflowObjectLog', 'BibWorkflowEngineLog']
diff --git a/invenio/modules/workflows/tasks/marcxml_tasks.py b/invenio/modules/workflows/tasks/marcxml_tasks.py
index 47bcb0e1f..7aa260fb6 100644
--- a/invenio/modules/workflows/tasks/marcxml_tasks.py
+++ b/invenio/modules/workflows/tasks/marcxml_tasks.py
@@ -1,671 +1,669 @@
## This file is part of Invenio.
## Copyright (C) 2012, 2013 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
import os
import random
import time
import glob
import re
import traceback
from invenio.legacy.bibupload.engine import (find_record_from_recid,
find_record_from_sysno,
find_records_from_extoaiid,
find_record_from_oaiid,
find_record_from_doi
)
from invenio.legacy.oaiharvest.dblayer import create_oaiharvest_log_str
from invenio.base.config import (CFG_TMPSHAREDDIR,
CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT,
CFG_TMPDIR,
CFG_INSPIRE_SITE)
from invenio.legacy.oaiharvest.utils import (record_extraction_from_file,
collect_identifiers,
harvest_step,
translate_fieldvalues_from_latex,
find_matching_files,
)
from invenio.legacy.bibsched.bibtask import (task_sleep_now_if_required,
task_low_level_submission
)
from invenio.modules.oai_harvest.models import OaiHARVEST
-from invenio.legacy.bibfield.bibfield_jsonreader import JsonReader
+from invenio.modules.records.api import Record
from invenio.modules.workflows.errors import WorkflowError
from invenio.legacy.refextract.api import extract_references_from_file_xml
from invenio.legacy.bibrecord import (create_records,
record_xml_output
)
from invenio.utils.plotextractor.output_utils import (create_MARC,
create_contextfiles,
prepare_image_data,
remove_dups
)
from invenio.utils.plotextractor.getter import (harvest_single,
make_single_directory
)
from invenio.utils.plotextractor.cli import (get_defaults,
extract_captions,
extract_context
)
from invenio.utils.shell import (run_shell_command,
Timeout
)
import invenio.legacy.template
from invenio.utils.plotextractor.converter import (untar,
convert_images
)
from invenio.utils.serializers import deserialize_via_marshal
oaiharvest_templates = invenio.legacy.template.load('oaiharvest')
REGEXP_REFS = re.compile(".*?.*?(.*?)", re.DOTALL)
REGEXP_AUTHLIST = re.compile("", re.DOTALL)
def add_metadata_to_extra_data(obj, eng):
"""
Creates bibrecord from object data and
populates extra_data with metadata
"""
from invenio.legacy.bibrecord import create_record, record_get_field_value
record = create_record(obj.data)
obj.extra_data['redis_search']['category'] = \
record_get_field_value(record[0], '037', code='c')
obj.extra_data['redis_search']['title'] = \
record_get_field_value(record[0], '245', code='a')
obj.extra_data['redis_search']['source'] = \
record_get_field_value(record[0], '035', code='9')
add_metadata_to_extra_data.__title__ = "Metadata Extraction"
add_metadata_to_extra_data.__description__ = "Populates object's extra_data with metadata"
def approve_record(obj, eng):
"""
Will add the approval widget to the record
"""
obj.extra_data["last_task_name"] = 'Record Approval'
eng.log.info("last task name: approve_record")
try:
obj.extra_data['message'] = 'Record needs approval. Click on widget to resolve.'
eng.log.info("Adding the approval widget to %s" % obj.id)
obj.extra_data['widget'] = 'approval_widget'
eng.halt("Record needs approval")
except KeyError:
# Log the error
obj.extra_data["error_msg"] = 'Could not assign widget'
approve_record.__title__ = "Record Approval"
approve_record.__description__ = "This task assigns the approval widget to a record."
def convert_record_to_bibfield(obj, eng):
"""
Convert a record in data into a 'dictionary'
thanks to BibField
"""
- from invenio.legacy.bibfield import create_record
+ from invenio.base.records.api import create_record
eng.log.info("last task name: convert_record_to_bibfield")
- obj.data = create_record(obj.data).rec_json
+ obj.data = create_record(obj.data).dumps()
eng.log.info("Conversion succeed")
def init_harvesting(obj, eng):
"""
This function gets all the option linked to the task and stores them into the
object to be used later.
"""
eng.log.info("last task name: init_harvesting")
try:
obj.extra_data["options"] = eng.extra_data["options"]
except KeyError:
eng.log.error("Non Critical Error: No options", "No options for this task have been found. It is possible"
"that the fillowing task could failed or work not as expected")
obj.extra_data["options"] = {}
eng.log.info("end of init_harvesting")
def get_repositories_list(repositories):
"""
Here we are retrieving the oaiharvest configuration for the task.
It will allows in the future to do all the correct operations.
"""
def _get_repositories_list(obj, eng):
eng.log.info("last task name: _get_repositories_list")
reposlist_temp = None
if repositories:
for reposname in repositories:
reposlist_temp = OaiHARVEST.get(OaiHARVEST.name == reposname).all()
else:
reposlist_temp = OaiHARVEST.get(OaiHARVEST.name != "").all()
return reposlist_temp
return _get_repositories_list
def harvest_records(obj, eng):
"""
Run the harvesting task. The row argument is the oaiharvest task
queue row, containing if, arguments, etc.
Return 1 in case of success and 0 in case of failure.
"""
eng.log.info("last task name: harvest_records")
obj.extra_data["last_task_name"] = 'harvest_records'
harvested_identifier_list = []
harvestpath = "%s_%d_%s_" % ("%s/oaiharvest_%s" % (CFG_TMPSHAREDDIR, eng.uuid),
1, time.strftime("%Y%m%d%H%M%S"))
# ## go ahead: check if user requested from-until harvesting
try:
if "dates" not in obj.extra_data["options"]:
obj.extra_data["options"]["dates"] = {}
if "identifiers" not in obj.extra_data["options"]:
obj.extra_data["options"]["identifiers"] = {}
except TypeError:
obj.extra_data["options"] = {"dates": {}, "identifiers": {}}
task_sleep_now_if_required()
arguments = obj.extra_data["repository"].get_arguments()
if arguments:
eng.log.info("running with post-processes: %r" % (arguments,))
# Harvest phase
try:
harvested_files_list = harvest_step(obj.data,
harvestpath,
obj.extra_data["options"]["identifiers"],
obj.extra_data["options"]["dates"])
except Exception:
eng.log.error("Error while harvesting %s. Skipping." % (obj.data,))
raise WorkflowError("Error while harvesting %r. Skipping." % (obj.data,),
id_workflow=eng.uuid)
if len(harvested_files_list) == 0:
eng.log.error("No records harvested for %s" % (obj.data.name,))
return None
# Retrieve all OAI IDs and set active list
harvested_identifier_list.append(collect_identifiers(harvested_files_list))
if len(harvested_files_list) != len(harvested_identifier_list[0]):
# Harvested files and its identifiers are 'out of sync', abort harvest
msg = "Harvested files miss identifiers for %s" % (arguments,)
eng.log.info(msg)
raise WorkflowError(msg, id_workflow=eng.uuid)
eng.log.info("%d files harvested and processed" % (len(harvested_files_list),))
eng.log.info("End harvest records task")
harvest_records.__id__ = "h"
def get_records_from_file(path=None):
def _get_records_from_file(obj, eng):
eng.log.info("last task name: _get_records_from_file")
if not "LoopData" in eng.extra_data:
eng.extra_data["LoopData"] = {}
if "get_records_from_file" not in eng.extra_data["LoopData"]:
if path:
eng.extra_data["LoopData"].update({"get_records_from_file": record_extraction_from_file(path)})
else:
eng.extra_data["LoopData"].update({"get_records_from_file": record_extraction_from_file(obj.data)})
return eng.extra_data["LoopData"]["get_records_from_file"]
return _get_records_from_file
def get_eng_uuid_harvested(obj, eng):
"""
Simple function which allows to retrieve the uuid of the eng in the workflow
for printing by example
"""
eng.log.info("last task name: get_eng_uuid_harvested")
return "*" + str(eng.uuid) + "*.harvested"
def get_files_list(path, parameter):
def _get_files_list(obj, eng):
eng.log.info("last task name: get_files_list")
if callable(parameter):
unknown = parameter(obj, eng)
else:
unknown = parameter
result = glob.glob1(path, unknown)
for i in range(0, len(result)):
result[i] = path + os.sep + result[i]
return result
return _get_files_list
def convert_record(stylesheet="oaidc2marcxml.xsl"):
def _convert_record(obj, eng):
"""
Will convert the object data, if XML, using the given stylesheet
"""
eng.log.info("last task name: convert_record")
from invenio.legacy.bibconvert.xslt_engine import convert
obj.extra_data["last_task_name"] = 'Convert Record'
eng.log.info("Starting conversion using %s stylesheet" %
(stylesheet,))
try:
obj.data = convert(obj.data, stylesheet)
except Exception as e:
msg = "Could not convert record: %s\n%s" % \
(str(e), traceback.format_exc())
obj.extra_data["error_msg"] = msg
eng.log.error("Error: %s" % (msg,))
raise WorkflowError("Error: %s" % (msg,),
id_workflow=eng.uuid)
return _convert_record
def fulltext_download(obj, eng):
"""
Performs the fulltext download step.
Only for arXiv
"""
eng.log.info("full-text attachment step started")
task_sleep_now_if_required()
if "pdf" not in obj.extra_data["options"]["identifiers"]:
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
tarball, pdf = harvest_single(obj.data["system_control_number"]["value"],
extract_path, ["pdf"])
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
arguments = obj.extra_data["repository"].get_arguments()
if not arguments['t_doctype'] == '':
doctype = arguments['t_doctype']
else:
doctype = 'arXiv'
if pdf:
obj.extra_data["options"]["identifiers"]["pdf"] = pdf
fulltext_xml = (" \n"
" %(url)s\n"
" %(doctype)s\n"
" "
) % {'url': obj.extra_data["options"]["identifiers"]["pdf"],
'doctype': doctype}
updated_xml = '\n\n\n' + fulltext_xml + \
'\n'
- from invenio.legacy.bibfield import create_record
+ from invenio.modules.records.api import create_record
- new_dict_representation = create_record(updated_xml).rec_json
+ new_dict_representation = create_record(updated_xml).dumps()
try:
obj.data['fft'].append(new_dict_representation["fft"])
except:
obj.data['fft'] = [new_dict_representation['fft']]
def quick_match_record(obj, eng):
"""
Retrieve the record Id from a record by using tag 001 or SYSNO or OAI ID or DOI
tag. opt_mod is the desired mode.
001 fields even in the insert mode
"""
eng.log.info("last task name: quick_match_record")
obj.extra_data["last_task_name"] = 'Quick Match Record'
function_dictionnary = {'recid': find_record_from_recid, 'system_number': find_record_from_sysno,
'oaiid': find_record_from_oaiid, 'system_control_number': find_records_from_extoaiid,
'doi': find_record_from_doi}
- my_json_reader = JsonReader()
- my_json_reader.rec_json = obj.data
+ my_json_reader = Record(obj.data)
try:
identifiers = {}
#identifiers = my_json_reader.get_persistent_identifiers()
except KeyError:
identifiers = {}
if not "recid" in identifiers:
for identifier in identifiers:
recid = function_dictionnary[identifier](identifiers[identifier]["value"])
if recid:
obj.data['recid']['value'] = recid
return True
return False
else:
return True
def upload_record(mode="ir"):
def _upload_record(obj, eng):
eng.log.info("last task name: upload_record")
from invenio.legacy.bibsched.bibtask import task_low_level_submission
obj.extra_data["last_task_name"] = 'Upload Record'
eng.log_info("Saving data to temporary file for upload")
filename = obj.save_to_file()
params = ["-%s" % (mode,), filename]
task_id = task_low_level_submission("bibupload", "bibworkflow",
*tuple(params))
eng.log_info("Submitted task #%s" % (task_id,))
_upload_record.__title__ = "Upload Record"
_upload_record.__description__ = "Uploads the record using BibUpload"
return _upload_record
upload_record.__id__ = "u"
def plot_extract(plotextractor_types):
def _plot_extract(obj, eng):
"""
Performs the plotextraction step.
"""
eng.log.info("last task name: plot_extract")
obj.extra_data["last_task_name"] = 'plotextraction'
eng.log.info("plotextraction step started")
# Download tarball for each harvested/converted record, then run plotextrator.
# Update converted xml files with generated xml or add it for upload
task_sleep_now_if_required()
if 'latex' in plotextractor_types:
# Run LaTeX plotextractor
if "tarball" not in obj.extra_data["options"]["identifiers"]:
# turn oaiharvest_23_1_20110214161632_converted -> oaiharvest_23_1_material
# to let harvested material in same folder structure
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
tarball, pdf = harvest_single(obj.data["system_control_number"]["value"], extract_path, ["tarball"])
tarball = str(tarball)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
if tarball is None:
raise WorkflowError("Error harvesting tarball from id: %s %s" %
(obj.data["system_control_number"]["value"], extract_path),
id_workflow=eng.uuid)
obj.extra_data["options"]["identifiers"]["tarball"] = tarball
else:
tarball = obj.extra_data["options"]["identifiers"]["tarball"]
sub_dir, refno = get_defaults(tarball, CFG_TMPDIR, "")
tex_files = None
image_list = None
try:
extracted_files_list, image_list, tex_files = untar(tarball, sub_dir)
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (tarball,))
converted_image_list = convert_images(image_list)
eng.log.info('converted %d of %d images found for %s' % (len(converted_image_list),
len(image_list),
os.path.basename(tarball)))
extracted_image_data = []
if tex_files == [] or tex_files is None:
eng.log.error('%s is not a tarball' % (os.path.split(tarball)[-1],))
run_shell_command('rm -r %s', (sub_dir,))
else:
for tex_file in tex_files:
# Extract images, captions and labels
partly_extracted_image_data = extract_captions(tex_file, sub_dir,
converted_image_list)
if partly_extracted_image_data:
# Add proper filepaths and do various cleaning
cleaned_image_data = prepare_image_data(partly_extracted_image_data,
tex_file, converted_image_list)
# Using prev. extracted info, get contexts for each image found
extracted_image_data.extend((extract_context(tex_file, cleaned_image_data)))
if extracted_image_data:
extracted_image_data = remove_dups(extracted_image_data)
create_contextfiles(extracted_image_data)
marc_xml = '\n\n'
marc_xml = marc_xml + create_MARC(extracted_image_data, tarball, None)
marc_xml += "\n"
if marc_xml:
- from invenio.legacy.bibfield import create_record
+ from invenio.modules.records.api import create_record
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
- new_dict_representation = create_record(marc_xml).rec_json
+ new_dict_representation = create_record(marc_xml).dumps()
try:
obj.data['fft'].append(new_dict_representation["fft"])
except KeyError:
obj.data['fft'] = [new_dict_representation['fft']]
return _plot_extract
def refextract(obj, eng):
"""
Performs the reference extraction step.
"""
eng.log.info("refextraction step started")
task_sleep_now_if_required()
if "pdf" not in obj.extra_data["options"]["identifiers"]:
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
tarball, pdf = harvest_single(obj.data["system_control_number"]["value"], extract_path, ["pdf"])
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
if pdf is not None:
obj.extra_data["options"]["identifiers"]["pdf"] = pdf
elif not os.path.isfile(obj.extra_data["options"]["identifiers"]["pdf"]):
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
tarball, pdf = harvest_single(obj.data["system_control_number"]["value"], extract_path, ["pdf"])
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
if pdf is not None:
obj.extra_data["options"]["identifiers"]["pdf"] = pdf
if os.path.isfile(obj.extra_data["options"]["identifiers"]["pdf"]):
cmd_stdout = extract_references_from_file_xml(obj.extra_data["options"]["identifiers"]["pdf"])
references_xml = REGEXP_REFS.search(cmd_stdout)
if references_xml:
updated_xml = '\n\n' + references_xml.group(1) + \
"\n"
- from invenio.legacy.bibfield import create_record
+ from invenio.modules.records.api import create_record
- new_dict_representation = create_record(updated_xml).rec_json
+ new_dict_representation = create_record(updated_xml).dumps()
try:
obj.data['reference'].append(new_dict_representation["reference"])
except KeyError:
if 'reference' in new_dict_representation:
obj.data['reference'] = [new_dict_representation['reference']]
else:
obj.log.error("Not able to download and process the PDF ")
def author_list(obj, eng):
"""
Performs the special authorlist extraction step (Mostly INSPIRE/CERN related).
"""
eng.log.info("last task name: author_list")
eng.log.info("authorlist extraction step started")
identifiers = obj.data["system_control_number"]["value"]
task_sleep_now_if_required()
if "tarball" not in obj.extra_data["options"]["identifiers"]:
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
tarball, pdf = harvest_single(obj.data["system_control_number"]["value"], extract_path, ["tarball"])
tarball = str(tarball)
time.sleep(CFG_PLOTEXTRACTOR_DOWNLOAD_TIMEOUT)
if tarball is None:
raise WorkflowError("Error harvesting tarball from id: %s %s" %
(identifiers, extract_path),
id_workflow=eng.uuid)
obj.extra_data["options"]["identifiers"]["tarball"] = tarball
sub_dir, dummy = get_defaults(obj.extra_data["options"]["identifiers"]["tarball"], CFG_TMPDIR, "")
try:
untar(obj.extra_data["options"]["identifiers"]["tarball"], sub_dir)
except Timeout:
eng.log.error('Timeout during tarball extraction on %s' % (obj.extra_data["options"]["identifiers"]["tarball"]))
xml_files_list = find_matching_files(sub_dir, ["xml"])
authors = ""
for xml_file in xml_files_list:
xml_file_fd = open(xml_file, "r")
xml_content = xml_file_fd.read()
xml_file_fd.close()
match = REGEXP_AUTHLIST.findall(xml_content)
if not match == []:
authors += match[0]
# Generate file to store conversion results
if authors is not '':
from invenio.legacy.bibconvert.xslt_engine import convert
authors = convert(authors, "authorlist2marcxml.xsl")
authorlist_record = create_records(authors)
if len(authorlist_record) == 1:
if authorlist_record[0][0] is None:
eng.log.error("Error parsing authorlist record for id: %s" % (identifiers,))
authorlist_record = authorlist_record[0][0]
# Convert any LaTeX symbols in authornames
translate_fieldvalues_from_latex(authorlist_record, '100', code='a')
translate_fieldvalues_from_latex(authorlist_record, '700', code='a')
# Look for any UNDEFINED fields in authorlist
#key = "UNDEFINED"
#matching_fields = record_find_matching_fields(key, authorlist_record, tag='100') +\
# record_find_matching_fields(key, authorlist_record, tag='700')
#if len(matching_fields) > 0:
# UNDEFINED found. Create ticket in author queue
# ticketid = create_authorlist_ticket(matching_fields, \
# identifiers, arguments.get('a_rt-queue'))
# if ticketid:
# eng.log.info("authorlist RT ticket %d submitted for %s" % (ticketid, identifiers))
# else:
# eng.log.error("Error while submitting RT ticket for %s" % (identifiers,))
updated_xml = '\n\n' + record_xml_output(authorlist_record) \
+ ''
if not None == updated_xml:
- from invenio.legacy.bibfield import create_record
+ from invenio.base.records.api import create_record
# We store the path to the directory the tarball contents live
# Read and grab MARCXML from plotextractor run
- new_dict_representation = create_record(updated_xml).rec_json
+ new_dict_representation = create_record(updated_xml).dumps()
obj.data['authors'] = new_dict_representation["authors"]
obj.data['number_of_authors'] = new_dict_representation["number_of_authors"]
author_list.__id__ = "u"
def upload_step(obj, eng):
"""
Perform the upload step.
"""
eng.log.info("upload step started")
uploaded_task_ids = []
#Work comment:
#
#Prepare in case of filtering the files to up,
#no filtering, no other things to do
- new_dict_representation = JsonReader()
- new_dict_representation.rec_json = obj.data
+ new_dict_representation = Record(obj.data)
marcxml_value = new_dict_representation.legacy_export_as_marc()
task_id = None
# Get a random sequence ID that will allow for the tasks to be
# run in order, regardless if parallel task execution is activated
sequence_id = random.randrange(1, 4294967296)
task_sleep_now_if_required()
extract_path = make_single_directory(CFG_TMPSHAREDDIR, eng.uuid)
# Now we launch BibUpload tasks for the final MARCXML files
filepath = extract_path + os.sep + str(obj.id)
file_fd = open(filepath, 'w')
file_fd.write(marcxml_value)
file_fd.close()
mode = ["-r", "-i"]
arguments = obj.extra_data["repository"].get_arguments()
if os.path.exists(filepath):
try:
args = mode
if sequence_id:
args.extend(['-I', str(sequence_id)])
if arguments.get('u_name', ""):
args.extend(['-N', arguments.get('u_name', "")])
if arguments.get('u_priority', 5):
args.extend(['-P', str(arguments.get('u_priority', 5))])
args.append(filepath)
task_id = task_low_level_submission("bibupload", "oaiharvest", *tuple(args))
create_oaiharvest_log(task_id, obj.extra_data["repository"].id, filepath)
except Exception, msg:
eng.log.error("An exception during submitting oaiharvest task occured : %s " % (str(msg)))
return None
else:
eng.log.error("marcxmlfile %s does not exist" % (filepath,))
if task_id is None:
eng.log.error("an error occurred while uploading %s from %s" %
(filepath, obj.extra_data["repository"].name))
else:
uploaded_task_ids.append(task_id)
eng.log.info("material harvested from source %s was successfully uploaded" %
(obj.extra_data["repository"].name,))
if CFG_INSPIRE_SITE:
# Launch BibIndex,Webcoll update task to show uploaded content quickly
bibindex_params = ['-w', 'collection,reportnumber,global',
'-P', '6',
'-I', str(sequence_id),
'--post-process',
'bst_run_bibtask[taskname="webcoll", user="oaiharvest", P="6", c="HEP"]']
task_low_level_submission("bibindex", "oaiharvest", *tuple(bibindex_params))
eng.log.info("end of upload")
def create_oaiharvest_log(task_id, oai_src_id, marcxmlfile):
"""
Function which creates the harvesting logs
@param task_id bibupload task id
"""
file_fd = open(marcxmlfile, "r")
xml_content = file_fd.read(-1)
file_fd.close()
create_oaiharvest_log_str(task_id, oai_src_id, xml_content)
diff --git a/invenio/testsuite/__init__.py b/invenio/testsuite/__init__.py
index 547639d6e..1afd6788c 100644
--- a/invenio/testsuite/__init__.py
+++ b/invenio/testsuite/__init__.py
@@ -1,1195 +1,1195 @@
# This file is part of Invenio.
# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Invenio is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
# pylint: disable=E1102
from __future__ import with_statement
"""
Helper functions for building and running test suites.
"""
CFG_TESTUTILS_VERBOSE = 1
import os
import sys
import time
pyv = sys.version_info
if pyv[0] == 2 and pyv[1] < 7:
import unittest2 as unittest
else:
import unittest
import cgi
import subprocess
import binascii
import StringIO
from flask import url_for
from functools import wraps
from warnings import warn
from urlparse import urlsplit, urlunsplit
from urllib import urlencode
from itertools import chain, repeat
try:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
except ImportError:
# web tests will not be available, but unit and regression tests will:
pass
#try:
# from nose.tools import nottest
#except ImportError:
# def nottest(f):
# """Helper decorator to mark a function as not to be tested by nose."""
# f.__test__ = False
# return f
nottest = unittest.skip('nottest')
#@nottest
def warn_user_about_tests(test_suite_type='regression'):
"""
Display a standard warning about running tests that might modify
user data, and wait for user confirmation, unless --yes-i-know
was specified in the comman line.
"""
# Provide a command line option to avoid having to type the
# confirmation every time during development.
if '--yes-i-know' in sys.argv:
return
if test_suite_type == 'web':
sys.stderr.write("""\
**********************************************************************
** **
** A B O U T T H E W E B T E S T S U I T E **
** **
** The web test suite will be launched in Firefox. You must have **
** the Selenium IDE extension installed to be able to run the web **
** test suite. If you do, please check out the results of the web **
** test suite in the Selenium IDE window. **
** **
**********************************************************************
""")
sys.stderr.write("""\
**********************************************************************
** **
** I M P O R T A N T W A R N I N G **
** **
** The %s test suite needs to be run on a clean demo site **
** that you can obtain by doing: **
** **
** $ inveniocfg --drop-demo-site **
** $ inveniocfg --create-demo-site **
** $ inveniocfg --load-demo-records **
** **
** Note that DOING THE ABOVE WILL ERASE YOUR ENTIRE DATABASE. **
** **
** In addition, due to the write nature of some of the tests, **
** the demo DATABASE will be ALTERED WITH JUNK DATA, so that **
** it is recommended to rebuild the demo site anew afterwards. **
** **
**********************************************************************
Please confirm by typing 'Yes, I know!': """ % test_suite_type)
answer = raw_input('')
if answer != 'Yes, I know!':
sys.stderr.write("Aborted.\n")
raise SystemExit(0)
return
#@nottest
def make_test_suite(*test_cases):
""" Build up a test suite given separate test cases"""
return unittest.TestSuite([unittest.makeSuite(case, 'test')
for case in test_cases])
from invenio.base.factory import create_app
import pyparsing # pylint: disable=W0611
# pyparsinf needed to import here before flask.ext.testing
# in order to avoid pyparsing troubles due to twill
from flask.ext.testing import TestCase
class InvenioFixture(object):
def __init__(self, fixture_builder=None):
self.fixture = None
self.fixture_builder = fixture_builder
def with_data(self, *datatypes):
def dictate(func):
@wraps(func)
def patched(*args, **kwargs):
if self.fixture is None:
self.fixture = self.fixture_builder()
@self.fixture.with_data(*datatypes)
def with_data_func(data):
return func(data, *args, **kwargs)
return with_data_func()
return patched
return dictate
class InvenioTestCase(TestCase, unittest.TestCase):
@property
def config(self):
cfg = {
'engine': 'CFG_DATABASE_TYPE',
'host': 'CFG_DATABASE_HOST',
'port': 'CFG_DATABASE_PORT',
'username': 'CFG_DATABASE_USER',
'password': 'CFG_DATABASE_PASS',
'database': 'CFG_DATABASE_NAME',
}
out = {}
for (k, v) in cfg.iteritems():
if hasattr(self, k):
out[v] = getattr(self, k)
return out
def create_app(self):
app = create_app(**self.config)
app.testing = True
return app
def login(self, username, password):
from invenio.config import CFG_SITE_SECURE_URL
#from invenio.utils.url import rewrite_to_secure_url
return self.client.post(url_for('webaccount.login'),
base_url=CFG_SITE_SECURE_URL,
#rewrite_to_secure_url(request.base_url),
data=dict(nickname=username, password=password),
follow_redirects=True)
def logout(self):
from invenio.config import CFG_SITE_SECURE_URL
return self.client.get(url_for('webaccount.logout'),
base_url=CFG_SITE_SECURE_URL,
follow_redirects=True)
def shortDescription(self):
return
class FlaskSQLAlchemyTest(InvenioTestCase):
def setUp(self):
from invenio.ext.sqlalchemy import db
db.create_all()
def tearDown(self):
from invenio.ext.sqlalchemy import db
db.session.expunge_all()
db.session.rollback()
db.drop_all()
#@nottest
def make_flask_test_suite(*test_cases):
""" Build up a Flask test suite given separate test cases"""
from operator import add
from invenio.config import CFG_DEVEL_TEST_DATABASE_ENGINES
create_type = lambda c: [type(k + c.__name__, (c,), d)
for k, d in CFG_DEVEL_TEST_DATABASE_ENGINES.iteritems()]
return unittest.TestSuite([unittest.makeSuite(case, 'test')
for case in reduce(add, map(create_type,
test_cases))])
-#@nottest
+@nottest
def run_test_suite(testsuite, warn_user=False):
"""
Convenience function to embed in test suites. Run given testsuite
and eventually ask for confirmation of warn_user is True.
"""
if warn_user:
warn_user_about_tests()
res = unittest.TextTestRunner(descriptions=False, verbosity=2).run(testsuite)
return res.wasSuccessful()
def make_url(path, **kargs):
""" Helper to generate an absolute invenio URL with query
arguments"""
from invenio.config import CFG_SITE_URL
url = CFG_SITE_URL + path
if kargs:
url += '?' + urlencode(kargs, doseq=True)
return url
def make_surl(path, **kargs):
""" Helper to generate an absolute invenio Secure URL with query
arguments"""
from invenio.config import CFG_SITE_SECURE_URL
url = CFG_SITE_SECURE_URL + path
if kargs:
url += '?' + urlencode(kargs, doseq=True)
return url
def base64_to_file(base64_file, filepath):
"""
Write a base64 encoded version of a file to disk.
"""
with open(filepath, 'wb') as f:
f.write(binascii.a2b_base64(base64_file))
def file_to_base64(filepath):
"""
Get base64 encoded version of a file. Useful to encode a test file for
inclusion in tests.
"""
with open(filepath, 'rb') as f:
return binascii.b2a_base64(f.read())
def stringio_to_base64(stringio_obj):
"""
Get base64 encoded version of a StringIO object.
"""
return binascii.b2a_base64(stringio_obj.getvalue())
def make_file_fixture(filename, base64_file):
"""
Generate a file fixture suitable for use with the Flask test client.
@param base64_file: A string encoding a file in base64. Use
file_to_base64() to get the base64 encoding of a file. If not provided
a PDF file be generated instead, including
"""
return (
StringIO.StringIO(binascii.a2b_base64(base64_file)),
filename
)
def make_pdf_fixture(filename, text=None):
"""
Generates a PDF fixture suitable for use with Werkzeug test client and
Flask test request context.
Use of this function requires that reportlab have been installed.
@param filename: Desired filename.
@param text: Text to include in PDF. Defaults to "Filename: ", if
not specified.
"""
if text is None:
text = "Filename: %s" % filename
# Generate simple PDF
from reportlab.pdfgen import canvas
output = StringIO.StringIO()
c = canvas.Canvas(output)
c.drawString(100, 100, text)
c.showPage()
c.save()
return make_file_fixture(filename, stringio_to_base64(output))
class InvenioTestUtilsBrowserException(Exception):
"""Helper exception for the regression test suite browser."""
pass
#@nottest
def test_web_page_existence(url):
"""
Test whether URL exists and is well accessible.
Return True or raise exception in case of problems.
"""
import mechanize
browser = mechanize.Browser()
try:
browser.open(url)
except:
raise
return True
def get_authenticated_mechanize_browser(username="guest", password=""):
"""
Return an instance of a mechanize browser already authenticated
to Invenio
"""
try:
import mechanize
except ImportError:
raise InvenioTestUtilsBrowserException(
'ERROR: Cannot import mechanize.')
browser = mechanize.Browser()
browser.set_handle_robots(False) # ignore robots.txt, since we test gently
if username == "guest":
return browser
from invenio.config import CFG_SITE_SECURE_URL
browser.open(CFG_SITE_SECURE_URL + "/youraccount/login")
browser.select_form(nr=0)
browser['nickname'] = username
browser['password'] = password
browser.submit()
username_account_page_body = browser.response().read()
try:
username_account_page_body.index("You are logged in as %s." % username)
except ValueError:
raise InvenioTestUtilsBrowserException(
'ERROR: Cannot login as %s.' % username)
return browser
#@nottest
def test_web_page_content(url,
username="guest",
password="",
expected_text="