diff --git a/configure-tests.py b/configure-tests.py index 3c2721780..2cc69f4a3 100644 --- a/configure-tests.py +++ b/configure-tests.py @@ -1,500 +1,500 @@ ## This file is part of Invenio. ## Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Test the suitability of Python core and the availability of various Python modules for running Invenio. Warn the user if there are eventual troubles. Exit status: 0 if okay, 1 if not okay. Useful for running from configure.ac. """ ## minimally recommended/required versions: CFG_MIN_PYTHON_VERSION = (2, 4) CFG_MAX_PYTHON_VERSION = (2, 9, 9999) CFG_MIN_MYSQLDB_VERSION = "1.2.1_p2" ## 0) import modules needed for this testing: import string import sys import getpass import subprocess import re error_messages = [] warning_messages = [] def wait_for_user(msg): """Print MSG and prompt user for confirmation.""" try: raw_input(msg) except KeyboardInterrupt: print "\n\nInstallation aborted." sys.exit(1) except EOFError: print " (continuing in batch mode)" return ## 1) check Python version: if sys.version_info < CFG_MIN_PYTHON_VERSION: error_messages.append( """ ******************************************************* ** ERROR: TOO OLD PYTHON DETECTED: %s ******************************************************* ** You seem to be using a too old version of Python. ** ** You must use at least Python %s. ** ** ** ** Note that if you have more than one Python ** ** installed on your system, you can specify the ** ** --with-python configuration option to choose ** ** a specific (e.g. non system wide) Python binary. ** ** ** ** Please upgrade your Python before continuing. ** ******************************************************* """ % (string.replace(sys.version, "\n", ""), '.'.join(CFG_MIN_PYTHON_VERSION)) ) if sys.version_info > CFG_MAX_PYTHON_VERSION: error_messages.append( """ ******************************************************* ** ERROR: TOO NEW PYTHON DETECTED: %s ******************************************************* ** You seem to be using a too new version of Python. ** ** You must use at most Python %s. ** ** ** ** Perhaps you have downloaded and are installing an ** ** old Invenio version? Please look for more recent ** ** Invenio version or please contact the development ** ** team at about this ** ** problem. ** ** ** ** Installation aborted. ** ******************************************************* """ % (string.replace(sys.version, "\n", ""), '.'.join(CFG_MAX_PYTHON_VERSION)) ) ## 2) check for required modules: try: import MySQLdb import base64 import cPickle import cStringIO import cgi import copy import fileinput import getopt import sys if sys.hexversion < 0x2060000: import md5 else: import hashlib import marshal import os import signal import tempfile import time import traceback import unicodedata import urllib import zlib import wsgiref import sqlalchemy import werkzeug import jinja2 import flask import fixture import flask.ext.assets - import flaskext.cache + import flask.ext.cache import flask.ext.sqlalchemy import flask.ext.testing import wtforms import flask.ext.wtf ## Check Werkzeug version werkzeug_ver = werkzeug.__version__.split(".") if werkzeug_ver[0] == "0" and int(werkzeug_ver[1]) < 8: error_messages.append( """ ***************************************************** ** Werkzeug version %s detected ***************************************************** ** Your are using an outdated version of Werkzeug ** ** with known problems. Please upgrade Werkzeug to ** ** at least v0.8 by running e.g.: ** ** pip install Werkzeug --upgrade ** ***************************************************** """ % werkzeug.__version__ ) except ImportError, msg: error_messages.append(""" ************************************************* ** IMPORT ERROR %s ************************************************* ** Perhaps you forgot to install some of the ** ** prerequisite Python modules? Please look ** ** at our INSTALL file for more details and ** ** fix the problem before continuing! ** ************************************************* """ % msg ) ## 3) check for recommended modules: try: import rdflib except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that rdflib is needed only if you plan ** ** to work with the automatic classification of ** ** documents based on RDF-based taxonomies. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import pyRXP except ImportError, msg: warning_messages.append(""" ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that PyRXP is not really required but ** ** we recommend it for fast XML MARC parsing. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import dateutil except ImportError, msg: warning_messages.append(""" ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that dateutil is not really required but ** ** we recommend it for user-friendly date ** ** parsing. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import libxml2 except ImportError, msg: warning_messages.append(""" ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that libxml2 is not really required but ** ** we recommend it for XML metadata conversions ** ** and for fast XML parsing. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import libxslt except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that libxslt is not really required but ** ** we recommend it for XML metadata conversions. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import Gnuplot except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that Gnuplot.py is not really required but ** ** we recommend it in order to have nice download ** ** and citation history graphs on Detailed record ** ** pages. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import magic if not hasattr(magic, "open"): raise StandardError except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that magic module is not really required ** ** but we recommend it in order to have detailed ** ** content information about fulltext files. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) except StandardError: warning_messages.append( """ ***************************************************** ** IMPORT WARNING python-magic ***************************************************** ** The python-magic package you installed is not ** ** the one supported by Invenio. Please refer to ** ** the INSTALL file for more details. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ ) try: import reportlab except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that reportlab module is not really ** ** required, but we recommend it you want to ** ** enrich PDF with OCR information. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) try: import pyPdf except ImportError, msg: warning_messages.append( """ ***************************************************** ** IMPORT WARNING %s ***************************************************** ** Note that pyPdf module is not really ** ** required, but we recommend it you want to ** ** enrich PDF with OCR information. ** ** ** ** You can safely continue installing Invenio ** ** now, and add this module anytime later. (I.e. ** ** even after your Invenio installation is put ** ** into production.) ** ***************************************************** """ % msg ) ## 4) check for versions of some important modules: if MySQLdb.__version__ < CFG_MIN_MYSQLDB_VERSION: error_messages.append( """ ***************************************************** ** ERROR: PYTHON MODULE MYSQLDB %s DETECTED ***************************************************** ** You have to upgrade your MySQLdb to at least ** ** version %s. You must fix this problem ** ** before continuing. Please see the INSTALL file ** ** for more details. ** ***************************************************** """ % (MySQLdb.__version__, CFG_MIN_MYSQLDB_VERSION) ) try: import Stemmer try: from Stemmer import algorithms except ImportError, msg: error_messages.append( """ ***************************************************** ** ERROR: STEMMER MODULE PROBLEM %s ***************************************************** ** Perhaps you are using an old Stemmer version? ** ** You must either remove your old Stemmer or else ** ** upgrade to Snowball Stemmer ** ** before continuing. Please see the INSTALL file ** ** for more details. ** ***************************************************** """ % (msg) ) except ImportError: pass # no prob, Stemmer is optional ## 5) check for Python.h (needed for intbitset): try: from distutils.sysconfig import get_python_inc path_to_python_h = get_python_inc() + os.sep + 'Python.h' if not os.path.exists(path_to_python_h): raise StandardError, "Cannot find %s" % path_to_python_h except StandardError, msg: error_messages.append( """ ***************************************************** ** ERROR: PYTHON HEADER FILE ERROR %s ***************************************************** ** You do not seem to have Python developer files ** ** installed (such as Python.h). Some operating ** ** systems provide these in a separate Python ** ** package called python-dev or python-devel. ** ** You must install such a package before ** ** continuing the installation process. ** ***************************************************** """ % (msg) ) ## Check if ffmpeg is installed and if so, with the minimum configuration for bibencode try: try: process = subprocess.Popen('ffprobe', stderr=subprocess.PIPE, stdout=subprocess.PIPE) except OSError: raise StandardError, "FFMPEG/FFPROBE does not seem to be installed!" returncode = process.wait() output = process.communicate()[1] RE_CONFIGURATION = re.compile("(--enable-[a-z0-9\-]*)") CONFIGURATION_REQUIRED = ( '--enable-gpl', '--enable-version3', '--enable-nonfree', '--enable-libtheora', '--enable-libvorbis', '--enable-libvpx', '--enable-libopenjpeg' ) options = RE_CONFIGURATION.findall(output) if sys.version_info < (2, 6): import sets s = sets.Set(CONFIGURATION_REQUIRED) if not s.issubset(options): raise StandardError, options.difference(s) else: if not set(CONFIGURATION_REQUIRED).issubset(options): raise StandardError, set(CONFIGURATION_REQUIRED).difference(options) except StandardError, msg: warning_messages.append( """ ***************************************************** ** WARNING: FFMPEG CONFIGURATION MISSING %s ***************************************************** ** You do not seem to have FFmpeg configured with ** ** the minimum video codecs to run the demo site. ** ** Please install the necessary libraries and ** ** re-install FFmpeg according to the Invenio ** ** installation manual (INSTALL). ** ***************************************************** """ % (msg) ) if warning_messages: print """ ****************************************************** ** WARNING MESSAGES ** ****************************************************** """ for warning in warning_messages: print warning if error_messages: print """ ****************************************************** ** ERROR MESSAGES ** ****************************************************** """ for error in error_messages: print error if warning_messages and error_messages: print """ There were %(n_err)s error(s) found that you need to solve. Please see above, solve them, and re-run configure. Note that there are also %(n_wrn)s warnings you may want to look into. Aborting the installation. """ % {'n_wrn': len(warning_messages), 'n_err': len(error_messages)} sys.exit(1) elif error_messages: print """ There were %(n_err)s error(s) found that you need to solve. Please see above, solve them, and re-run configure. Aborting the installation. """ % {'n_err': len(error_messages)} sys.exit(1) elif warning_messages: print """ There were %(n_wrn)s warnings found that you may want to look into, solve, and re-run configure before you continue the installation. However, you can also continue the installation now and solve these issues later, if you wish. """ % {'n_wrn': len(warning_messages)} wait_for_user("Press ENTER to continue the installation...") diff --git a/modules/miscutil/lib/cache.py b/modules/miscutil/lib/cache.py index 5628c9e7b..49857a931 100644 --- a/modules/miscutil/lib/cache.py +++ b/modules/miscutil/lib/cache.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- ## ## This file is part of Invenio. ## Copyright (C) 2012 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -from flaskext.cache import Cache +from flask.ext.cache import Cache cache = Cache() diff --git a/modules/miscutil/lib/jinja2utils.py b/modules/miscutil/lib/jinja2utils.py index f093e112a..bd71346b5 100644 --- a/modules/miscutil/lib/jinja2utils.py +++ b/modules/miscutil/lib/jinja2utils.py @@ -1,241 +1,241 @@ # -*- coding: utf-8 -*- ## This file is part of Invenio. ## Copyright (C) 2012 CERN. ## ## Invenio is free software; you can redistribute it and/or ## modify it under the terms of the GNU General Public License as ## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. from jinja2 import nodes from jinja2.ext import Extension from flask import current_app from flask.ext.assets import Environment, Bundle from flask import _request_ctx_stack -from flaskext.cache import JINJA_CACHE_ATTR_NAME +from flask.ext.cache import JINJA_CACHE_ATTR_NAME ENV_PREFIX = '_collected_' def prepare_tag_bundle(cls, tag): """ Construct function that returns collected data specified in jinja2 template like `{% %}` in correct order. Here is an example that shows the final order when template inheritance is used:: example.html ------------ {%\ extends 'page.html' %} {%\ css 'template2.css' %} {%\ css 'template3.css' %} page.html --------- {%\ css 'template1.css' %} {{ get_css_bundle() }} Output: ------- [template1.css, template2.css, template3.css] """ def get_bundle(): data = getattr(cls.environment, ENV_PREFIX+tag) cls._reset(tag) return cls.environment.new_bundle(tag, data) return get_bundle class CollectionExtension(Extension): """ CollectionExtension adds new tags `css` and `js` and functions ``get_css_bundle`` and ``get_js_bundle`` for jinja2 templates. The ``new_bundle`` method is used to create bundle from list of file names collected using `css` and `js` tags. Example: {% css 'css/invenio.css' %} {% js 'js/jquery.js' %} {% js 'js/invenio.js' %} ... {% assets get_css_bundle() %} {% endassets %} {% assets get_js_bundle() %} In template, use {{ ASSETS_URL }} for printing file URL. {% endassets %} Note: If you decide not to use assets bundle but directly print stylesheet and script html tags, you MUST define: ``` _app.jinja_env.extend( use_bundle = False, collection_templates = { 'css': '', 'js': '' }) ``` Both callable and string with '%s' are allowed in ``collection_templates``. """ tags = set(['css', 'js']) def __init__(self, environment): super(CollectionExtension, self).__init__(environment) ext = dict(('get_%s_bundle' % tag, prepare_tag_bundle(self, tag)) for tag in self.tags) environment.extend( use_bundle=True, collection_templates=dict((tag, lambda x:x) for tag in self.tags), new_bundle=lambda tag, collection: collection, **ext) for tag in self.tags: self._reset(tag) def _reset(self, tag): """ Empty list of used scripts. """ setattr(self.environment, ENV_PREFIX+tag, []) def _update(self, tag, value, caller=None): """ Update list of used scripts. """ try: values = getattr(self.environment, ENV_PREFIX+tag) values.append(value) except: values = [value] #current_app.logger.info(values) setattr(self.environment, ENV_PREFIX+tag, values) return '' #return values def parse(self, parser): """ Parse Jinja statement tag defined in `self.tags` (default: css, js). This accually tries to build corresponding html script tag or collect script file name in jinja2 environment variable. If you use bundles it is important to call ``get_css_bundle`` or ``get_js_bundle`` in template after all occurrences of script tags (e.g. {% css ... %}, {% js ...%}). """ tag = parser.stream.current.value lineno = next(parser.stream).lineno value = parser.parse_tuple() #current_app.logger.info("%s: Collecting %s (%s)" % (parser.name, tag, value)) # Return html tag with link to corresponding script file. if self.environment.use_bundle is False: value = value.value if callable(self.environment.collection_templates[tag]): node = self.environment.collection_templates[tag](value) else: node = self.environment.collection_templates[tag] % value return nodes.Output([nodes.MarkSafeIfAutoescape(nodes.Const(node))]) # Call :meth:`_update` to collect names of used scripts. return nodes.CallBlock( self.call_method('_update', args=[nodes.Const(tag), value], lineno=lineno), [], [], '') def render_template_to_string(input, _from_string=False, **context): """Renders a template from the template folder with the given context and return the. :param input: the string template, or name of the template to be rendered, or an iterable with template names the first one existing will be rendered :param context: the variables that should be available in the context of the template. :note: code based on [https://github.com/mitsuhiko/flask/blob/master/flask/templating.py] """ ctx = _request_ctx_stack.top ctx.app.update_template_context(context) if _from_string: template = ctx.app.jinja_env.from_string(input) else: template = ctx.app.jinja_env.get_or_select_template(input) return template.render(context) ## Following code has been written by `guotie` and it has been ## waiting for integration to flaskext/cache/jinja2ext.py. ## ## See: ## https://github.com/guotie/flask-cache/commit/3d7f43d3e0545e6e07972f01b989a83fcc0ea1b2 ## ## Info: ## Update flaskext/cache/jinja2ext.py ## Add dynamic cache for jinja template ## for example, you can cache a query list as this: ## ## >>> {% cache 60 page, options %} ## >>> {% for post in posts %}{{ render_post(post) }} {%endfor%} ## >>> {% endcache %} class DynCacheExtension(Extension): tags = set(['dyncache']) def parse(self, parser): lineno = parser.stream.next().lineno #cache key name is "template file path" + "line no" default_cache_key_name = u"%s%s" % (parser.filename, lineno) default_cache_key_name.encode('utf-8') cache_key_names = [nodes.Const(default_cache_key_name)] #parse timeout if parser.stream.current.type != 'block_end': timeout = parser.parse_expression() while parser.stream.skip_if('comma'): keyname = parser.parse_expression() if isinstance(keyname, nodes.Name): keyname = nodes.Name(keyname.name, 'load') cache_key_names.append(keyname) else: timeout = nodes.Const(None) args = [nodes.List(cache_key_names), timeout] body = parser.parse_statements(['name:endcache'], drop_needle=True) return nodes.CallBlock(self.call_method('_cache', args), [], [], body).set_lineno(lineno) def _cache(self, keys_list, timeout, caller): try: cache = getattr(self.environment, JINJA_CACHE_ATTR_NAME) except AttributeError, e: raise e if timeout == "del": cache.delete_many(*keys_list) return caller() key = '_'.join(keys_list) rv = cache.get(key) if rv is None: rv = caller() cache.set(key, rv, timeout) return rv