diff --git a/PythonLatex/generate_latex.py b/PythonLatex/generate_latex.py index b14f637..18438a2 100644 --- a/PythonLatex/generate_latex.py +++ b/PythonLatex/generate_latex.py @@ -1,373 +1,371 @@ -#! /usr/bin/python -################################################################ -from __future__ import print_function +#! /usr/bin/env python3 ################################################################ import re as myre import PythonLatex.latex_structure as tex import types import sympy as sym import hashlib import dill ################################################################ from sympy.printing.latex import LatexPrinter from sympy import latex from sympy.core.function import Derivative ################################################################ class CustomStrPrinter(LatexPrinter): def nbIdenticalDx(self, expr): first_dx = expr.args[1:][0] count = 0 for dx in expr.args[1:]: if dx == first_dx: count += 1 if count == len(expr.args[1:]): return count, first_dx return 0, None def _print_Derivative(self, expr): # print expr # print LatexPrinter._print_Derivative(self,expr) count, dx = self.nbIdenticalDx(expr) f = expr.args[0] if len(f.args) == 0: return LatexPrinter._print_Derivative(self, expr) dxf = f.args[0] f_nargs = len(f.args) # case where ' applies if f_nargs == 1 and dx == dxf: return (latex(expr.args[0].func) + "'" * len(expr.args[1:]) + "(" + latex(dxf) + ")") return LatexPrinter._print_Derivative(self, expr) def _print_Subs(self, expr): if not isinstance(expr.args[0], Derivative): return LatexPrinter._print_Subs(self, expr) # print expr deriv = expr.args[0] _dxs = deriv.args[1:] # print _dxs p1 = expr.args[1] p2 = expr.args[2] # print len(_dxs) _subst = {} _subst2 = {} for i in range(0, len(p1)): _subst[p1[i]] = p2[i] if hasattr(p2[i], 'args'): _p2 = type(p2[i]) _subst2[p1[i]] = _p2 _dxs = [e.subs(_subst2) for e in _dxs] new_f = deriv.args[0].subs(_subst) # print 'CCCCCCCCC' # print new_f # print _dxs # print _subst tmp = Derivative(new_f, *_dxs) # print tmp return latex(tmp) # return LatexPrinter._print_Subs(self,expr) ################################################################ class PyCode(object): """ """ def checkLine(self, line): m = myre.match(r"\\end{python}", line) if m: return Block() return Block.checkLine(self, line) def checkValidity(self): for c in self.content: if isinstance(c, types.InstanceType): raise Exception("python block cannot have subblocks") def __str__(self): return "" def generateHashSource(self): txt_content = self.content.encode() self.hash_source = hashlib.sha1(txt_content).hexdigest() gl = globals() self.global_keys_before = gl.keys() self.local_keys_before = self.__dict__.keys() def needReexecute(self): if self.alias == 'header': return True if 'loaded_hash_source' in self.__dict__: return not self.loaded_hash_source == self.hash_source return True def saveBlockOutputInfo(self): if self.alias == 'header': return global_keys = globals().keys() global_keys_to_save = set(global_keys) - set(self.global_keys_before) global_keys_to_save = list(global_keys_to_save) global_entries_to_save = dict( [(e, globals()[e]) for e in global_keys_to_save]) local_keys = self.__dict__.keys() local_keys_to_save = set(local_keys) - set(self.local_keys_before) local_keys_to_save = list(local_keys_to_save) local_entries_to_save = dict( [(e, self.__dict__[e]) for e in local_keys_to_save]) try: for k in global_entries_to_save: try: print('try to save ', k, ' ', latex(global_entries_to_save[k])) global_entries_to_save[k] = dill.dumps( global_entries_to_save[k]) print('saved ', k) except Exception as e: print(e) self.global_block_result = dill.dumps( global_entries_to_save) self.local_block_result = dill.dumps(local_entries_to_save) self.global_hash_result = hashlib.sha1( self.global_block_result).hexdigest() self.local_hash_result = hashlib.sha1( self.local_block_result).hexdigest() except Exception as e: print("cannot save block: {0}\n{1}".format(self.alias, e)) self.global_block_result = None self.global_hash_result = None self.local_block_result = None self.local_hash_result = None self.hash_source = None filename = self.alias + '.pytex.out' f = open(filename, 'w') # print "save symbols {0}: hash_source {1}".format( # self.alias,self.hash_source) raw_save = [self.hash_source, self.global_hash_result, self.global_block_result, self.local_hash_result, self.local_block_result] dill.dump(raw_save, f) f.close() def loadBlockOutputInfo(self): return if self.alias == 'header': return filename = self.alias + '.pytex.out' try: f = open(filename, 'r') self.loaded_hash_source, self.global_hash_result, self.global_block_result, self.local_hash_result, self.local_block_result = dill.load(f) # print "loaded symbols" # print "AAAAAAAAAAA " + str(self.loaded_hash_source) globals().update(dill.loads(self.global_block_result)) self.__dict__.update(dill.loads(self.local_block_result)) # print pickle.loads(self.global_block_result).keys() # print self.__dict__.keys() f.close() print('loaded {0}'.format(filename)) except Exception: # print "Could not load block " + self.alias # print e pass def evalBlock(self): txt_content = self.content self.generateHashSource() self.loadBlockOutputInfo() need_reexecute = self.needReexecute() if need_reexecute or reexec: print("execute block: {0}".format(self.alias)) gl = globals() gl.update({'self': self}) try: exec(txt_content, gl) except Exception as e: mesg = """{0}:0: For block '{1}' there was an execution problem {2} Block was: ************ {3} """.format(parsed_latex_file, self.alias, e, txt_content) print(mesg) if not continue_flag: raise RuntimeError('have to stop') del gl['self'] # self.saveBlockOutputInfo() def getContent(self, varname): try: c = self.__dict__[varname] except Exception as ex: print(ex) raise Exception("cannot retreive variable " + varname + " from block " + self.alias) if (myre.search('sympy', str(type(c)))): c = sym.latex(c) return c def __init__(self, name, alias, content): self.name = name self.alias = alias self.content = content if not self.name == 'python': raise Exception( 'cannot create a PyCode block from block type ' + self.name) if self.alias is None: raise Exception("not valid python block") self.block_eval = None ################################################################ def evaluatePythonBlock(b): if isinstance(b, tex.LatexEnvironment) and b.name == 'python': b.hide = True b = PyCode(b.name, b.option, str(b.toks[1])) globals()[b.alias] = b b.evalBlock() def replaceAliases(b, text): if isinstance(b, tex.LatexEnvironment) and b.name == 'python': return text if type(text) == str: regex = r'\\py{(.*?)}' m = myre.findall(regex, text) if not m: return text # print text for expression in m: # print('replaceAliasses: ', expression) try: d = dict() exec("ans = " + expression, globals(), d) except Exception as e: print("Block " + str(b.name) + " Cannot parse expression '" + str(expression) + "'\n" + str(e)) if continue_flag: continue raise(e) ans = d['ans'] rep = '\\py{' + expression + '}' try: ans_str = str(latex(ans)) except Exception as e: print("AAAAAAAA " + str(e)) return text # print "ans is now " + str(ans) globals().update({'ans': ans}) # print "replace " + rep + " with " + cont text = text.replace(rep, ans_str, 1) return text ################################################################ def collectScript(b, text): if isinstance(b, tex.LatexEnvironment) and b.name == 'python': # print('AAAA ', b) globals()['total_python_script'] += "{0} = CodeEval()\n".format( b.option) globals()['total_python_script'] += text.replace( 'self', b.option) return text if type(text) == str: # print text regex = r'\\py{(.*?)}' m = myre.findall(regex, text) if not m: return text # print m for expression in m: globals()['total_python_script'] += ( "print('printing expression: {0}')\n".format( str(expression).replace("'", "\\'"))) globals()['total_python_script'] += ( "ans = " + expression + "\n") globals()['total_python_script'] += ( "print(str(latex(ans)) + '\\n\\n')\n") return text ################################################################ continue_flag = False reexec = True def interpretPython(filename, c_flag, reexec_flag): continue_flag = c_flag reexec = reexec_flag tex_struct = tex.LatexStructure() # tex_struct.buildLatexBlocks(filename) tex_struct.parseLatexFile(filename) total_python_script = """ from PythonLatex.generate_latex import * class CodeEval: pass """ globals().update({'total_python_script': total_python_script}) globals().update({'reexec': reexec}) globals().update({'continue_flag': continue_flag}) globals().update({'parsed_latex_file': filename}) tex_struct.pathInBlock(text_functor=collectScript) total_python_script = globals()['total_python_script'] tmp_file = open('python_script.py', 'w') tmp_file.write(total_python_script) tmp_file.close() tex_struct.pathInBlock(begin_functor=evaluatePythonBlock, text_functor=replaceAliases) # pyblocks = tex_struct.getBlocksFromType("python") # # for b in pyblocks: # total_python_script += b.content[0] return str(tex_struct) diff --git a/PythonLatex/generic_headers.py b/PythonLatex/generic_headers.py index a4c54b1..2d06cbb 100644 --- a/PythonLatex/generic_headers.py +++ b/PythonLatex/generic_headers.py @@ -1,85 +1,99 @@ -#! /usr/bin/python +#! /usr/bin/python3 import matplotlib.pyplot as plt ################################################################ -def generateFigure(fig,name,caption=None,width=1): - if type(fig) is not list: fig = [fig] - if type(name) is not list: name = [name] - if type(width) is not list: width = [width] + +def generateFigure(fig, name, caption=None, width=1): + + if type(fig) is not list: + fig = [fig] + if type(name) is not list: + name = [name] + if type(width) is not list: + width = [width] res = '{\\center' - for f,n,w in zip(fig,name,width): + for f, n, w in zip(fig, name, width): f.savefig(n + ".png") if type(w) == float or type(w) == int: w = str(w) + "\\textwidth" - + res += "\\includegraphics" + "[width=" + w + "]{" + n + ".png}" res += '}' if caption is not None: res += "\\caption{" res += caption res += "}" return res - + ################################################################ -def make2Dplot(np_x,np_y,fig=None,axe=None,label=None,xlabel=None,ylabel=None,loglog_flag=False,**kwargs): - if fig is None: fig = plt.figure() - if axe is None: axe = fig.add_subplot(1,1,1) +def make2Dplot(np_x, np_y, fig=None, axe=None, label=None, xlabel=None, ylabel=None, loglog_flag=False, **kwargs): + + if fig is None: + fig = plt.figure() + if axe is None: + axe = fig.add_subplot(1, 1, 1) - if xlabel is not None: axe.set_xlabel(xlabel) - if ylabel is not None: axe.set_ylabel(ylabel) + if xlabel is not None: + axe.set_xlabel(xlabel) + if ylabel is not None: + axe.set_ylabel(ylabel) - if loglog_flag is False: _plot = axe.plot - if loglog_flag is True: _plot = axe.loglog - - if label is not None: - _plot(np_x,np_y,label=label,**kwargs) + if loglog_flag is False: + _plot = axe.plot + if loglog_flag is True: + _plot = axe.loglog + + if label is not None: + _plot(np_x, np_y, label=label, **kwargs) axe.legend(loc='best') - else: _plot(np_x,np_y,**kwargs) + else: + _plot(np_x, np_y, **kwargs) - return fig,axe + return fig, axe ################################################################ + + def generateArticle(): import os path = os.path.dirname(__file__) - f = open(os.path.join(path,'./article.header')) + f = open(os.path.join(path, './article.header')) return f.read() - + ################################################################ + def generateBeamer(): import os path = os.path.dirname(__file__) - f = open(os.path.join(path,'./beamer.header')) + f = open(os.path.join(path, './beamer.header')) return f.read() + flags = {} flags['slide_open'] = False + def closeSlide(): flags['slide_open'] = False return "\\end{frame}\n" - -def newSlide(title,subtitle=None): + +def newSlide(title, subtitle=None): res = "" - if flags['slide_open']: res += closeSlide() + if flags['slide_open']: + res += closeSlide() flags['slide_open'] = True res += """\\begin{frame} \\frametitle{""" + '{0}'.format(title) + "}\n" if subtitle is not None: res += """\\framesubtitle{""" + '{0}'.format(subtitle) + '}\n\n' return res - - - - - diff --git a/PythonLatex/latex_structure.py b/PythonLatex/latex_structure.py index 516d984..cbbf1bd 100644 --- a/PythonLatex/latex_structure.py +++ b/PythonLatex/latex_structure.py @@ -1,358 +1,357 @@ -#!/ usr / bin / python -################################################################ -from __future__ import print_function +#!/usr/bin/env python3 ################################################################ import re import types import pyparsing as pp ################################################################ class LatexEnvironment(object): def __init__(self, toks): self.toks = toks self.content = self.toks[1:-1] self.head = self.toks[0] self.tail = self.toks[-1] self.name = self.head.toks[3] self.hide = False try: self.option = self.head.toks[6] - except Exception as e: + except Exception: self.option = None - # print('env: {0}:{1}:{2}'.format(self.name, self.option, self.content)) + # print('env: {0}:{1}:{2}'.format( + # self.name, self.option, self.content)) def __str__(self): if self.hide: return '' return ''.join([str(self.head)] + [str(i) for i in self.content] + [str(self.tail)]) def __getitem__(self, index): return self.toks[index] class LatexCommand(object): def __init__(self, toks): self.name = toks[1] self.toks = toks # print('create_command:', self.name, toks) def __str__(self): return ''.join(self.toks) def __getitem__(self, index): return self.toks[index] class LatexBlock(object): def __init__(self, toks, name=None): self.toks = toks self.name = name def __str__(self): return ''.join([str(t) for t in self.toks]) class LatexMain(object): def __init__(self, toks): self.toks = toks self.content = self.toks self.name = "main" def __str__(self): # for i in self.toks: # print('LatexMain:\n', type(i), str(i)) res = ''.join([str(t) for t in self.toks]) # print(res) return res def __getitem__(self, index): return self.toks[index] ################################################################ class LatexStructure: def getBlocksFromType(self, typ): mylist = [] def foo(b): if b.name == typ: mylist.append(b) self.pathInBlock(fbegin=foo) return mylist @staticmethod def ppValidCharacters(): valid_characters = pp.printables valid_characters = valid_characters.replace('%', '') valid_characters = valid_characters.replace('{', '') valid_characters = valid_characters.replace('}', '') valid_characters = valid_characters.replace('\\', '') valid_characters += ' \t\r\n' return valid_characters @property def text(self): if '_text' not in self.__dict__: self._text = pp.Word(self.ppValidCharacters()) self._text = (self._text | pp.Literal('\\\\') | pp.Literal(r'\&') | pp.Literal(r'\%') | pp.Literal(r'\#') | pp.Literal(r'\_')) self._text.leaveWhitespace() # self._text.addParseAction(lambda toks: print('text:', toks)) return self._text @property def comment(self): if '_comment' not in self.__dict__: self._comment = pp.Literal('%') self._comment += pp.SkipTo(pp.LineEnd()) # self._comment.leaveWhitespace() # self._comment.addParseAction(lambda toks: print('comment: ', toks)) return self._comment @property def block(self): if '_block' not in self.__dict__: _start = pp.Literal('{') _end = pp.Literal('}') _content = pp.Forward().leaveWhitespace() _block = _start + _content + _end self._block = _block _content << pp.ZeroOrMore(self.environment() | self.ppCommand() | self.block | self.text | self.comment) self._block.leaveWhitespace() def createBlock(toks): b = LatexBlock(toks) return b # _block.addParseAction(lambda toks: print('block:', toks)) _block.addParseAction(createBlock) return self._block.leaveWhitespace() @staticmethod def ppCommand(name=None): _command = pp.Literal('\\').leaveWhitespace() if name is None: _command += pp.Word(pp.alphanums + '@').leaveWhitespace() else: _command += pp.Literal(name).leaveWhitespace() option = ( pp.Literal('[') + pp.delimitedList(pp.Word(pp.alphanums), combine=True) + pp.Literal(']')).leaveWhitespace() valid_param_character = pp.printables + ' \t\r\n\\' valid_param_character = valid_param_character.replace('{', '') valid_param_character = valid_param_character.replace('}', '') param_name = pp.delimitedList(pp.Word(valid_param_character), combine=True) parameters = ( pp.Literal('{') + param_name + pp.Literal('}').leaveWhitespace()).leaveWhitespace() _command += pp.ZeroOrMore(option | parameters).leaveWhitespace() def createCommand(toks): c = LatexCommand(toks) return c _command.addParseAction(createCommand) # _command.addParseAction(lambda cmd: print('create_command:', # cmd[0].name, cmd[0].toks)) return _command.leaveWhitespace() def environment(self): _env_start = self.ppCommand('begin') _env_end = self.ppCommand('end') _env_content = pp.Forward().leaveWhitespace() _environment = _env_start + _env_content _environment += _env_end def set_excluding_command(toks): env_name = toks[0][3] # print('startenv:', env_name) _command_excluding = self.ppCommand() # print('command_excluding:', env_name) if env_name == 'python': python_block = pp.SkipTo(pp.Literal(r'\end{python}')) _env_content << python_block else: _env_content << pp.ZeroOrMore( self.environment().leaveWhitespace() | _command_excluding | self.block | self.text | self.comment) def check(toks, env_name): # print('check:', env_name) if toks[0][1] != 'end': return toks # print('check:', toks, env_name) # print('check2:', toks[0], env_name) # print('check2:', type(toks[0]), env_name) # print('check3:', toks[0][3], env_name) # print('check2:', toks[0][3]) if toks[0][3] == env_name: # print ('biiiiip') return toks[652336456] return toks # _command_excluding.addParseAction( # lambda toks: print('command_excluding:', toks)) _command_excluding.addParseAction( lambda toks: check(toks, env_name)) # print('set_excluding_command: done') _env_start.addParseAction(set_excluding_command) # _env_content.addParseAction(lambda toks: print('found_content:', toks)) # _env_end.addParseAction(lambda toks: print('found_end:', toks[0][3])) def createEnvironment(toks): e = LatexEnvironment(toks) return e _environment.addParseAction(createEnvironment) return _environment.leaveWhitespace() def parseLatexFile(self, filename): fin = open(filename, 'r') inp = fin.read() fin.close() self.parseLatex(inp) def parseLatex(self, latex_code): _content = pp.ZeroOrMore(self.environment() | self.block | self.ppCommand() | self.text | self.comment) _content.leaveWhitespace() self._content = LatexMain(_content.parseString(latex_code)) def buildLatexBlocks(self, filename, herited_types=dict()): fin = open(filename, 'r') inp = fin.readlines() inp = "".join(inp) latex_cmd_expr = r'(\\\w+(?:\[\w*\])*(?:{[\w|,|\.|(|)]*?})+)' splitted = re.split(latex_cmd_expr, inp) self.main_block = LatexBlock() self.current_block = self.main_block for i in splitted: m = re.match(r'\\begin{(.*?)}(.*)', i) if m: name = m.group(1) options = m.group(2) self.current_block = self.current_block.createSubBlock( name, options, herited_types) continue m = re.match(r'\\end{(.*?)}', i) if m: name = m.group(1) try: self.current_block = self.current_block.endSubBlock(name) except Exception as e: print("AAAAAAAAAAAAAAAAAA") print(e) continue self.current_block.appendContent(i) if not self.current_block == self.main_block: raise Exception( "one latex block was not closed: {0}".format( self.current_block.name)) def pathInBlockOld(self, block=None, begin_functor=None, end_functor=None, text_functor=None): if block is None: block = self.main_block if begin_functor is not None: begin_functor(block) for i in range(len(block.content)): c = block.content[i] if isinstance(c, types.InstanceType): try: self.pathInBlock(c, begin_functor, end_functor, text_functor) except Exception as e: print(e) else: if text_functor is not None: block.content[i] = text_functor(block, c) if end_functor is not None: end_functor(block) def pathInBlock(self, block=None, begin_functor=None, end_functor=None, text_functor=None): if text_functor is None: def text_functor(b, c): return c if block is None: block = self._content if begin_functor is not None: begin_functor(block) for i, c in enumerate(block.content): # print(type(c), c) if isinstance(c, LatexEnvironment): self.pathInBlock(c, begin_functor, end_functor, text_functor) else: block.content[i] = text_functor(block, str(c)) if end_functor is not None: end_functor(block) def __str__(self): return str(self._content) @property def content(self): return self._content def __init__(self): self._content = None ################################################################ diff --git a/Slides/__init__.py b/Slides/__init__.py index 80c2421..90c8b01 100644 --- a/Slides/__init__.py +++ b/Slides/__init__.py @@ -1,6 +1,6 @@ -#!/bin/env python +#!/bin/env python3 from . import cell_manager from . import presentation_helper from . import count_slides from . import math_helper diff --git a/Slides/cell_manager.py b/Slides/cell_manager.py index 22cb6f3..b3a4b9c 100644 --- a/Slides/cell_manager.py +++ b/Slides/cell_manager.py @@ -1,132 +1,134 @@ -#!/bin/env python - -from __future__ import print_function +#!/bin/env python3 import os import subprocess import shutil import nbformat ################################################################ class NotFoundNotebooks(Exception): "Could not find the notebooks to treat" ################################################################ def mergeNotebooks(notebook_files, add_cell_number=False, select_cells=None): "Merges several notebooks in a single one named: talk.ipynb" print("Merging notebooks {0}".format(notebook_files)) notebooks = [nbformat.read(f, as_version=4) for f in notebook_files] if len(notebooks) == 0: raise NotFoundNotebooks cells = notebooks[0]['cells'] for n in notebooks[1:]: cells += n['cells'] if select_cells is None: select_cells = set(range(0, len(cells))) select_cells = set(select_cells) discard_cells = set(range(0, len(cells))) - select_cells new_cells = [] for cell_number, c in enumerate(cells): if cell_number in discard_cells: continue # print(c) new_cells.append(c) if add_cell_number: new_cell = nbformat.notebooknode.NotebookNode() new_cell['cell_type'] = 'markdown' new_cell['source'] = 'Cell number {0}'.format(cell_number) new_cell['metadata'] = {} new_cells.append(new_cell) # print(new_cells) notebooks[0]['cells'] = new_cells nbformat.write(notebooks[0], 'talk.ipynb') ################################################################ def launchSlideServer(options="", font_size="25px"): " Launch the HTTP server for reveal slides " dir_name = os.path.dirname(__file__) slides_config_file = os.path.join(dir_name, 'slides_config.py') reveal_path = os.path.join(dir_name, '..', 'js', 'reveal.js') - # cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter --config {1} --post serve ' + # cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter + # --config {1} --post serve ' # '--ServePostProcessor.open_in_browser=False' # ' --reveal-prefix={0} {2} talk.ipynb').format( # reveal_path, slides_config_file, options) - cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter --config {1} --post serve --SlidesExporter.font_size={3} ' + cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter ' + '--config {1} --post serve --SlidesExporter.font_size={3} ' '--ServePostProcessor.open_in_browser=False' ' {2} talk.ipynb').format( reveal_path, slides_config_file, options, font_size) print(cmd) subprocess.call(cmd, shell=True) ################################################################ def generateHTML(options="", font_size="25px"): " Launch the HTTP server for reveal slides " dir_name = os.path.dirname(__file__) slides_config_file = os.path.join(dir_name, 'slides_config.py') reveal_path = os.path.join(dir_name, 'js', './reveal.js') - # cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter --config {1}' + # cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter + # --config {1}' # ' --reveal-prefix={0} {2} talk.ipynb').format( # reveal_path, slides_config_file, options) - cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter --config {1} ' - '--SlidesExporter.pdf_format="true" --SlidesExporter.font_size={3}' + cmd = ('jupyter nbconvert --to=Slides.myslides.SlidesExporter ' + '--config {1} --SlidesExporter.pdf_format="true" ' + '--SlidesExporter.font_size={3}' ' {2} talk.ipynb').format( reveal_path, slides_config_file, options, font_size) print(cmd) subprocess.call(cmd, shell=True) ################################################################ def generateBeamer(options="", font_size="25px"): " Launch the HTTP server for reveal slides " dir_name = os.path.dirname(__file__) slides_config_file = os.path.join(dir_name, 'beamer_config.py') reveal_path = os.path.join(dir_name, 'js', './reveal.js') cmd = ('jupyter nbconvert --to=Slides.beamer.BeamerExporter --config {1} ' '--BeamerExporter.font_size={3}' ' {2} talk.ipynb').format( reveal_path, slides_config_file, options, font_size) print(cmd) subprocess.call(cmd, shell=True) ################################################################ def executeNotebook(filename, options=""): " Request the execution of all cells " print('execute notebook: ', filename) dir_name = os.path.dirname(__file__) slides_config_file = os.path.join(dir_name, 'slides_execute_config.py') subprocess.call('jupyter nbconvert --to=notebook --output tmp.ipynb' ' --config {0} {1}'.format( slides_config_file, filename), shell=True) shutil.move('tmp.ipynb', filename) subprocess.call('jupyter trust {0}'.format(filename), shell=True) ################################################################ diff --git a/Slides/count_slides.py b/Slides/count_slides.py index b3af057..61eb111 100644 --- a/Slides/count_slides.py +++ b/Slides/count_slides.py @@ -1,39 +1,38 @@ -#!/bin/env python +#!/bin/env python3 -from __future__ import print_function ################################################################ import nbformat import sys ################################################################ def countSlides(notebook_files): notebooks = [nbformat.read(f, as_version=4) for f in notebook_files] cpt = 0 for notebook in notebooks: cells = notebook['cells'] for c in cells: if 'slideshow' not in c.metadata: continue md = c.metadata.slideshow if 'slide_type' in md: if md.slide_type == 'slide': cpt += 1 if md.slide_type == 'subslide': cpt += 1 return cpt ################################################################ if __name__ == "__main__": notebook_files = sys.argv[1:] nb_slides = countSlides(notebook_files) print("found {0} slides".format(nb_slides)) ################################################################ diff --git a/Slides/execute_markdown.py b/Slides/execute_markdown.py index 9ed6334..83fb995 100644 --- a/Slides/execute_markdown.py +++ b/Slides/execute_markdown.py @@ -1,265 +1,268 @@ -#!/bin/env python +#!/bin/env python3 ################################################################ """Module containing a preprocessor that removes the outputs from code cells""" ################################################################ import os import re from textwrap import dedent try: from queue import Empty # Py 3 except ImportError: from Queue import Empty # Py 2 from traitlets import List, Unicode, Bool from nbformat.v4 import output_from_msg from nbconvert.preprocessors import Preprocessor from nbconvert.utils.exceptions import ConversionException from traitlets import Integer class CellExecutionError(ConversionException): """ Custom exception to propagate exceptions that are raised during notebook execution to the caller. This is mostly useful when using nbconvert as a library, since it allows to deal with failures gracefully. """ def __init__(self, traceback): self.traceback = traceback class ExecutePreprocessor(Preprocessor): """ Executes all the cells in a notebook """ timeout = Integer( 30, config=True, help="The time to wait (in seconds) for output from executions.") interrupt_on_timeout = Bool( False, config=True, help=dedent( """ If execution of a cell times out, interrupt the kernel and continue executing other cells rather than throwing an error and stopping. """ ) ) allow_errors = Bool( False, config=True, help=dedent( """ If `True`, a `CellExecutionError` is raised if any of the notebook cells raises an exception during execution. Otherwise, execution is continued and the output from the exception is included in the cell output. """ ) ) extra_arguments = List(Unicode()) def preprocess_cells(self, nb, resources): nb_cells = len(nb.cells) for index, cell in enumerate(nb.cells): print("Doing cell {0}/{1}".format(index, nb_cells)) nb.cells[index], resources = self.preprocess_cell(cell, resources, index) return nb, resources def preprocess(self, nb, resources): path = resources.get('metadata', {}).get('path', '') if path == '': path = None from jupyter_client.manager import start_new_kernel kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python') self.log.info("Executing notebook with kernel: %s" % kernel_name) self.km, self.kc = start_new_kernel( kernel_name=kernel_name, extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w'), cwd=path) self.kc.allow_stdin = False try: nb, resources = self.preprocess_cells(nb, resources) finally: self.kc.stop_channels() self.km.shutdown_kernel(now=True) return nb, resources def preprocess_cell(self, cell, resources, cell_index): """ Apply a transformation on each code cell. See base.py for details. """ if cell.cell_type == 'markdown': self.run_cell_markdown(cell) return cell, resources if cell.cell_type != 'code': return cell, resources outputs = self.run_cell(cell) cell.outputs = outputs if not self.allow_errors: for out in outputs: if out.output_type == 'error': pattern = """\ An error occurred while executing the following cell: ------------------ {cell.source} ------------------ {out.ename}: {out.evalue} """ msg = dedent(pattern).format(out=out, cell=cell) print(msg) raise CellExecutionError(msg) return cell, resources def run_cell_markdown(self, cell): errors = [] if 'variables' not in cell.metadata: cell.metadata.variables = {} def evaluate_code(code, cell): _code = code.group(1) ev = self.run_source(_code) for e in ev[1]: if e['output_type'] == 'display_data': ev = e break if e['output_type'] == 'execute_result': ev = e break # print type(ev) # if type(ev) == tuple: # print ev # print ev[1] if ev.output_type == 'error': cell.metadata.variables[_code] = ev.evalue errors.append(ev.evalue) if 'data' in ev: ev = ev['data'] if 'text/html' in ev: cell.metadata.variables[_code] = ev['text/html'] elif 'image/png' in ev: - cell.metadata.variables[_code] = r''.format( - ev['image/png']) + cell.metadata.variables[_code] = ( + r''.format( + ev['image/png'])) elif 'text/plain' in ev: cell.metadata.variables[_code] = ev['text/plain'] else: raise Exception( 'unknown mime type {0} for expression {1}'.format( ev.keys(), _code)) return '{{' + _code + '}}' # print cell.source re.sub(r'{{(.*?)}}', lambda c: evaluate_code(c, cell), cell.source) if len(errors) > 0: pattern = """\ An error occurred while executing the following cell: ------------------ {0} ------------------ {1} """ msg = dedent(pattern).format(cell.source, "\n".join(errors)) print(msg) raise CellExecutionError(msg) # print cell.metadata.variables # print 'done' def run_cell(self, cell): content, outs = self.run_source(cell.source) # set the prompt number for the input and the output if 'execution_count' in content: cell['execution_count'] = content['execution_count'] return outs def run_source(self, source): msg_id = self.kc.execute(source) self.log.debug("Executing cell:\n%s", source) # wait for finish, with timeout while True: try: msg = self.kc.shell_channel.get_msg(timeout=self.timeout) except Empty: - self.log.error("""Timeout waiting for execute reply (%is). - If your cell should take longer than this, you can increase the timeout with: + self.log.error(""" +Timeout waiting for execute reply (%is). +If your cell should take longer than this, +you can increase the timeout with: - c.ExecutePreprocessor.timeout = SECONDS +c.ExecutePreprocessor.timeout = SECONDS - in jupyter_nbconvert_config.py +in jupyter_nbconvert_config.py """ % self.timeout) if self.interrupt_on_timeout: self.log.error("Interrupting kernel") self.km.interrupt_kernel() break else: try: exception = TimeoutError except NameError: exception = RuntimeError raise exception("Cell execution timed out, see log" " for details.") if msg['parent_header'].get('msg_id') == msg_id: break else: # not our reply continue outs = [] while True: try: msg = self.kc.iopub_channel.get_msg(timeout=self.timeout) except Empty: self.log.warn("Timeout waiting for IOPub output") break if msg['parent_header'].get('msg_id') != msg_id: # not an output from our execution continue msg_type = msg['msg_type'] self.log.debug("output: %s", msg_type) content = msg['content'] if msg_type == 'status': if content['execution_state'] == 'idle': break else: continue elif msg_type == 'execute_input': continue elif msg_type == 'clear_output': outs = [] continue elif msg_type.startswith('comm'): continue try: out = output_from_msg(msg) except ValueError: self.log.error("unhandled iopub msg: " + msg_type) else: outs.append(out) return content, outs diff --git a/Slides/inject-markdown.py b/Slides/inject-markdown.py index eb16839..2553fc7 100755 --- a/Slides/inject-markdown.py +++ b/Slides/inject-markdown.py @@ -1,15 +1,19 @@ -#!/usr/bin/python +#!/usr/bin/python3 import subprocess -import nbformat -import io -import re -import sys,os -from Queue import Empty # Py 2 +import sys + +# import nbformat +# import io +# import re +# import os +# from Queue import Empty # Py 2 notebook_files = sys.argv[1:] for f in notebook_files: - cmd = 'jupyter nbconvert --inplace --to=notebook --config markdown_config.py {0}'.format(f) - print cmd - subprocess.call(cmd,shell=True) + cmd = ( + 'jupyter nbconvert --inplace --to=notebook ' + '--config markdown_config.py {0}'.format(f)) + print(cmd) + subprocess.call(cmd, shell=True) diff --git a/Slides/myslides.py b/Slides/myslides.py index 1479c8b..9dd8470 100644 --- a/Slides/myslides.py +++ b/Slides/myslides.py @@ -1,141 +1,142 @@ """HTML slide show Exporter class""" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import os from copy import deepcopy from warnings import warn from traitlets import Unicode, default from nbconvert.exporters.html import HTMLExporter def prepare(nb): """Add some convenience metadata on cells for the slide template.""" nb = deepcopy(nb) for cell in nb.cells: # Make sure every cell has a slide_type cell.metadata.slide_type = cell.metadata.get( 'slideshow', {}).get('slide_type', '-') # Find the first visible cell for index, cell in enumerate(nb.cells): if cell.metadata.slide_type not in {'notes', 'skip'}: cell.metadata.slide_type = 'slide' cell.metadata.slide_start = True cell.metadata.subslide_start = True first_slide_ix = index break else: raise ValueError("All cells are hidden, cannot create slideshow") in_fragment = False for index, cell in enumerate(nb.cells[first_slide_ix + 1:], start=(first_slide_ix + 1)): previous_cell = nb.cells[index - 1] # Slides are
elements in the HTML, subslides (the vertically # stacked slides) are also
elements inside the slides, # and fragments are
s within subslides. Subslide and fragment # elements can contain content: #
#
# (content) #
(content)
#
#
# Get the slide type. If type is subslide or slide, # end the last slide/subslide/fragment as applicable. if cell.metadata.slide_type == 'slide': previous_cell.metadata.slide_end = True cell.metadata.slide_start = True if cell.metadata.slide_type in {'subslide', 'slide'}: previous_cell.metadata.fragment_end = in_fragment previous_cell.metadata.subslide_end = True cell.metadata.subslide_start = True in_fragment = False elif cell.metadata.slide_type == 'fragment': cell.metadata.fragment_start = True if in_fragment: previous_cell.metadata.fragment_end = True else: in_fragment = True if cell.metadata.get('cell_style') == 'split': cell.metadata.split = True # The last cell will always be the end of a slide nb.cells[-1].metadata.fragment_end = in_fragment nb.cells[-1].metadata.subslide_end = True nb.cells[-1].metadata.slide_end = True return nb class SlidesExporter(HTMLExporter): """Exports HTML slides with reveal.js""" reveal_url_prefix = Unicode( help="""The URL prefix for reveal.js. This can be a a relative URL for a local copy of reveal.js, or point to a CDN. For speaker notes to work, a local reveal.js prefix must be used. """ ).tag(config=True) @default('reveal_url_prefix') def _reveal_url_prefix_default(self): if 'RevealHelpPreprocessor.url_prefix' in self.config: warn("Please update RevealHelpPreprocessor.url_prefix to " "SlidesExporter.reveal_url_prefix in config files.") return self.config.RevealHelpPreprocessor.url_prefix return 'reveal.js' font_size = Unicode(help="The size of the fonts").tag(config=True) @default('font_size') def _font_size_default(self): return '25px' pdf_format = Unicode(help="Set the pdf format").tag(config=True) @default('pdf_format') def _pdf_format(self): return 'False' @default('file_extension') def _file_extension_default(self): return '.slides.html' @default('template_file') def _template_file_default(self): return 'slides_reveal' output_mimetype = 'text/html' def from_notebook_node(self, nb, resources=None, **kw): resources = self._init_resources(resources) if 'reveal' not in resources: resources['reveal'] = {} resources['reveal']['url_prefix'] = self.reveal_url_prefix if self.pdf_format.lower() == 'true': css = os.path.join(os.path.split(__file__)[ 0], 'templates', 'html.css') else: css = os.path.join(os.path.split(__file__)[ 0], 'templates', 'reveal.css') resources['css'] = open(css).read() resources['css'] = resources['css'].replace( '{{font_size}}', self.font_size) nb = prepare(nb) - return super(SlidesExporter, self).from_notebook_node(nb, resources=resources, **kw) + return super(SlidesExporter, self).from_notebook_node( + nb, resources=resources, **kw) diff --git a/Slides/presentation_helper.py b/Slides/presentation_helper.py index e36d50c..6683708 100644 --- a/Slides/presentation_helper.py +++ b/Slides/presentation_helper.py @@ -1,307 +1,309 @@ #!/usr/bin/env pyton ################################################################ from sympy import * import base64 import sympy import PyPDF2 import io import IPython from IPython.display import display from IPython.display import HTML import os import fnmatch import shutil import subprocess import matplotlib.pyplot as plt import matplotlib.figure from IPython.core.interactiveshell import InteractiveShell from wand.image import Image as WImage from wand.color import Color from io import BytesIO from tempfile import NamedTemporaryFile from tempfile import mkdtemp ################################################################ def findImageFile(filename): if os.path.isfile(filename): return filename tool_dir = os.path.dirname(__file__) image_dir = os.path.join(tool_dir, '..', 'images') pattern = os.path.basename(filename) found = [] for root, dirnames, filenames in os.walk(image_dir): for filename in fnmatch.filter(filenames, pattern): found.append(os.path.join(root, filename)) if len(found) == 0: raise Exception("file not found: {0}".format(filename)) if len(found) == 1: return found[0] raise Exception("Several files were found:\n{0}".format(found)) ################################################################ def _print_latex_png(o): o = o._repr_latex_() dvioptions = None exprbuffer = BytesIO() # from IPython.core.debugger import Tracer # Tracer()() preview(o, output='png', viewer='BytesIO', outputbuffer=exprbuffer, packages=('xcolor',), dvioptions=dvioptions) return exprbuffer.getvalue() ################################################################ def registerFormatter(_type, _format, formatter_functor): f = InteractiveShell.instance().display_formatter.formatters[_format] f.for_type(_type, func=formatter_functor) ################################################################ def init_printing(): sympy.init_printing() registerFormatter(IPython.core.display.Math, 'image/png', _print_latex_png) # f = InteractiveShell.instance().display_formatter.formatters['image/png'] # f.for_type(IPython.core.display.Math, func=_print_latex_png) ################################################################ VIDEO_TAG = """""" def anim_to_html(anim): if not hasattr(anim, '_encoded_video'): with NamedTemporaryFile(suffix='.mp4') as f: anim.save(f.name, fps=1./(anim._interval*1e-3), extra_args=['-vcodec', 'libx264', '-pix_fmt', 'yuv420p']) video = open(f.name, "rb").read() toto = base64.b64encode(video).decode() # print anim._encoded_video return VIDEO_TAG.format(toto) ################################################################ def display_animation(anim): plt.close(anim._fig) return HTML(anim_to_html(anim)) ################################################################ def display_images(objs, display_flag=True, **kwargs): htmls = [display_image(o, display_flag=False, **kwargs) for o in objs] width = 100/len(htmls) htmls_withdiv = ["" for h in htmls] for i, h in enumerate(htmls): htmls_withdiv[i] = '
{0}
'.format(h) html = '\n'.join(htmls_withdiv) html = '
' + html + '
' if display_flag: display(HTML(html)) return html def display_image(obj, resolution=150, width=None, height=None, display_flag=True): if type(obj) == matplotlib.figure.Figure: png = InteractiveShell.instance().display_formatter.format(obj) elif type(obj) == str: try: fname = findImageFile(obj) except Exception as e: return str(e) img = WImage(filename=fname, resolution=resolution) img.trim() with Color('#FDFDFD') as white: twenty_percent = int(65535 * 0.2) img.transparent_color(white, alpha=0.0, fuzz=twenty_percent) if (width is None) and (height is None): display(img) return img png = InteractiveShell.instance().display_formatter.format(img) else: raise Exception('unknown obj type for an image') size_tag = "" if width is not None: size_tag += 'width="{0}"'.format(width) if height is not None: size_tag += 'height="{0}"'.format(height) data = png[0]['image/png'] data = base64.b64encode(data).decode('utf-8') - html = r'You cannot see the image with this browser/mailer'.format( + html = r'You cannot see the image with this browser/mailer'.format( base64.b64encode(data).decode('utf8'), size_tag) display(HTML(html)) return html ################################################################ def makeClassDiagramImage(classes, resolution=150, width=None, height=None): with NamedTemporaryFile(suffix='.classes', delete=False) as f: f.write(classes.encode('utf-8')) f.flush() fout = os.path.splitext(f.name)[0] + '.png' _command = ('class_dumper_dot.py --collaboration_no ' '-c {0} -o {1} -f svg'.format(f.name, fout)) # print(_command) subprocess.call(_command, shell=True) return fout ################################################################ def pdf_page_to_png(src_filename, pagenums=None, resolution=72): """ Returns specified PDF page as wand.image.Image png. :param PyPDF2.PdfFileReader src_pdf: PDF from which to take pages. :param int pagenum: Page number to take. :param int resolution: Resolution for resulting png in DPI. """ src_pdf = PyPDF2.PdfFileReader(file(src_filename, "rb")) # print dir(src_pdf) # print src_pdf.numPages if pagenums is None: pagenums = [0] res = [] for n in pagenums: dst_pdf = PyPDF2.PdfFileWriter() dst_pdf.addPage(src_pdf.getPage(n)) pdf_bytes = io.BytesIO() dst_pdf.write(pdf_bytes) pdf_bytes.seek(0) img = WImage(file=pdf_bytes, resolution=resolution) img.convert("png") res.append(img) return res ################################################################ init_printing() diff --git a/Slides/slides_config.py b/Slides/slides_config.py index 8f2246f..44412ee 100644 --- a/Slides/slides_config.py +++ b/Slides/slides_config.py @@ -1,18 +1,21 @@ +#! /usr/bin/env python3 -from __future__ import print_function import os import sys dirname = os.path.dirname(__file__) print(dirname) sys.path.append(dirname) c = get_config() c.Exporter.template_file = 'presentation.tpl' print(c.Exporter.__dict__) -#c.Exporter.template_file = 'mybasic.tpl' -c.Exporter.template_path.append(os.path.join(dirname,'templates')) -#c.Exporter.preprocessors = ['execute_markdown.ExecutePreprocessor','pre_pymarkdown.PyMarkdownPreprocessor'] -c.Exporter.preprocessors = ['jupyter_contrib_nbextensions.nbconvert_support.pre_pymarkdown.PyMarkdownPreprocessor'] +# c.Exporter.template_file = 'mybasic.tpl' +c.Exporter.template_path.append(os.path.join(dirname, 'templates')) +# c.Exporter.preprocessors = ['execute_markdown.ExecutePreprocessor', +# 'pre_pymarkdown.PyMarkdownPreprocessor'] +c.Exporter.preprocessors = [ + 'jupyter_contrib_nbextensions.nbconvert_support.pre_pymarkdown.' + 'PyMarkdownPreprocessor'] diff --git a/Slides/snippet_helper.py b/Slides/snippet_helper.py index 1597a04..60a4c57 100644 --- a/Slides/snippet_helper.py +++ b/Slides/snippet_helper.py @@ -1,405 +1,406 @@ #!/usr/bin/env python3 import subprocess import os import re from pygments import highlight from pygments.lexers import CppLexer from pygments.lexers import PythonLexer # from pygments.lexer import RegexLexer from pygments.lexers import MatlabLexer from pygments.formatters import HtmlFormatter from pygments.formatters import LatexFormatter from pygments import token from IPython.display import HTML from tempfile import NamedTemporaryFile ################################################################ class Snippet(object): """ Snippet objects manipulate codes. At construction it can be formatted using clang-format and it can be compiled to check for consistency. """ class CompilationError(Exception): """ Compilation error """ pass default_output = 'html' default_line_numbering = False def __getitem__(self, slice): return self.pigment(start=slice.start, end=slice.stop) def __init__(self, inp, format_flag=False, compile_flag=False, include_paths=None, language='cpp'): try: self.loadfile(inp) except Exception: self.content = inp.strip() if include_paths is None: self.include_paths = [] else: self.include_paths = [ os.path.abspath(i) for i in include_paths] if format_flag: self.clang_format() if compile_flag: self.compile() if language == 'cpp': self.lexer = CppLexer() elif language == 'python': self.lexer = PythonLexer() elif language == 'matlab': self.lexer = MatlabLexer() def save(self, filename): """ Saves the snippet in another file :param filename: The filename to save the snippet to """ f = open(filename, 'w') f.write(self.content) f.close() def loadfile(self, inp): filename = inp full_filename = os.path.realpath(filename) f = open(full_filename) self.content = f.read() f.close() def clang_format(self): f = NamedTemporaryFile() f.write(self.content.encode()) f.flush() p = subprocess.Popen( 'clang-format {0}'.format(f.name), shell=True, stdout=subprocess.PIPE) stdout, stderr = p.communicate() formatted_file = stdout self.content = formatted_file.decode('utf-8') def execute(self, working_dir='/tmp', args='', libraries=''): self.compile(working_dir=working_dir) f = NamedTemporaryFile(suffix='.cpp', delete=False) f.write(self.content.encode()) f.flush() working_dir = os.path.dirname(f.name) # file_dir = os.path.dirname(self.full_filename) previous_dir = os.getcwd() os.chdir(working_dir) include_paths = self.include_paths + [working_dir] include_paths = ['-I {0}'.format(a) for a in include_paths] include_paths = ' '.join(include_paths) p = subprocess.Popen( 'g++ -Wall -Wextra {1} {2} -o exec {0}'.format( f.name, include_paths, libraries), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() ret = p.returncode if ret != 0: return ret, stdout, stderr p = subprocess.Popen( './exec {0}'.format(args), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() ret = p.returncode os.chdir(previous_dir) return ret, stdout, stderr def compile(self, working_dir='/tmp'): f = NamedTemporaryFile(suffix='.cpp', delete=False) f.write(self.content.encode()) f.flush() working_dir = os.path.dirname(f.name) # file_dir = os.path.dirname(self.full_filename) previous_dir = os.getcwd() os.chdir(working_dir) include_paths = self.include_paths + [working_dir] include_paths = ['-I {0}'.format(a) for a in include_paths] include_paths = ' '.join(include_paths) p = subprocess.Popen( 'g++ -Wall -Wextra {1} -c {0}'.format(f.name, include_paths), shell=True, stderr=subprocess.PIPE) stdout, stderr = p.communicate() ret = p.returncode os.chdir(previous_dir) if ret != 0: - raise Snippet.CompilationError('compilation failed\n' + stderr.decode('utf-8')) + raise Snippet.CompilationError('compilation failed\n' + + stderr.decode('utf-8')) os.remove(f.name) o_file = os.path.splitext(f.name)[0] + '.o' os.remove(o_file) def get_lines(self): return self.content.split('\n') def get_content(self): return self.content def __str__(self): _content = self.content.split('\n') ret = "" for i, line in enumerate(_content): ret += '{0}: {1}\n'.format(i, line) return ret class KeywordLexer(object): def __init__(self, keyword): if keyword == 'curly_brackets': keyword = '{|}' self.keyword = keyword def get_tokens(self, text): pattern = '(' + self.keyword + ')' splits = re.split(pattern, text) if len(splits) == 1: return [(token.Text, text)] res = [] # print(splits) # print(token.Keyword.Type) for count, s in enumerate(splits): # print(count, s) if count % 2 == 0: res.append((token.Text, s)) else: res.append((token.Keyword.Type, s)) # print(res) return res class LineLexer(object): def __init__(self, line_number): self.line_number = line_number def get_tokens(self, text): splits = text.split('\n') res = [] splits1 = '\n'.join(splits[:self.line_number]) + '\n' splits2 = splits[self.line_number] + '\n' splits3 = '\n' + '\n'.join(splits[self.line_number + 1:]) + '\n' splits = [splits1] + [splits2] + [splits3] # print(splits) for count, s in enumerate(splits): if count == 1: res.append((token.Keyword.Type, s)) else: res.append((token.Text, s)) # print(res) return res class CustomLatexFormatter(LatexFormatter): def __init__(self, **kwargs): LatexFormatter.__init__(self, **kwargs) def format(self, tokens, outfile): res = [] for t, s in tokens: if t == token.Keyword.Type: import pygments.formatters.latex as la s = la.escape_tex(s, self.commandprefix) s = '\color{red}{\huge{' + s + '}}' t = token.Escape res.append((t, s)) LatexFormatter.format(self, res, outfile) class CustomHtmlFormatter(HtmlFormatter): def __init__(self, **kwargs): HtmlFormatter.__init__(self, **kwargs) def format(self, tokens, outfile_final): # HtmlFormatter.format(self, tokens, outfile_final) # return from io import StringIO outfile = StringIO() res = [] ss = [] for t, s in tokens: if t == token.Keyword.Type: ss.append(s) s = 'A' * 24 t = token.Text res.append((t, s)) # print(res) HtmlFormatter.format(self, res, outfile) final = outfile.getvalue() while ss: s = ss.pop(0) final = final.replace( 'A' * 24, '{0}'.format( s), 1) outfile_final.write(final) def pigment(self, keyword=None, line_highlight=None, start=None, end=None, output=None, line_numbering=None): _content = self.content.split('\n') _content = [(c + '\n') for c in _content] if start is None: start = 1 if end is None: end = len(_content) if output is None: output = self.default_output lexer = self.lexer if keyword is not None: lexer = self.KeywordLexer(keyword) elif line_highlight is not None: if line_highlight < start or line_highlight > end: raise RuntimeError('line_highlight out of the sub snippet') line_highlight -= start lexer = self.LineLexer(line_highlight) if line_numbering is None: line_numbering = self.default_line_numbering if output == 'html': formatter = HtmlFormatter( full=False, linenos=line_numbering, linenostart=start) elif output == 'latex': formatter = LatexFormatter( full=False, linenos=line_numbering, linenostart=start) if (keyword is not None) or (line_highlight is not None): format_class = self.CustomHtmlFormatter if output == 'latex': format_class = self.CustomLatexFormatter formatter = format_class( full=False, linenos=line_numbering, linenostart=start) snip = _content[start - 1:end] snip = ''.join(snip) colored_snippet = highlight(snip, lexer, formatter) # print(colored_snippet) if output == 'html': HTML_TEMPLATE = """ {} """ css = formatter.get_style_defs() colored_snippet = HTML(HTML_TEMPLATE.format(css, colored_snippet)) return colored_snippet @staticmethod def getLatexStyleDefs(): return LatexFormatter(full=True).get_style_defs() ################################################################ class SnippetCollection(Snippet): def __init__(self): self.functions = dict() self.func_names = [] def addFunction(self, f_name, code): if f_name in self.functions: raise RuntimeError('function already declared') self.func_names.append(f_name) self.functions[f_name] = Snippet(code, format_flag=True) self.compile() def __getitem__(self, key): return self.functions[key] def flatten(self): global_input = """ #include #include #include #include """ + '/' * 70 + '\n\n' for f_name in self.func_names: global_input += "void {0}(){{\n".format(f_name) global_input += self.functions[f_name].content global_input += '\n}\n\n' + '/' * 70 + '\n\n' global_input += """ int main(int argc, char ** argv){\n""" for f_name in self.func_names: global_input += f_name + '();\n' global_input += '\n}\n' flat = Snippet(global_input, format_flag=True) return flat def compile(self, **kwargs): global_snippet = self.flatten() global_snippet.compile(**kwargs) def save(self, filename): global_snippet = self.flatten() open(filename, 'w').write(global_snippet.content) def pigment(self, *args): return self.flatten().pigment(*args) def last(self): return self.functions[self.func_names[-1]] ################################################################ # def printMatlab(code): # # html_snippet = highlight( # code, MatlabLexer(), # HtmlFormatter(full=True, style='colorful', # classprefix='matlab_')) # # display(HTML(html_snippet)) # # return html_snippet # ## testing -#Snippet.default_output = 'latex' -#Snippet.default_line_numbering = True +# Snippet.default_output = 'latex' +# Snippet.default_line_numbering = True # -#snippet = SnippetCollection() -# snippet.addFunction('if_syntax', ''' -# int p, q; -# if (p > q) { -# /* -# Statement1; -# Statement2; -# */ -# } -#''') +# snippet = SnippetCollection() +# snippet.addFunction('if_syntax', ''' +# int p, q; +# if (p > q) { +# /* +# Statement1; +# Statement2; +# */ +# } +# ''') # -#print (snippet['if_syntax'].pigment()) +# print (snippet['if_syntax'].pigment()) diff --git a/scripts/platex2slides b/scripts/platex2slides index 9fc90f0..9e2fec4 100755 --- a/scripts/platex2slides +++ b/scripts/platex2slides @@ -1,77 +1,77 @@ -#!/usr/bin/env python +#!/bin/env python3 import sys import nbformat from PythonLatex.latex_structure import LatexStructure from PythonLatex.latex_structure import LatexEnvironment filename_in = sys.argv[1] filename_out = sys.argv[2] tex_struct = LatexStructure() tex_struct.parseLatexFile(filename_in) print(str(tex_struct)) nb = nbformat.notebooknode.NotebookNode({ u'nbformat_minor': 2, u'cells': [], u'nbformat': 4, u'metadata': {u'kernelspec': {u'display_name': u'Python 2', u'name': u'python2', u'language': u'python'}, u'toc': {u'nav_menu': {u'width': u'252px', u'height': u'12px'}, u'toc_window_display': False, u'widenNotebook': False, u'toc_section_display': u'block', u'colors': {u'hover_highlight': u'#DAA520', u'running_highlight': u'#FF0000', u'selected_highlight': u'#FFD700'}, u'navigate_menu': True, u'number_sections': True, u'threshold': 4, u'moveMenuLeft': True, u'sideBar': True, u'toc_cell': False}, u'hide_input': False, u'language_info': {u'mimetype': u'text/x-python', u'nbconvert_exporter': u'python', u'version': u'2.7.13', u'name': u'python', u'file_extension': u'.py', u'pygments_lexer': u'ipython2', u'codemirror_mode': { u'version': 2, u'name': u'ipython'}}}}) nb['cells'] = [] ################################################################ def createNewCell(): print('create new cell') new_cell = nbformat.notebooknode.NotebookNode() new_cell['cell_type'] = 'markdown' new_cell['source'] = '' new_cell['metadata'] = {} nb['cells'].append(new_cell) return new_cell ################################################################ current_cell = None for c in tex_struct.content: if current_cell is None: current_cell = createNewCell() if isinstance(c, LatexEnvironment): current_cell = createNewCell() print ('AAA:', type(c), str(c)) print ('adding:', type(c), str(c)) current_cell['source'] += str(c) # print(nb) # print(type(nb)) nbformat.write(nb, 'test.ipynb') diff --git a/scripts/slides b/scripts/slides index 4d0fa7c..9866465 100755 --- a/scripts/slides +++ b/scripts/slides @@ -1,110 +1,109 @@ #!/usr/bin/python3 -from __future__ import print_function """ This executable permits to generate an advanced presentation from a Jupyter notebook. The capabilities are: - a server of reveal content - a pptx (under developement) - a beamer tex file (TODO) """ ################################################################ import argparse import os import Slides.count_slides as count_slides import Slides.cell_manager as cell_manager ################################################################ def getNotebookFilenames(slides_file): """ When a directory is provided to the executable, a file named 'slides' is searched for with the expected list of ipython notebooks. The list of notebooks to consider is then returned. """ if not os.path.isfile(slides_file): raise Exception("Cannot find slides file") f = open(slides_file) slides = [l.strip() for l in f.readlines() if l != ""] return slides ################################################################ def main(): parser = argparse.ArgumentParser(description='Slide launcher') parser.add_argument( "--execute", action='store_true', help=('Requests to execute all the notebooks launching ' 'the slides server')) parser.add_argument("--no_count", action='store_false', help="Requests not to count slides generated") parser.add_argument("notebooks", nargs='*', help="The notebooks to associate in a presentation") parser.add_argument( "--internet", action='store_true', help="Weather the http server should serve on the internet") parser.add_argument("--html", action='store_true', help="only produce a html file") parser.add_argument("--beamer", action='store_true', help="produce a beamer/latex file and the pdf") parser.add_argument("--add_cell_number", action='store_true', help="add the cell number before each cell") parser.add_argument("--select_cells", type=int, nargs='*', help="Provides a list of cells to select") parser.add_argument("--font_size", type=str, default='25px', help="Provides the font size of the presentation") arguments = parser.parse_args() if len(arguments.notebooks) == 1: if os.path.isdir(arguments.notebooks[0]): slides_path = arguments.notebooks[0] slides_file = os.path.join(slides_path, 'slides') arguments.notebooks = getNotebookFilenames(slides_file) else: _file = arguments.notebooks[0] f, ext = os.path.splitext(_file) if ext == '.sld': arguments.notebooks = getNotebookFilenames(_file) if arguments.execute: for n in arguments.notebooks: cell_manager.executeNotebook(n) try: cell_manager.mergeNotebooks(arguments.notebooks, add_cell_number=arguments.add_cell_number, select_cells=arguments.select_cells) except cell_manager.NotFoundNotebooks: print('Could not find the notebooks to treat: check path provided') return if arguments.no_count: nb_slides = count_slides.countSlides(['talk.ipynb']) print("*" * 20) print("You generated a talk with {0} slides".format(nb_slides)) print("*" * 20) if arguments.html: cell_manager.generateHTML(font_size=arguments.font_size) elif arguments.beamer: cell_manager.generateBeamer(font_size=arguments.font_size) elif arguments.internet: cell_manager.launchSlideServer(options="--ServePostProcessor.ip='*'", font_size=arguments.font_size) else: cell_manager.launchSlideServer(font_size=arguments.font_size) ################################################################ if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index 606a939..f8e7ad1 100644 --- a/setup.py +++ b/setup.py @@ -1,45 +1,46 @@ from setuptools import setup, find_packages try: from sphinx.setup_command import BuildDoc cmdclass={'doc': BuildDoc} except Exception: cmdclass={} name = 'slides' version = '1.0' release = '1.0.0' setup(name=name, packages=find_packages(), version="0.0.1", author="Guillaume Anciaux", author_email="guillaume.anciaux@epfl.ch", description=("Slides making/convertion tools"), install_requires=["sphinx", "jupyter", "sympy", "pypdf2", - "matplotlib", "wand", "dill"], + "matplotlib", "wand", "dill", + "jupyter_contrib_nbextensions"], tests_requires=["pytest"], setup_requires=["pytest-runner"], test_suite="tests", cmdclass=cmdclass, command_options={ 'doc': { 'project': ('setup.py', name), 'version': ('setup.py', version), 'release': ('setup.py', release), 'source_dir': ('setup.py', 'doc')}}, scripts=['scripts/slides', 'scripts/pLatex'], license=""" This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . """) diff --git a/tests/__init__.py b/tests/__init__.py index bee0b98..97206f2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,3 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding:utf-8 -*- diff --git a/tests/pylint.rc b/tests/pylint.rc new file mode 100644 index 0000000..ad89695 --- /dev/null +++ b/tests/pylint.rc @@ -0,0 +1,378 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=getslice-method,filter-builtin-not-iterating,old-raise-syntax,metaclass-assignment,long-suffix,basestring-builtin,cmp-builtin,reduce-builtin,delslice-method,hex-method,map-builtin-not-iterating,round-builtin,apply-builtin,print-statement,execfile-builtin,long-builtin,buffer-builtin,suppressed-message,standarderror-builtin,using-cmp-argument,next-method-called,coerce-method,old-octal-literal,dict-iter-method,import-star-module-level,intern-builtin,backtick,cmp-method,useless-suppression,indexing-exception,file-builtin,old-division,unicode-builtin,old-ne-operator,zip-builtin-not-iterating,oct-method,no-absolute-import,nonzero-method,setslice-method,raw_input-builtin,input-builtin,unpacking-in-except,unichr-builtin,reload-builtin,range-builtin-not-iterating,xrange-builtin,coerce-builtin,parameter-unpacking,raising-string,dict-view-method + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))|print$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/tests/test_coding_convention.py b/tests/test_coding_convention.py new file mode 100644 index 0000000..cfe5072 --- /dev/null +++ b/tests/test_coding_convention.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import os +import pep8 +import Slides +import pytest +# import warnings +# from pylint import epylint as lint + + +@pytest.fixture +def source_files(): + """ + builds a list of source files. If you find a smarter way, please let me + know + """ + print() # for emacs to evaluate the first line of errors + mod_files = list() + for dirpath, _, filenames in os.walk(Slides.__path__[0]): + mod_files += [os.path.join(dirpath, filename) + for filename in filenames + if filename.endswith((".py", ".pyx"))] + return mod_files + + +def test_pep8_conformity(source_files): + """ + check all files for pep8 conformity + """ + pep8style = pep8.StyleGuide() + pep8style.check_files((mod_f for mod_f in source_files)) + + +# def test_pylint_bitchiness(source_files): +# print() # for emacs to evaluate the first line of errors +# options = ' --rcfile=tests/pylint.rc --disable=locally-disabled ' +# command_options = " ".join(source_files) + options +# with warnings.catch_warnings(): +# warnings.simplefilter("ignore") +# lint.py_run(command_options=command_options)