Page Menu
Home
c4science
Search
Configure Global Search
Log In
Files
F79689722
ttree_lex.py
No One
Temporary
Actions
Download File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Subscribers
None
File Metadata
Details
File Info
Storage
Attached
Created
Tue, Aug 27, 18:32
Size
80 KB
Mime Type
text/x-objective-c
Expires
Thu, Aug 29, 18:32 (1 d, 23 h)
Engine
blob
Format
Raw Data
Handle
20218713
Attached To
rLAMMPS lammps
ttree_lex.py
View Options
This document is not UTF8. It was detected as ISO-8859-1 (Latin 1) and converted to UTF8 for display.
# -*- coding: iso-8859-1 -*-
# Author: Andrew Jewett (jewett.aij at g mail)
# http://www.chem.ucsb.edu/~sheagroup
# License: 3-clause BSD License (See LICENSE.TXT)
# Copyright (c) 2012, Regents of the University of California
# All rights reserved.
"""A lexical analyzer class for simple shell-like syntaxes.
This version has been modified slightly to work better with unicode.
It was forked from the version of shlex that ships with python 3.2.2.
A few minor features and functions have been added. """
# Module and documentation by Eric S. Raymond, 21 Dec 1998
# Input stacking and error message cleanup added by ESR, March 2000
# push_source() and pop_source() made explicit by ESR, January 2001.
# Posix compliance, split(), string arguments, and
# iterator interface by Gustavo Niemeyer, April 2003.
# ("wordterminators" (unicode support) hack by Andrew Jewett September 2011)
import
os.path
import
sys
from
collections
import
deque
import
re
,
fnmatch
import
string
#import gc
try
:
from
cStringIO
import
StringIO
except
ImportError
:
try
:
from
StringIO
import
StringIO
except
ImportError
:
from
io
import
StringIO
__all__
=
[
"TtreeShlex"
,
"split"
,
"LineLex"
,
"SplitQuotedString"
,
"EscCharStrToChar"
,
"SafelyEncodeString"
,
"RemoveOuterQuotes"
,
"MaxLenStr"
,
"HasWildCard"
,
#"IsRegex",
"InputError"
,
"ErrorLeader"
,
"SrcLoc"
,
"OSrcLoc"
,
"TextBlock"
,
"VarRef"
,
"VarNPtr"
,
"VarBinding"
,
"SplitTemplate"
,
"SplitTemplateMulti"
,
"TableFromTemplate"
,
"ExtractCatName"
,
#"_TableFromTemplate",
#"_DeleteLineFromTemplate",
"DeleteLinesWithBadVars"
,
"TemplateLexer"
]
class
TtreeShlex
(
object
):
""" A lexical analyzer class for simple shell-like syntaxes.
TtreeShlex is a backwards-compatible version of python's standard shlex
module. It has the additional member: "self.wordterminators", which
overrides the "self.wordchars" member. This enables better handling of
unicode characters by allowing a much larger variety of characters to
appear in words or tokens parsed by TtreeShlex.
"""
custom_path
=
None
def
__init__
(
self
,
instream
=
None
,
infile
=
None
,
custom_include_path
=
None
,
posix
=
False
):
if
isinstance
(
instream
,
str
):
instream
=
StringIO
(
instream
)
if
instream
is
not
None
:
self
.
instream
=
instream
self
.
infile
=
infile
else
:
self
.
instream
=
sys
.
stdin
self
.
infile
=
None
self
.
posix
=
posix
if
posix
:
self
.
eof
=
None
else
:
self
.
eof
=
''
self
.
commenters
=
'#'
self
.
wordchars
=
(
'abcdfeghijklmnopqrstuvwxyz'
'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
)
if
self
.
posix
:
self
.
wordchars
+=
(
'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ'
)
self
.
wordterminators
=
set
([])
#WORDTERMINATORS
self
.
prev_space_terminator
=
''
#WORDTERMINATORS
self
.
whitespace
=
'
\t\r\f\n
'
self
.
whitespace_split
=
False
self
.
quotes
=
'
\'
"'
self
.
escape
=
'
\\
'
self
.
escapedquotes
=
'"'
self
.
state
=
' '
self
.
pushback
=
deque
()
self
.
lineno
=
1
self
.
debug
=
0
self
.
token
=
''
self
.
filestack
=
deque
()
# self.source_triggers
# are tokens which allow the seamless insertion of other
# files into the file being read.
self
.
source_triggers
=
set
([
'source'
])
self
.
source_triggers_x
=
set
([])
#Note: self.source_triggers_x
# This is a subset of self.source_triggers.
# In this case file inclusion is exclusive.
# In other words, if one of these tokens
# is encountered, the file is only included
# if it has not been included already.
self
.
source_files_restricted
=
set
([])
self
.
include_path
=
[]
if
TtreeShlex
.
custom_path
:
include_path_list
=
TtreeShlex
.
custom_path
.
split
(
':'
)
self
.
include_path
+=
[
d
for
d
in
include_path_list
if
len
(
d
)
>
0
]
if
'TTREE_PATH'
in
os
.
environ
:
include_path_list
=
os
.
environ
[
'TTREE_PATH'
]
.
split
(
':'
)
self
.
include_path
+=
[
d
for
d
in
include_path_list
if
len
(
d
)
>
0
]
if
self
.
debug
:
print
(
'TtreeShlex: reading from
%s
, line
%d
'
\
%
(
self
.
instream
,
self
.
lineno
))
self
.
end_encountered
=
False
@staticmethod
#WORDTERMINATORS
def
_belongs_to
(
char
,
include_chars
,
exclude_chars
):
#WORDTERMINATORS
if
((
not
exclude_chars
)
or
(
len
(
exclude_chars
)
==
0
)):
#WORDTERMINATORS
return
char
in
include_chars
#WORDTERMINATORS
else
:
#WORDTERMINATORS
return
char
not
in
exclude_chars
#WORDTERMINATORS
def
push_raw_text
(
self
,
text
):
"""Push a block of text onto the stack popped by the ReadLine() method.
(If multiple lines are present in the text, (which is determined by
self.line_terminators), then the text is split into multiple lines
and each one of them is pushed onto this stack individually.
The "self.lineno" counter is also adjusted, depending on the number
of newline characters in "line".
Do not strip off the newline, or other line terminators
at the end of the text block before using push_raw_text()!
"""
if
self
.
debug
>=
1
:
print
(
"TtreeShlex: pushing token "
+
repr
(
text
))
for
c
in
reversed
(
text
):
#WORDTERMINATORS
self
.
pushback
.
appendleft
(
c
)
#WORDTERMINATORS
if
c
==
'
\n
'
:
#WORDTERMINATORS
self
.
lineno
-=
1
#WORDTERMINATORS
if
len
(
text
)
>
0
:
#WORDTERMINATORS
self
.
end_encountered
=
False
#WORDTERMINATORS
def
push_token
(
self
,
text
):
"Push a token onto the stack popped by the get_token method"
self
.
push_raw_text
(
text
+
self
.
prev_space_terminator
)
def
push_source
(
self
,
newstream
,
newfile
=
None
):
"Push an input source onto the lexer's input source stack."
if
isinstance
(
newstream
,
str
):
newstream
=
StringIO
(
newstream
)
self
.
filestack
.
appendleft
((
self
.
infile
,
self
.
instream
,
self
.
lineno
))
self
.
infile
=
newfile
self
.
instream
=
newstream
self
.
lineno
=
1
if
self
.
debug
:
if
newfile
is
not
None
:
print
(
'TtreeShlex: pushing to file
%s
'
%
(
self
.
infile
,))
else
:
print
(
'TtreeShlex: pushing to stream
%s
'
%
(
self
.
instream
,))
def
pop_source
(
self
):
"Pop the input source stack."
self
.
instream
.
close
()
(
self
.
infile
,
self
.
instream
,
self
.
lineno
)
=
self
.
filestack
.
popleft
()
if
self
.
debug
:
print
(
'TtreeShlex: popping to
%s
, line
%d
'
\
%
(
self
.
instream
,
self
.
lineno
))
self
.
state
=
' '
def
get_token
(
self
):
"Get a token from the input stream (or from stack if it's nonempty)"
#### #CHANGING: self.pushback is now a stack of characters, not tokens #WORDTERMINATORS
#### if self.pushback: #WORDTERMINATORS
#### tok = self.pushback.popleft() #WORDTERMINATORS
#### if self.debug >= 1: #WORDTERMINATORS
#### print("TtreeShlex: popping token " + repr(tok)) #WORDTERMINATORS
#### return tok #WORDTERMINATORS
#### No pushback. Get a token. #WORDTERMINATORS
raw
=
self
.
read_token
()
# Handle inclusions
if
self
.
source_triggers
is
not
None
:
while
raw
in
self
.
source_triggers
:
fname
=
self
.
read_token
()
spec
=
self
.
sourcehook
(
fname
)
if
spec
:
(
newfile
,
newstream
)
=
spec
if
((
raw
not
in
self
.
source_triggers_x
)
or
(
newfile
not
in
self
.
source_files_restricted
)):
self
.
push_source
(
newstream
,
newfile
)
if
raw
in
self
.
source_triggers_x
:
self
.
source_files_restricted
.
add
(
newfile
)
else
:
if
self
.
debug
>=
0
:
sys
.
stderr
.
write
(
'
\n
debug warning: duplicate attempt to import file:
\n
\"
'
+
newfile
+
'
\"\n
'
)
raw
=
self
.
get_token
()
# Maybe we got EOF instead?
while
raw
==
self
.
eof
:
if
not
self
.
filestack
:
return
self
.
eof
else
:
self
.
pop_source
()
raw
=
self
.
get_token
()
# Neither inclusion nor EOF
if
self
.
debug
>=
1
:
if
raw
!=
self
.
eof
:
print
(
"TtreeShlex: token="
+
repr
(
raw
))
else
:
print
(
"TtreeShlex: token=EOF"
)
if
raw
==
self
.
eof
:
#WORDTERMINATORS
self
.
end_encountered
=
True
#WORDTERMINATORS
return
raw
def
read_char
(
self
):
if
self
.
pushback
:
#WORDTERMINATORS
nextchar
=
self
.
pushback
.
popleft
()
#WORDTERMINATORS
assert
((
type
(
nextchar
)
is
str
)
and
(
len
(
nextchar
)
==
1
))
#WORDTERMINATORS
else
:
#WORDTERMINATORS
nextchar
=
self
.
instream
.
read
(
1
)
#WORDTERMINATORS
return
nextchar
def
read_token
(
self
):
self
.
prev_space_terminator
=
''
#WORDTERMINATORS
quoted
=
False
escapedstate
=
' '
while
True
:
#### self.pushback is now a stack of characters, not tokens #WORDTERMINATORS
nextchar
=
self
.
read_char
()
if
nextchar
==
'
\n
'
:
self
.
lineno
=
self
.
lineno
+
1
if
self
.
debug
>=
3
:
print
(
"TtreeShlex: in state"
,
repr
(
self
.
state
),
\
"I see character:"
,
repr
(
nextchar
))
if
self
.
state
is
None
:
self
.
token
=
''
# past end of file
break
elif
self
.
state
==
' '
:
if
not
nextchar
:
self
.
state
=
None
# end of file
break
elif
nextchar
in
self
.
whitespace
:
if
self
.
debug
>=
2
:
print
(
"TtreeShlex: I see whitespace in whitespace state"
)
if
self
.
token
or
(
self
.
posix
and
quoted
):
# Keep track of which whitespace
# character terminated the token.
self
.
prev_space_terminator
=
nextchar
#WORDTERMINATORS
break
# emit current token
else
:
continue
elif
nextchar
in
self
.
commenters
:
self
.
instream
.
readline
()
self
.
lineno
=
self
.
lineno
+
1
elif
self
.
posix
and
nextchar
in
self
.
escape
:
escapedstate
=
'a'
self
.
state
=
nextchar
elif
TtreeShlex
.
_belongs_to
(
nextchar
,
#WORDTERMINATORS
self
.
wordchars
,
#WORDTERMINATORS
self
.
wordterminators
):
#WORDTERMINATORS
self
.
token
=
nextchar
self
.
state
=
'a'
elif
nextchar
in
self
.
quotes
:
if
not
self
.
posix
:
self
.
token
=
nextchar
self
.
state
=
nextchar
elif
self
.
whitespace_split
:
self
.
token
=
nextchar
self
.
state
=
'a'
else
:
self
.
token
=
nextchar
if
self
.
token
or
(
self
.
posix
and
quoted
):
break
# emit current token
else
:
continue
elif
self
.
state
in
self
.
quotes
:
quoted
=
True
if
not
nextchar
:
# end of file
if
self
.
debug
>=
2
:
print
(
"TtreeShlex: I see EOF in quotes state"
)
# XXX what error should be raised here?
raise
ValueError
(
"Error at or before "
+
self
.
error_leader
()
+
"
\n
"
" No closing quotation."
)
if
nextchar
==
self
.
state
:
if
not
self
.
posix
:
self
.
token
=
self
.
token
+
nextchar
self
.
state
=
' '
break
else
:
self
.
state
=
'a'
elif
self
.
posix
and
nextchar
in
self
.
escape
and
\
self
.
state
in
self
.
escapedquotes
:
escapedstate
=
self
.
state
self
.
state
=
nextchar
else
:
self
.
token
=
self
.
token
+
nextchar
elif
self
.
state
in
self
.
escape
:
if
not
nextchar
:
# end of file
if
self
.
debug
>=
2
:
print
(
"TtreeShlex: I see EOF in escape state"
)
# XXX what error should be raised here?
raise
ValueError
(
"No escaped character"
)
# In posix shells, only the quote itself or the escape
# character may be escaped within quotes.
if
escapedstate
in
self
.
quotes
and
\
nextchar
!=
self
.
state
and
nextchar
!=
escapedstate
:
self
.
token
=
self
.
token
+
self
.
state
self
.
token
=
self
.
token
+
nextchar
self
.
state
=
escapedstate
elif
self
.
state
==
'a'
:
if
not
nextchar
:
self
.
state
=
None
# end of file
break
elif
nextchar
in
self
.
whitespace
:
if
self
.
debug
>=
2
:
print
(
"TtreeShlex: I see whitespace in word state"
)
self
.
state
=
' '
if
self
.
token
or
(
self
.
posix
and
quoted
):
# Keep track of which whitespace
# character terminated the token.
self
.
prev_space_terminator
=
nextchar
#WORDTERMINATORS
break
# emit current token
else
:
continue
elif
nextchar
in
self
.
commenters
:
comment_contents
=
self
.
instream
.
readline
()
self
.
lineno
=
self
.
lineno
+
1
if
self
.
posix
:
self
.
state
=
' '
if
self
.
token
or
(
self
.
posix
and
quoted
):
# Keep track of which character(s) terminated
# the token (including whitespace and comments).
self
.
prev_space_terminator
=
nextchar
+
comment_contents
#WORDTERMINATORS
break
# emit current token
else
:
continue
elif
self
.
posix
and
nextchar
in
self
.
quotes
:
self
.
state
=
nextchar
elif
self
.
posix
and
nextchar
in
self
.
escape
:
escapedstate
=
'a'
self
.
state
=
nextchar
elif
(
TtreeShlex
.
_belongs_to
(
nextchar
,
#WORDTERMINATORS
self
.
wordchars
,
#WORDTERMINATORS
self
.
wordterminators
)
#WORDTERMINATORS
or
(
nextchar
in
self
.
quotes
)
#WORDTERMINATORS
or
(
self
.
whitespace_split
)):
#WORDTERMINATORS
self
.
token
=
self
.
token
+
nextchar
else
:
self
.
pushback
.
appendleft
(
nextchar
)
if
self
.
debug
>=
2
:
print
(
"TtreeShlex: I see punctuation in word state"
)
self
.
state
=
' '
if
self
.
token
:
break
# emit current token
else
:
continue
result
=
self
.
token
self
.
token
=
''
if
self
.
posix
and
not
quoted
and
result
==
''
:
result
=
None
if
self
.
debug
>
1
:
if
result
:
print
(
"TtreeShlex: raw token="
+
repr
(
result
))
else
:
print
(
"TtreeShlex: raw token=EOF"
)
return
result
def
sourcehook
(
self
,
newfile
):
"Hook called on a filename to be sourced."
newfile
=
RemoveOuterQuotes
(
newfile
)
# This implements cpp-like semantics for relative-path inclusion.
if
isinstance
(
self
.
infile
,
str
)
and
not
os
.
path
.
isabs
(
newfile
):
newfile_full
=
os
.
path
.
join
(
os
.
path
.
dirname
(
self
.
infile
),
newfile
)
try
:
f
=
open
(
newfile_full
,
"r"
)
except
IOError
:
# If not found,
err
=
True
# ...then check to see if the file is in one of the
# directories in the self.include_path list.
for
d
in
self
.
include_path
:
newfile_full
=
os
.
path
.
join
(
d
,
newfile
)
try
:
f
=
open
(
newfile_full
,
"r"
)
err
=
False
break
except
IOError
:
err
=
True
if
err
:
raise
InputError
(
'Error at '
+
self
.
error_leader
()
+
'
\n
'
' unable to open file
\"
'
+
newfile
+
'
\"\n
'
' for reading.
\n
'
)
return
(
newfile
,
f
)
def
error_leader
(
self
,
infile
=
None
,
lineno
=
None
):
"Emit a C-compiler-like, Emacs-friendly error-message leader."
if
infile
is
None
:
infile
=
self
.
infile
if
lineno
is
None
:
lineno
=
self
.
lineno
return
"
\"
%s
\"
, line
%d
: "
%
(
infile
,
lineno
)
def
__iter__
(
self
):
return
self
def
__next__
(
self
):
token
=
self
.
get_token
()
if
token
==
self
.
eof
:
raise
StopIteration
return
token
def
__bool__
(
self
):
return
not
self
.
end_encountered
# For compatibility with python 2.x, I must also define:
def
__nonzero__
(
self
):
return
self
.
__bool__
()
# The split() function was originally from shlex
# It is included for backwards compatibility.
def
split
(
s
,
comments
=
False
,
posix
=
True
):
lex
=
TtreeShlex
(
s
,
posix
=
posix
)
lex
.
whitespace_split
=
True
if
not
comments
:
lex
.
commenters
=
''
return
list
(
lex
)
##################### NEW ADDITIONS (may be removed later) #################
#"""
# -- linelex.py --
#linelex.py defines the LineLex class, which inherits from, and further
#augments the capabilities of TtreeShlex by making it easier to parse
#individual lines one at a time. (The original shlex's "source" inclusion
#ability still works when reading entire lines, and lines are still counted.)
#
#"""
#import sys
class
InputError
(
Exception
):
""" A generic exception object containing a string for error reporting.
(Raising this exception implies that the caller has provided
a faulty input file or argument.)
"""
def
__init__
(
self
,
err_msg
):
self
.
err_msg
=
err_msg
def
__str__
(
self
):
return
self
.
err_msg
def
__repr__
(
self
):
return
str
(
self
)
def
ErrorLeader
(
infile
,
lineno
):
return
'
\"
'
+
infile
+
'
\"
, line '
+
str
(
lineno
)
class
SrcLoc
(
object
):
""" SrcLoc is essentially nothing more than a 2-tuple containing the name
of a file (str) and a particular line number inside that file (an integer).
"""
def
__init__
(
self
,
infile
=
''
,
lineno
=-
1
):
self
.
infile
=
infile
self
.
lineno
=
lineno
def
SplitQuotedString
(
string
,
quotes
=
'
\'\"
'
,
delimiters
=
'
\t\r\f\n
'
,
escape
=
'
\\
'
,
comment_char
=
'#'
):
tokens
=
[]
token
=
''
reading_token
=
True
escaped_state
=
False
quote_state
=
None
for
c
in
string
:
if
(
c
in
comment_char
)
and
(
not
escaped_state
)
and
(
quote_state
==
None
):
tokens
.
append
(
token
)
return
tokens
elif
(
c
in
delimiters
)
and
(
not
escaped_state
)
and
(
quote_state
==
None
):
if
reading_token
:
tokens
.
append
(
token
)
token
=
''
reading_token
=
False
elif
c
in
escape
:
if
escaped_state
:
token
+=
c
reading_token
=
True
escaped_state
=
False
else
:
escaped_state
=
True
# and leave c (the '\' character) out of token
elif
(
c
in
quotes
)
and
(
not
escaped_state
):
if
(
quote_state
!=
None
):
if
(
c
==
quote_state
):
quote_state
=
None
else
:
quote_state
=
c
token
+=
c
reading_token
=
True
else
:
if
(
c
==
'n'
)
and
(
escaped_state
==
True
):
c
=
'
\n
'
elif
(
c
==
't'
)
and
(
escaped_state
==
True
):
c
=
'
\t
'
elif
(
c
==
'r'
)
and
(
escaped_state
==
True
):
c
=
'
\r
'
elif
(
c
==
'f'
)
and
(
escaped_state
==
True
):
c
=
'
\f
'
token
+=
c
reading_token
=
True
escaped_state
=
False
if
len
(
string
)
>
0
:
tokens
.
append
(
token
)
return
tokens
def
EscCharStrToChar
(
s_in
,
escape
=
'
\\
'
):
"""
EscCharStrToChar() replaces any escape sequences
in a string with their 1-character equivalents.
"""
assert
(
len
(
escape
)
>
0
)
out_lstr
=
[]
escaped_state
=
False
for
c
in
s_in
:
if
escaped_state
:
if
(
c
==
'n'
):
out_lstr
.
append
(
'
\n
'
)
elif
(
c
==
't'
):
out_lstr
.
append
(
'
\t
'
)
elif
(
c
==
'r'
):
out_lstr
.
append
(
'
\r
'
)
elif
(
c
==
'f'
):
out_lstr
.
append
(
'
\f
'
)
elif
(
c
==
'
\'
'
):
out_lstr
.
append
(
'
\'
'
)
elif
(
c
==
'
\"
'
):
out_lstr
.
append
(
'
\"
'
)
elif
c
in
escape
:
out_lstr
.
append
(
c
)
else
:
out_lstr
.
append
(
escape
+
c
)
# <- keep both characters
escaped_state
=
False
else
:
if
c
in
escape
:
escaped_state
=
True
else
:
out_lstr
.
append
(
c
)
return
''
.
join
(
out_lstr
)
def
SafelyEncodeString
(
in_str
,
quotes
=
'
\'\"
'
,
delimiters
=
'
\t\r\f\n
'
,
escape
=
'
\\
'
,
comment_char
=
'#'
):
"""
SafelyEncodeString(in_str) scans through the input string (in_str),
and returns a new string in which probletic characters
(like newlines, tabs, quotes, etc), are replaced by their two-character
backslashed equivalents (like '\n', '\t', '\'', '\"', etc).
The escape character is the backslash by default, but it too can be
overridden to create custom escape sequences
(but this does not effect the encoding for characters like '\n', '\t').
"""
assert
(
len
(
escape
)
>
0
)
out_lstr
=
[]
use_outer_quotes
=
False
for
c
in
in_str
:
if
(
c
==
'
\n
'
):
c
=
'
\\
n'
elif
(
c
==
'
\t
'
):
c
=
'
\\
t'
elif
(
c
==
'
\r
'
):
c
=
'
\\
r'
elif
(
c
==
'
\f
'
):
c
=
'
\\
f'
elif
c
in
quotes
:
c
=
escape
[
0
]
+
c
elif
c
in
escape
:
c
=
c
+
c
elif
c
in
delimiters
:
use_outer_quotes
=
True
# hmm... that's all that comes to mind. Did I leave anything out?
out_lstr
.
append
(
c
)
if
use_outer_quotes
:
out_lstr
=
[
'
\"
'
]
+
out_lstr
+
[
'
\"
'
]
return
''
.
join
(
out_lstr
)
def
RemoveOuterQuotes
(
text
,
quotes
=
'
\"\'
'
):
if
((
len
(
text
)
>=
2
)
and
(
text
[
0
]
in
quotes
)
and
(
text
[
-
1
]
==
text
[
0
])):
return
text
[
1
:
-
1
]
else
:
return
text
def
MaxLenStr
(
s1
,
s2
):
if
len
(
s2
)
>
len
(
s1
):
return
s2
else
:
return
s1
#def IsRegex(pat):
# """
# Check to see if string (pat) is bracketed by slashes.
#
# """
# return (len(pat)>=2) and (pat[0]=='/') and (pat[-1] == '/')
def
HasWildCard
(
pat
):
"""
Returns true if a string (pat) contains a '*' or '?' character.
"""
return
(
pat
.
find
(
'*'
)
!=
-
1
)
or
(
pat
.
find
(
'?'
)
!=
-
1
)
#def HasWildCard(pat):
# """
# Returns true if a string (pat) contains a non-backslash-protected
# * or ? character.
#
# """
# N=len(pat)
# i=0
# while i < N:
# i = pat.find('*', i, N)
# if i == -1:
# break
# elif (i==0) or (pat[i-1] != '\\'):
# return True
# i += 1
# i=0
# while i < N:
# i = pat.find('?', i, N)
# if i == -1:
# break
# elif (i==0) or (pat[i-1] != '\\'):
# return True
# i += 1
# return False
def
MatchesPattern
(
s
,
pattern
):
if
type
(
pattern
)
is
str
:
#old code:
#if ((len(s) > 1) and (s[0] == '/') and (s[-1] == '/'):
# re_string = p[1:-1] # strip off the slashes '/' and '/'
# if not re.search(re_string, s):
# return False
#new code:
# uses precompiled regular expressions (See "pattern.search" below)
if
HasWildCard
(
pattern
):
if
not
fnmatch
.
fnmatchcase
(
s
,
pattern
):
return
False
elif
s
!=
pattern
:
return
False
else
:
#assert(type(p) is _sre.SRE_Match)
# I assume pattern = re.compile(some_reg_expr)
if
not
pattern
.
search
(
s
):
return
False
return
True
def
MatchesAll
(
multi_string
,
pattern
):
assert
(
len
(
multi_string
)
==
len
(
pattern
))
for
i
in
range
(
0
,
len
(
pattern
)):
if
not
MatchesPattern
(
multi_string
[
i
],
pattern
[
i
]):
return
False
return
True
class
LineLex
(
TtreeShlex
):
""" This class extends the TtreeShlex module (a slightly modified
version of the python 3.2.2 version of shlex). LineLex has the
ability to read one line at a time (in addition to one token at a time).
(Many files and scripts must be parsed one line at a time instead of one
token at a time. In these cases, the whitespace position also matters.)
Arguably, this class might not be necessary.
I could get rid of this class completely. That would be nice. To do that
we would need to augment and generalize shlex's get_token() member function
to make it read lines, not just tokens. Of course, you can always
change the wordchars (or wordterminators). Even so, there are two other
difficulties using the current version of shlex.get_token() to read lines:
1) File inclusion happen whenever the beginning of a line/token matches one
of the "source_triggers" (not the whole line as required by get_token()).
2) Lines ending in a special character (by default the backslash character)
continue on to the next line.
This code seems to work on our test files, but I'm sure there are bugs.
Andrew 2012-3-25
"""
def
__init__
(
self
,
instream
=
None
,
infile
=
None
,
posix
=
False
):
TtreeShlex
.
__init__
(
self
,
instream
,
infile
,
posix
)
self
.
line_terminators
=
'
\n
'
self
.
line_extend_chars
=
'
\\
'
self
.
skip_comments_during_readline
=
True
def
_StripComments
(
self
,
line
):
if
self
.
skip_comments_during_readline
:
for
i
in
range
(
0
,
len
(
line
)):
if
((
line
[
i
]
in
self
.
commenters
)
and
((
i
==
0
)
or
(
line
[
i
-
1
]
not
in
self
.
escape
))):
return
line
[:
i
]
return
line
def
_ReadLine
(
self
,
recur_level
=
0
):
"""
This function retrieves a block of text, halting at a
terminal character. Escape sequences are respected.
The self.lineno (newline counter) is also maintained.
The main difference between Readline and get_token()
is the way they handle the "self.source_triggers" member.
Both Readline() and get_token() insert text from other files when they
encounter a string in "self.source_triggers" in the text they read.
However ReadLine() ONLY inserts text from other files if the token which
matches with self.source_triggers appears at the beginning of the line.
get_token() inserts text only if lex.source matches the entire token.
comment-to-self:
At some point, once I'm sure this code is working, I should replace
shlex.get_token() with the code from ReadLine() which is more general.
It would be nice to get rid of "class LineLex" entirely. ReadLine()
is the only new feature that LineLex which was lacking in shlex.
To do this I would need to add a couple optional arguments to
"get_token()", allowing it to mimic ReadLine(), such as:
"override_wordterms" argument (which we can pass a '\n'), and
"token_extender" argument (like '\' for extending lines)
"""
first_token
=
''
line
=
''
escaped_state
=
False
found_space
=
False
while
True
:
nextchar
=
self
.
read_char
()
#sys.stderr.write('nextchar=\"'+nextchar+'\"\n')
while
nextchar
==
''
:
if
not
self
.
filestack
:
return
self
.
_StripComments
(
line
),
''
,
first_token
,
found_space
else
:
self
.
pop_source
()
nextchar
=
self
.
read_char
()
if
nextchar
==
'
\n
'
:
self
.
lineno
+=
1
if
escaped_state
:
escaped_state
=
False
else
:
if
nextchar
in
self
.
escape
:
line
+=
nextchar
escaped_state
=
True
else
:
escaped_state
=
False
if
not
escaped_state
:
if
(
nextchar
in
self
.
whitespace
):
found_space
=
True
while
first_token
in
self
.
source_triggers
:
fname
=
RemoveOuterQuotes
(
self
.
get_token
())
if
(
fname
==
''
)
or
(
fname
in
self
.
source_triggers
):
raise
InputError
(
'Error: near '
+
self
.
error_leader
()
+
'
\n
'
' Nonsensical file inclusion request.
\n
'
)
if
self
.
debug
>=
0
:
sys
.
stderr
.
write
(
(
' '
*
recur_level
)
+
'reading file
\"
'
+
fname
+
'
\"\n
'
)
spec
=
self
.
sourcehook
(
fname
)
if
spec
:
(
fname
,
subfile
)
=
spec
if
((
first_token
not
in
self
.
source_triggers_x
)
or
(
fname
not
in
self
.
source_files_restricted
)):
self
.
push_source
(
subfile
,
fname
)
if
first_token
in
self
.
source_triggers_x
:
self
.
source_files_restricted
.
add
(
fname
)
else
:
if
self
.
debug
>=
0
:
sys
.
stderr
.
write
(
'
\n
Warning at '
+
self
.
error_leader
()
+
':
\n
'
' duplicate attempt to import file:
\n
\"
'
+
fname
+
'
\"\n
'
)
line
,
nextchar
,
first_token
,
found_space
=
\
self
.
_ReadLine
(
recur_level
+
1
)
if
nextchar
in
self
.
line_terminators
:
line_nrw
=
line
.
rstrip
(
self
.
whitespace
)
#sys.stderr.write('line_nrw=\"'+line_nrw+'\"\n')
if
((
len
(
line_nrw
)
>
0
)
and
(
line_nrw
[
-
1
]
in
self
.
line_extend_chars
)
and
((
len
(
line_nrw
)
<
2
)
or
(
line_nrw
[
-
2
]
not
in
self
.
escape
))):
line
=
line_nrw
[:
-
1
]
#delete the line_extend character
# from the end of that line and keep reading...
else
:
return
self
.
_StripComments
(
line
),
nextchar
,
first_token
,
found_space
else
:
line
+=
nextchar
if
not
found_space
:
first_token
+=
nextchar
def
ReadLine
(
self
,
recur_level
=
0
):
line
,
nextchar
,
first_token
,
found_space
=
\
self
.
_ReadLine
(
recur_level
)
if
nextchar
==
self
.
eof
:
self
.
end_encountered
=
True
return
line
+
nextchar
@staticmethod
def
TextBlock2Lines
(
text
,
delimiters
,
keep_delim
=
True
):
""" This splits a string into a list of sub-strings split by delimiter
characters. This function is different from the standard str.split()
function: The string is split at every character which belongs to the
"delimiters" argument (which can be a string or some other container).
This character is included at the end of every substring. Example:
TextBlock2Lines('\nabc\nde^fg\nhi j\n', '^\n')
returns:
['\n', 'abc\n', 'de^', 'fg\n', 'hi j\n']
"""
ls
=
[]
i
=
0
i_prev
=
0
while
i
<
len
(
text
):
if
text
[
i
]
in
delimiters
:
if
keep_delim
:
ls
.
append
(
text
[
i_prev
:
i
+
1
])
else
:
ls
.
append
(
text
[
i_prev
:
i
])
i_prev
=
i
+
1
i
+=
1
if
(
i_prev
<
len
(
text
)):
ls
.
append
(
text
[
i_prev
:
i
+
1
])
return
ls
def
__iter__
(
self
):
return
self
def
__next__
(
self
):
line
=
self
.
ReadLine
()
if
line
==
self
.
eof
:
raise
StopIteration
return
line
class
OSrcLoc
(
object
):
""" OSrcLoc is barely more than a 2-tuple containing the name of a file
(a string) and a particular line number inside that file (an integer).
These objects are passed around and stored in the nodes of
every tree, so that if a syntax error or broken link in that node
is discovered, an error message can be provided to the user.
"order"
Later on, during development, the "order" member was added. Why:
If you want to know whether block of text comes before or after a
different block of text, unfortunately you can not just compare the
corresponding line numbers of the files they come from because the
files may differ, and multiple short blocks of text may occupy the
same line. Consequently, "OSrcLoc" also maintains an internal
counter which keeps track of how many OSrcLoc() objects have been
created so far. (This can be useful if the user requests that
variables and commands be assigned in a non-standard order.)
The "order" member is assigned to this counter.
Most of the time, the "order" member can be ignored.
"""
count
=
0
def
__init__
(
self
,
infile
=
''
,
lineno
=-
1
):
self
.
infile
=
infile
self
.
lineno
=
lineno
OSrcLoc
.
count
+=
1
self
.
order
=
OSrcLoc
.
count
def
__lt__
(
self
,
x
):
return
self
.
order
<
x
.
order
#def __repr__(self):
# return repr((self.infile, self.lineno, self.order))
class
TextBlock
(
object
):
"""TextBlock is just a 3-tuple consisting of a string, and an OSrcLoc
to help locate it in the original file from which it was read."""
def
__init__
(
self
,
text
,
srcloc
):
#srcloc_end):
self
.
text
=
text
if
srcloc
==
None
:
self
.
srcloc
=
OSrcLoc
()
else
:
self
.
srcloc
=
srcloc
#if srcloc_end == None:
# self.srcloc_end = OSrcLoc()
#else:
# self.srcloc_end = srcloc_end
def
__repr__
(
self
):
return
'
\"
'
+
self
.
text
+
'
\"
'
class
VarRef
(
object
):
"""VarRef stores variable names, and paths, and other attribute information,
as well as a "OSrcLoc" to keep track of the file it was defined in."""
def
__init__
(
self
,
prefix
=
''
,
# '$' or '${'
descr_str
=
''
,
# <- descriptor string: "cpath/category:lpath"
suffix
=
''
,
# '}'
srcloc
=
None
,
# location in file where defined
binding
=
None
,
# a pointer to a tuple storing the value
nptr
=
None
):
# <- see class VarNPtr
self
.
prefix
=
prefix
#Any text before the descriptor string goes here
self
.
suffix
=
suffix
#Any text after the descriptor string goes here
self
.
descr_str
=
descr_str
if
srcloc
==
None
:
# <- Location in text file where variable appears
self
.
srcloc
=
OSrcLoc
()
else
:
self
.
srcloc
=
srcloc
self
.
binding
=
binding
if
nptr
==
None
:
self
.
nptr
=
VarNPtr
()
else
:
self
.
nptr
=
nptr
def
__lt__
(
self
,
x
):
return
self
.
order
<
x
.
order
#def __repr__(self):
# return repr((self.prefix + self.descr_str + self.suffix, srcloc))
class
VarNPtr
(
object
):
"""
Every time a variable appears in a template, it has has a "descritpor".
For example, consider the variable
"$atom:CA"
This is a string which encodes 3 pieces of information.
1) the category name: This is essentialy indicates the variable's type.
(ie "atom", in the example above)
2) the category node: Some TYPES have limited scope. Users can
specify the root node of the portion of the tree
in which this variable's type makes sense.
If this node is the root node, then that category
is relevant everywhere, and is not molecule or class
specific. All variables have a category node, which
is often not explicitly defined to by the user.
It must be inferred/determined.)
(Category node = the root "/", in the example above.)
3) the leaf node: This is a node whose ".name" member matches the name
of a variable. This node is created for this purpose
and it's position in the tree is a reflection of
that variable's intended scope.
In a molecule this "name" might be the name
of a type of atom, or an atom ID, or a bond type,
which is found in a particular molecule.
(Leaf node would be named "CA" in the example above.)
The VarNPtr class is simply a 3-tuple which
keeps these 3 pieces of data together.
"""
def
__init__
(
self
,
cat_name
=
''
,
cat_node
=
None
,
leaf_node
=
None
):
self
.
cat_name
=
cat_name
self
.
cat_node
=
cat_node
self
.
leaf_node
=
leaf_node
#def __repr__(self):
# return repr((self.cat_name, self.cat_node.name, self.leaf_node.name))
class
VarBinding
(
object
):
""" VarBinding is essentially a tuple consistng of (full_name, binding, refs):
"self.full_name" is canonical name for this variable. This is a string
which specifies full path leading to the category node (beginning with '/'),
the category name (followed by a ':'),
as well as the leaf node (including the path leading up to it from cat_node)
This triplet identifies the variable uniquely.
"self.value" is the data that the variable refers to (usually a string).
"self.refs" stores a list of VarRefs which mention the same variable
from the various places inside various templates in the tree.
"""
def
__init__
(
self
,
full_name
=
''
,
nptr
=
None
,
value
=
None
,
refs
=
None
,
order
=
None
,
category
=
None
):
self
.
full_name
=
full_name
self
.
nptr
=
nptr
self
.
value
=
value
self
.
refs
=
refs
self
.
order
=
order
self
.
category
=
category
def
__lt__
(
self
,
x
):
return
self
.
order
<
x
.
order
def
__repr__
(
self
):
return
repr
((
self
.
full_name
,
self
.
value
,
self
.
order
))
def
ExtractCatName
(
descr_str
):
""" When applied to a VarRef's "descr_str" member,
this function will extract the "catname" of it's corresponding
"nptr" member. This can be useful for error reporting.
(I use it to insure that the user is using the correct counter
variable types at various locations in their input files.)
"""
ib
=
descr_str
.
find
(
':'
)
if
ib
==
-
1
:
ib
=
len
(
descr_str
)
ia
=
descr_str
.
rfind
(
'/'
)
if
ia
==
-
1
:
ia
=
0
return
descr_str
[
ia
:
ib
]
else
:
str_before_colon
=
descr_str
[
0
:
ib
]
ia
=
str_before_colon
.
rfind
(
'/'
)
if
ia
==
-
1
:
return
str_before_colon
else
:
return
str_before_colon
[
ia
+
1
:]
def
_DeleteLineFromTemplate
(
tmpl_list
,
i_entry
,
# index into tmpl_list
newline_delimiter
=
'
\n
'
):
""" Delete a single line from tmpl_list.
tmpl_list is an alternating list of VarRefs and TextBlocks.
To identify the line, the index corresponding to one of the
entries in the tmpl_list is used. (Usually it is a VarRef)
The text after the preceeding newline, and the text up to the next newline
(starting from the beginning of the current entry, if a TextBlock)
is deleted, including any VarRef (variables) located in between.
It returns the index corresponding to the next
entry in the list (after deletion).
"""
i_prev_newline
=
i_entry
while
i_prev_newline
>=
0
:
entry
=
tmpl_list
[
i_prev_newline
]
if
isinstance
(
entry
,
TextBlock
):
i_char_newline
=
entry
.
text
.
rfind
(
newline_delimiter
)
if
i_char_newline
!=
-
1
:
# then newline found
# Delete the text after this newline
entry
.
text
=
entry
.
text
[:
i_char_newline
+
1
]
break
i_prev_newline
-=
1
first_var
=
True
#i_next_newline = i_entry
i_next_newline
=
i_prev_newline
+
1
while
i_next_newline
<
len
(
tmpl_list
):
entry
=
tmpl_list
[
i_next_newline
]
if
isinstance
(
entry
,
TextBlock
):
i_char_newline
=
entry
.
text
.
find
(
newline_delimiter
)
if
i_char_newline
!=
-
1
:
# then newline found
# Delete the text before this newline (including the newline)
entry
.
text
=
entry
.
text
[
i_char_newline
+
1
:]
break
# Invoke DeleteSelf() on the first variables on this line. This will
# insure that it is deleted from the ttree_assignments.txt file.
elif
isinstance
(
entry
,
VarRef
):
if
first_var
:
entry
.
nptr
.
leaf_node
.
DeleteSelf
()
first_var
=
False
i_next_newline
+=
1
del
tmpl_list
[
i_prev_newline
+
1
:
i_next_newline
]
return
i_prev_newline
+
1
def
DeleteLinesWithBadVars
(
tmpl_list
,
delete_entire_template
=
False
,
newline_delimiter
=
'
\n
'
):
"""
Loop through the entries in a template,
an alternating list of TextBlocks and VarRefs (tmpl_list).
If a VarRef points to a leaf_node which no longer exists
(ie. no longer in the corresponding category's .bindings list).
Then delete the line it came from from the template (tmpl_list).
"""
out_str_list
=
[]
i
=
0
while
i
<
len
(
tmpl_list
):
entry
=
tmpl_list
[
i
]
if
isinstance
(
entry
,
VarRef
):
var_ref
=
entry
var_bindings
=
var_ref
.
nptr
.
cat_node
.
categories
[
var_ref
.
nptr
.
cat_name
]
.
bindings
#if var_ref.nptr.leaf_node not in var_bindings:
if
var_ref
.
nptr
.
leaf_node
.
IsDeleted
():
if
delete_entire_template
:
del
tmpl_list
[:]
return
0
else
:
i
=
_DeleteLineFromTemplate
(
tmpl_list
,
i
,
newline_delimiter
)
else
:
i
+=
1
else
:
i
+=
1
def
SplitTemplate
(
ltmpl
,
delim
,
delete_blanks
=
False
):
"""
Split a template "ltmpl" into a list of "tokens" (sub-templates)
using a single delimiter string "delim".
INPUT arguments:
"ltmpl" should be an list of TextBlocks and VarRefs.
"delim" should be a simple string (type str)
"delete_blanks" should be a boolean True/False value.
When true, successive occurrences of the delimiter
should not create blank entries in the output list.
OUTPUT:
A list of tokens.
Each "token" is either a TextBlock, a VarRef,
or a (flat, 1-dimensional) list containing more than one of these objects.
The number of "tokens" returned equals the number of times the delimiter
is encountered in any of the TextBlocks in the "ltmpl" argument, plus one.
(... Unless "delete_blanks" is set to True.
Again, in that case, empty entries in this list are deleted.)
"""
assert
(
type
(
delim
)
is
str
)
if
not
hasattr
(
ltmpl
,
'__len__'
):
ltmpl
=
[
ltmpl
]
tokens_lltmpl
=
[]
token_ltmpl
=
[]
i
=
0
while
i
<
len
(
ltmpl
):
entry
=
ltmpl
[
i
]
if
isinstance
(
entry
,
TextBlock
):
#if hasattr(entry, 'text'):
prev_src_loc
=
entry
.
srcloc
tokens_str
=
entry
.
text
.
split
(
delim
)
lineno
=
entry
.
srcloc
.
lineno
j
=
0
while
j
<
len
(
tokens_str
):
token_str
=
tokens_str
[
j
]
delim_found
=
False
if
(
j
<
len
(
tokens_str
)
-
1
):
delim_found
=
True
if
token_str
==
''
:
if
delete_blanks
:
if
delim
==
'
\n
'
:
lineno
+=
1
if
len
(
token_ltmpl
)
>
0
:
if
len
(
token_ltmpl
)
==
1
:
tokens_lltmpl
.
append
(
token_ltmpl
[
0
])
else
:
tokens_lltmpl
.
append
(
token_ltmpl
)
del
token_ltmpl
token_ltmpl
=
[]
j
+=
1
continue
new_src_loc
=
OSrcLoc
(
prev_src_loc
.
infile
,
lineno
)
new_src_loc
.
order
=
prev_src_loc
.
order
for
c
in
token_str
:
# Reminder to self: c != delim (so c!='\n' if delim='\n')
# (We keep track of '\n' characters in delimiters above.)
if
c
==
'
\n
'
:
lineno
+=
1
new_src_loc
.
lineno
=
lineno
text_block
=
TextBlock
(
token_str
,
new_src_loc
)
prev_src_loc
=
new_src_loc
if
len
(
token_ltmpl
)
==
0
:
if
delim_found
:
tokens_lltmpl
.
append
(
text_block
)
del
token_ltmpl
token_ltmpl
=
[]
else
:
token_ltmpl
.
append
(
text_block
)
else
:
if
delim_found
:
if
len
(
token_str
)
>
0
:
token_ltmpl
.
append
(
text_block
)
tokens_lltmpl
.
append
(
token_ltmpl
)
del
token_ltmpl
token_ltmpl
=
[]
else
:
assert
(
not
delete_blanks
)
if
(
isinstance
(
token_ltmpl
[
-
1
],
VarRef
)
and
((
j
>
0
)
or
((
j
==
len
(
tokens_str
)
-
1
)
and
(
i
==
len
(
ltmpl
)
-
1
))
)):
# In that case, this empty token_str corresponds
# to a delimiter which was located immediately
# after the variable name,
# AND
# -there is more text to follow,
# OR
# -we are at the end of the template.
token_ltmpl
.
append
(
text_block
)
if
len
(
token_ltmpl
)
==
1
:
tokens_lltmpl
.
append
(
token_ltmpl
[
0
])
else
:
tokens_lltmpl
.
append
(
token_ltmpl
)
del
token_ltmpl
token_ltmpl
=
[]
else
:
token_ltmpl
.
append
(
text_block
)
if
(
delim_found
and
(
delim
==
'
\n
'
)):
lineno
+=
1
j
+=
1
elif
isinstance
(
entry
,
VarRef
):
#elif hasattr(entry, 'descr_str'):
lineno
=
entry
.
srcloc
.
lineno
if
((
len
(
token_ltmpl
)
==
1
)
and
isinstance
(
token_ltmpl
[
0
],
TextBlock
)
and
(
len
(
token_ltmpl
[
0
]
.
text
)
==
0
)):
# special case: if the previous entry was "", then it means
# the delimeter appeared at the end of the previous text block
# leading up to this variable. It separates the variable from
# the previous text block. It is not a text block of length 0.
token_ltmpl
[
0
]
=
entry
else
:
token_ltmpl
.
append
(
entry
)
elif
entry
==
None
:
token_ltmpl
.
append
(
entry
)
else
:
assert
(
False
)
i
+=
1
# Append left over remains of the last token
if
len
(
token_ltmpl
)
==
1
:
tokens_lltmpl
.
append
(
token_ltmpl
[
0
])
elif
len
(
token_ltmpl
)
>
1
:
tokens_lltmpl
.
append
(
token_ltmpl
)
del
token_ltmpl
return
tokens_lltmpl
def
SplitTemplateMulti
(
ltmpl
,
delims
,
delete_blanks
=
False
):
"""
Split a template "ltmpl" into a list of templates using a
single one or more delimiter strings "delim_list".
If multiple delimiter strings are provided, splitting
begins using the first delimiter string in the list.
Then each token in the resulting list of templates
is split using the next delimiter string
and so on until we run out of delimiter strings.
"ltmpl" should be an list of TextBlocks and VarRefs.
"delims" should be a simple string (type str) or a list of strings
"delete_blanks" is either True or False
If True, then any blank entries in the resulting list of
tokens (sub-templates) will be deleted.
"""
if
hasattr
(
delims
,
'__len__'
):
# then it hopefully is a list of strings
delim_list
=
delims
else
:
delim_list
=
[
delims
]
# then it hopefully is a string
tokens
=
[
ltmpl
]
for
delim
in
delim_list
:
assert
(
type
(
delim
)
is
str
)
tokens_il
=
[]
for
t
in
tokens
:
sub_tokens
=
SplitTemplate
(
t
,
delim
,
delete_blanks
)
for
st
in
sub_tokens
:
if
hasattr
(
st
,
'__len__'
):
if
(
len
(
st
)
>
0
)
or
(
not
delete_blanks
):
tokens_il
.
append
(
st
)
else
:
tokens_il
.
append
(
st
)
tokens
=
tokens_il
del
tokens_il
return
tokens
def
_TableFromTemplate
(
d
,
ltmpl
,
delimiters
,
delete_blanks
):
"""
See the docstring for the TableFromTemplate() function for an explanation.
(This _TableFromTemplate() and SplitTemplate() are the workhorse functions
for TableFromTemplate().)
"""
output
=
SplitTemplateMulti
(
ltmpl
,
delimiters
[
d
],
delete_blanks
[
d
])
if
d
>
0
:
i
=
0
while
i
<
len
(
output
):
output
[
i
]
=
_TableFromTemplate
(
d
-
1
,
output
[
i
],
delimiters
,
delete_blanks
)
# Delete empty LISTS?
if
(
delete_blanks
[
d
]
and
hasattr
(
output
[
i
],
'__len__'
)
and
(
len
(
output
[
i
])
==
0
)):
del
output
[
i
]
else
:
i
+=
1
return
output
def
TableFromTemplate
(
ltmpl
,
delimiters
,
delete_blanks
=
True
):
"""
This function can be used to split a template
(a list containing TextBlocks and VarRefs) into a table
into a multidimensional table, with an arbitrary number of dimensions.
Arguments:
ltmpl
An alternating list of TextBlocks and VarRefs containing
the contents of this text template.
delimiters
The user must supply a list or tuple of delimiters: one delimiter for
each dimension in the table, with low-priority delimiters
(such as spaces ' ') appearing first, and higher-priority delimiters
(sich as newlines '\n') appearing later on in the list.
This function will divide the entire "ltmpl" into an n-dimensional
table. Initially the text is split into a list of text using the
highest-priority delimiter. Then each entry in the resulting list is
split into another list according to the next highest-priority delimiter.
This continues until all of the delimiters are used up and an
n-dimensional list-of-lists is remaining.
delete_blanks
The optional "delete_blanks" argument can be used to indicate whether
or not to delete blank entries in the table (which occur as a result
of placing two delimiters next to each other). It should be either
None (default), or it should be an array of booleans matching the
size of the "delimiters" argument. This allows the caller to customize
the merge settings separately for each dimension (for example: to allow
merging of whitespace within a line, without ignoring blank lines).
---- Details: ----
1) Multi-character delimiters ARE allowed (like '\n\n').
2) If a delimiter in the "delimiters" argument is not a string
but is a tuple (or a list) of strings, then the text is split according
to any of the delimiters in that tuple/list (starting from the last entry).
This way, users can use this feature to split text according to multiple
different kinds of whitespace characters (such as ' ' and '\t'), for
example, buy setting delimiters[0] = (' ','\t'). If, additionally,
delete_blanks[0] == True, then this will cause this function to
divide text in without regard to whitespace on a given line (for example).
Detailed example:
table2D = TableFromTmplList(ltmpl,
delimiters = ((' ','\t'), '\n'),
delete_blanks = (True, False))
This divides text in a similar way that the "awk" program does by default,
ie, by ignoring various kinds of whitespace between text fields, but NOT
ignoring blank lines.
3) Any text contained in variable-names is ignored.
"""
# Make a copy of ltmpl
# (The workhorse function "_TableFromTemplate()" makes in-place changes to
# its "ltmpl" argument. I don't want to modify "ltmpl", so I make a copy
# of it before I invoke "_TableFromTemplate()" on it.)
output
=
[
ltmpl
[
i
]
for
i
in
range
(
0
,
len
(
ltmpl
))]
d
=
len
(
delimiters
)
-
1
output
=
_TableFromTemplate
(
d
,
output
,
delimiters
,
delete_blanks
)
return
output
class
TemplateLexer
(
TtreeShlex
):
""" This class extends the standard python lexing module, shlex, adding a
new member function (ReadTemplate()), which can read in a block of raw text,
(halting at an (non-escaped) terminal character), and split the text into
alternating blocks of text and variables. (As far as this lexer is
concerned, "variables" are simply tokens preceeded by $ or @ characters,
and surrounded by optional curly-brackets {}.)
"""
def
__init__
(
self
,
instream
=
None
,
infile
=
None
,
posix
=
False
):
TtreeShlex
.
__init__
(
self
,
instream
,
infile
,
posix
)
self
.
var_delim
=
'$@'
#characters which can begin a variable name
self
.
var_open_paren
=
'{'
#optional parenthesis surround a variable
self
.
var_close_paren
=
'}'
#optional parenthesis surround a variable
self
.
newline
=
'
\n
'
self
.
comment_skip_var
=
'#'
# Which characters belong in words?
#
# We want to allow these characters:
# ./$@&%^!*~`-_:;?<>[]()
# to appear inside the tokens that TtreeShlex.get_token()
# retrieves (TtreeShlex.get_token() is used to read class
# names, and instance names, and variable names)
#
# settings.lex.wordchars+='./$@&%^!*~`-_+:;?<>[]' #Allow these chars
#
# Ommisions:
# Note: I left out quotes, whitespace, comment chars ('#'), and escape
# characters ('\\') because they are also dealt with separately.
# Those characters should not overlap with settings.lex.wordchars.
#
# Enabling unicode support requires that we override this choice
# by specifying "lex.wordterminators" instead of "wordchars".
#
# lex.wordterminators should be the (printable) set inverse of lex.wordchars
# I'm not sure which ascii characters are NOT included in the string above
# (We need to figure that out, and put them in settings.lex.wordterminators)
# To figure that out, uncomment the 8 lines below:
#
#self.wordterminators=''
#for i in range(0,256):
# c = chr(i)
# if c not in self.wordchars:
# self.wordterminators += c
#sys.stderr.write('-------- wordterminators = --------\n')
#sys.stderr.write(self.wordterminators+'\n')
#sys.stderr.write('-----------------------------------\n')
#
# Here is the result:
self
.
wordterminators
=
'(),={|}'
+
\
self
.
whitespace
+
\
self
.
quotes
+
\
self
.
escape
+
\
self
.
commenters
# Note:
# self.whitespace = ' \t\r\f\n'
# self.quotes = '\'"'
# self.escape = '\\'
# self.commenters = '#'
self
.
source_triggers
=
set
([
'include'
,
'import'
])
self
.
source_triggers_x
=
set
([
'import'
])
def
GetSrcLoc
(
self
):
return
OSrcLoc
(
self
.
infile
,
self
.
lineno
)
def
ReadTemplate
(
self
,
simplify_output
=
False
,
terminators
=
'}'
,
other_esc_chars
=
'{'
,
keep_terminal_char
=
True
):
"""
ReadTemplate() reads a block of text (between terminators)
and divides it into variables (tokens following a '$' or '@' character)
and raw text. This is similar to pythons string.Template(),
however it reads from streams (files), not strings, and it allows use
of more complicated variable names with multiple variable delimiters
(eg '$' and '@').
This readline()-like member function terminates when reaching a
user-specified terminator character character (second argument),
or when variable (eg: "$var"$ is encountered). The result is
a list of variable-separated text-blocks (stored in the first
argument). For example, the string:
"string with $var1 and $var2 variables.}" contains:
"string with ",
$var1,
" and ",
$var2,
" variables.}"
This simplifies the final process of rendering
(substituting text into) the text blocks later on.
Output:
This function returns a list of (alternating) blocks of
text, and variable names. Each entry in the list is either:
1) a text block:
Raw text is copied from the source, verbatim, along with
some additional data (filename and line numbers), to
help retroactively identify where the text came from
(in case a syntax error in the text is discovered later).
In this case, the list entry is stored as a list
The format (TextBlock) is similar to:
[text_string, ((filenameA,lineBegin), (filenameB,lineEnd))],
where the tuples, (filenameA,lineBegin) and (filenameB,lineEnd)
denote the source file(s) from which the text was read, and
line number at the beginning and ending of the text block.
(This information is useful for generating helpful error
messages. Note that the "TtreeShlex" class allows users to
combine multiple files transparently into one stream using
the "source" (or "sourcehook()") member. For this reason, it
is possible, although unlikely, that the text-block
we are reading could span multiple different files.)
2) a variable (for example "$var" or "${var}"):
In this case, the list entry is stored in the "VarRef" format
which is essentialy shown below:
[[var_prefix, var_nptr, var_suffix], (filename,lineno)]
where var_prefix and var_suffix are strings containing brackets
and other text enclosing the variable name (and may be empty).
As an example, we consider a file named "datafile" which
contains the text containing 2 text blocks and 1 variable:
"some\n text\n before ${var}. Text after\n".
ReadTemplate() will read this and return a list with 3 entries:
[ ['some\n text\n before', (('datafile', 1), ('datafile', 3))],
[['${', 'var', '}'], ('datafile', 3, 3)],
['Text after\n', (('datafile', 3), ('datafile', 4))] ]
Note that while parsing the text, self.lineno counter is
incremented whenever a newline character is encountered.
(Also: Unlike shlex.get_token(), this function does not
delete commented text, or insert text from other files.)
Exceptional Cases:
Terminator characters are ignored if they are part of a variable
reference. (For example, the '}' in "${var}", is used to denote a
bracketed variable, and does not cause ReadTemplate() to stop reading)
OR if they are part of a two-character escape sequence
(for example, '}' in "\}" does not cause terminate parsing).
In that case, the text is considered normal text. (However the
'\' character is also stripped out. It is also stripped out if it
preceeds any characters in "other_esc_chars", which is
the second argument. Otherwise it is left in the text block.)
"""
#print(' ReadTemplate('+terminators+') invoked at '+self.error_leader())
# The main loop of the parser reads only one variable at time.
# The following variables keep track of where we are in the template.
reading_var
=
False
# Are we currently reading in the name of a variable?
prev_char_delim
=
False
#True iff we just read a var_delim character like '$'
escaped_state
=
False
#True iff we just read a (non-escaped) esc character '\'
commented_state
=
False
#True iff we are in a region of text where vars should be ignored
var_paren_depth
=
0
# This is non-zero iff we are inside a
# bracketed variable's name for example: "${var}"
var_terminators
=
self
.
whitespace
+
self
.
newline
+
self
.
var_delim
+
'{}'
tmpl_list
=
[]
# List of alternating tuples of text_blocks and
# variable names (see format comment above)
# This list will be returned to the caller.
#sys.stderr.write('report_progress='+str(report_progress))
prev_filename
=
self
.
infile
prev_lineno
=
self
.
lineno
var_prefix
=
''
var_descr_plist
=
[]
var_suffix
=
''
text_block_plist
=
[]
done_reading
=
False
while
not
done_reading
:
terminate_text
=
False
terminate_var
=
False
#delete_prior_escape = False
nextchar
=
self
.
read_char
()
#print(' ReadTemplate() nextchar=\''+nextchar+'\' at '+self.error_leader()+' esc='+str(escaped_state)+', pvar='+str(prev_char_delim)+', paren='+str(var_paren_depth))
# Count newlines:
if
nextchar
in
self
.
newline
:
commented_state
=
False
self
.
lineno
+=
1
elif
((
nextchar
in
self
.
comment_skip_var
)
and
(
not
escaped_state
)):
commented_state
=
True
# Check for end-of-file:
if
nextchar
==
''
:
if
escaped_state
:
raise
InputError
(
'Error: in '
+
self
.
error_leader
()
+
'
\n\n
'
'No escaped character.'
)
if
reading_var
:
terminate_var
=
True
else
:
terminate_text
=
True
done_reading
=
True
# --- Now process the character: ---
# What we do next depends on which "mode" we are in.
# If we are reading a regular text block (reading_var == False),
# then we keep appending characters onto the end of "text_block",
# checking for terminal characters, or variable delimiters.
# If we are reading a variable name (reading_var == True),
# then we append characters to the end of "var_descr_plist[]",
# checking for variable terminator characters, as well as
# parenthesis (some variables are surrounded by parenthesis).
elif
reading_var
:
if
nextchar
in
terminators
:
#sys.stdout.write(' ReadTemplate() readmode found terminator.\n')
if
escaped_state
:
# In this case, the '\' char was only to prevent terminating
# string prematurely, so delete the '\' character.
#delete_prior_escape = True
if
not
(
nextchar
in
self
.
var_close_paren
):
del
var_descr_plist
[
-
1
]
var_descr_plist
.
append
(
nextchar
)
elif
not
((
var_paren_depth
>
0
)
and
(
nextchar
in
self
.
var_close_paren
)):
terminate_var
=
True
done_reading
=
True
if
nextchar
in
self
.
var_open_paren
:
# eg: nextchar == '{'
#sys.stdout.write(' ReadTemplate() readmode found {.\n')
if
escaped_state
:
# In this case, the '\' char was only to prevent
# interpreting '{' as a variable prefix
#delete_prior_escape=True # so delete the '\' character
del
var_descr_plist
[
-
1
]
var_descr_plist
.
append
(
nextchar
)
else
:
# "${var}" is a valid way to refer to a variable
if
prev_char_delim
:
var_prefix
+=
nextchar
var_paren_depth
=
1
# "${{var}}" is also a valid way to refer to a variable,
# (although strange), but "$va{r}" is not.
# Parenthesis (in bracketed variable names) must
# immediately follow the '$' character (as in "${var}")
elif
var_paren_depth
>
0
:
var_paren_depth
+=
1
elif
nextchar
in
self
.
var_close_paren
:
#sys.stdout.write(' ReadTemplate() readmode found }.\n')
if
escaped_state
:
# In this case, the '\' char was only to prevent
# interpreting '}' as a variable suffix,
#delete_prior_escape=True #so skip the '\' character
if
(
nextchar
not
in
terminators
):
del
var_descr_plist
[
-
1
]
var_descr_plist
.
append
(
nextchar
)
else
:
if
var_paren_depth
>
0
:
var_paren_depth
-=
1
if
var_paren_depth
==
0
:
var_suffix
=
nextchar
terminate_var
=
True
elif
nextchar
in
var_terminators
:
#sys.stdout.write(' ReadTemplate() readmode found var_terminator \"'+nextchar+'\"\n')
if
(
escaped_state
or
(
var_paren_depth
>
0
)):
# In this case, the '\' char was only to prevent
# interpreting nextchar as a variable terminator
#delete_prior_escape = True # so skip the '\' character
del
var_descr_plist
[
-
1
]
var_descr_plist
.
append
(
nextchar
)
else
:
terminate_var
=
True
elif
nextchar
in
self
.
var_delim
:
# such as '$'
#sys.stdout.write(' ReadTemplate() readmode found var_delim.\n')
if
escaped_state
:
# In this case, the '\' char was only to prevent
# interpreting '$' as a new variable name
#delete_prior_escape = True # so skip the '\' character
del
var_descr_plist
[
-
1
]
var_descr_plist
.
append
(
nextchar
)
else
:
prev_var_delim
=
True
# Then we are processing a new variable name
terminate_var
=
True
else
:
var_descr_plist
.
append
(
nextchar
)
prev_char_delim
=
False
else
:
# begin else clause for "if reading_var:"
# Then we are reading a text_block
if
nextchar
in
terminators
:
if
escaped_state
:
# In this case, the '\' char was only to prevent terminating
# string prematurely, so delete the '\' character.
#delete_prior_escape = True
del
text_block_plist
[
-
1
]
text_block_plist
.
append
(
nextchar
)
elif
commented_state
:
text_block_plist
.
append
(
nextchar
)
else
:
terminate_text
=
True
done_reading
=
True
elif
nextchar
in
self
.
var_delim
:
# such as '$'
if
escaped_state
:
# In this case, the '\' char was only to prevent
# interpreting '$' as a variable prefix.
#delete_prior_escape=True #so delete the '\' character
del
text_block_plist
[
-
1
]
text_block_plist
.
append
(
nextchar
)
elif
commented_state
:
text_block_plist
.
append
(
nextchar
)
else
:
prev_char_delim
=
True
reading_var
=
True
var_paren_depth
=
0
terminate_text
=
True
else
:
text_block_plist
.
append
(
nextchar
)
#TO DO: use "list_of_chars.join()" instead of '+='
prev_char_delim
=
False
# the previous character was not '$'
# Now deal with "other_esc_chars"
#if escaped_state and (nextchar in other_esc_chars):
if
escaped_state
and
(
nextchar
in
other_esc_chars
):
if
reading_var
:
#sys.stdout.write(' ReadTemplate: var_descr_str=\''+''.join(var_descr_plist)+'\'\n')
assert
(
var_descr_plist
[
-
2
]
in
self
.
escape
)
del
var_descr_plist
[
-
2
]
else
:
#sys.stdout.write(' ReadTemplate: text_block=\''+''.join(text_block_plist)+'\'\n')
assert
(
text_block_plist
[
-
2
]
in
self
.
escape
)
del
text_block_plist
[
-
2
]
if
terminate_text
:
#sys.stdout.write('ReadTemplate() appending: ')
#sys.stdout.write(text_block)
#tmpl_list.append( [text_block,
# ((prev_filename, prev_lineno),
# (self.infile, self.lineno))] )
if
simplify_output
:
tmpl_list
.
append
(
''
.
join
(
text_block_plist
))
else
:
tmpl_list
.
append
(
TextBlock
(
''
.
join
(
text_block_plist
),
OSrcLoc
(
prev_filename
,
prev_lineno
)))
#, OSrcLoc(self.infile, self.lineno)))
if
not
done_reading
:
# The character that ended the text block
# was a variable delimiter (like '$'), in which case
# we should put it (nextchar) in the variable's prefix.
var_prefix
=
nextchar
else
:
var_prefix
=
''
var_descr_plist
=
[]
var_suffix
=
''
prev_filename
=
self
.
infile
prev_lineno
=
self
.
lineno
del
text_block_plist
text_block_plist
=
[]
#gc.collect()
elif
terminate_var
:
# Print an error if we terminated in the middle of
# an incomplete variable name:
if
prev_char_delim
:
raise
InputError
(
'Error: near '
+
self
.
error_leader
()
+
'
\n\n
'
'Null variable name.'
)
if
var_paren_depth
>
0
:
raise
InputError
(
'Error: near '
+
self
.
error_leader
()
+
'
\n\n
'
'Incomplete bracketed variable name.'
)
var_descr_str
=
''
.
join
(
var_descr_plist
)
# Now check for variable format modifiers,
# like python's ".rjust()" and ".ljust()".
# If present, then put these in the variable suffix.
if
((
len
(
var_descr_plist
)
>
0
)
and
(
var_descr_plist
[
-
1
]
==
')'
)):
#i = len(var_descr_plist)-1
#while i >= 0:
# if var_descr_plist[i] == '(':
# break
# i -= 1
i
=
var_descr_str
.
rfind
(
'('
)
if
(((
i
-
6
)
>=
0
)
and
((
var_descr_str
[
i
-
6
:
i
]
==
'.rjust'
)
or
(
var_descr_str
[
i
-
6
:
i
]
==
'.ljust'
))):
var_suffix
=
''
.
join
(
var_descr_plist
[
i
-
6
:])
+
var_suffix
#var_descr_plist = var_descr_plist[:i-6]
var_descr_str
=
var_descr_str
[:
i
-
6
]
# Process any special characters in the variable name
var_descr_str
=
EscCharStrToChar
(
var_descr_str
)
#tmpl_list.append( [[var_prefix, var_descr_str, var_suffix],
# (self.infile, self.lineno)] )
if
simplify_output
:
tmpl_list
.
append
(
var_prefix
+
var_descr_str
+
var_suffix
)
else
:
tmpl_list
.
append
(
VarRef
(
var_prefix
,
var_descr_str
,
var_suffix
,
OSrcLoc
(
self
.
infile
,
self
.
lineno
))
)
#if report_progress:
#sys.stderr.write(' parsed variable '+var_prefix+var_descr_str+var_suffix+'\n')
#sys.stdout.write('ReadTemplate() appending: ')
#print(var_prefix + var_descr_str + var_suffix)
del
var_descr_plist
del
var_descr_str
prev_filename
=
self
.
infile
prev_lineno
=
self
.
lineno
var_prefix
=
''
var_descr_plist
=
[]
var_suffix
=
''
# Special case: Variable delimiters like '$'
# terminate the reading of variables,
# but they also signify that a new
# variable is being read.
if
nextchar
in
self
.
var_delim
:
# Then we are processing a new variable name
prev_var_delim
=
True
reading_var
=
True
var_paren_depth
=
0
var_prefix
=
nextchar
elif
nextchar
in
self
.
var_close_paren
:
del
text_block_plist
text_block_plist
=
[]
#gc.collect()
prev_var_delim
=
False
reading_var
=
False
else
:
# Generally, we don't want to initialize the next text block
# with the empty string. Consider that whatever character
# caused us to stop reading the previous variable and append
# it to the block of text that comes after.
del
text_block_plist
text_block_plist
=
[
nextchar
]
#gc.collect()
prev_var_delim
=
False
reading_var
=
False
# If we reached the end of the template (and the user requests it),
# then the terminal character can be included in the list
# of text_blocks to be returned to the caller.
if
done_reading
and
keep_terminal_char
:
#sys.stdout.write('ReadTemplate() appending: \''+nextchar+'\'\n')
# Here we create a new text block which contains only the
# terminal character (nextchar).
#tmpl_list.append( [nextchar,
# ((self.infile, self.lineno),
# (self.infile, self.lineno))] )
if
simplify_output
:
tmpl_list
.
append
(
nextchar
)
else
:
tmpl_list
.
append
(
TextBlock
(
nextchar
,
OSrcLoc
(
self
.
infile
,
self
.
lineno
)))
#, OSrcLoc(self.infile, self.lineno)))
if
escaped_state
:
escaped_state
=
False
else
:
if
nextchar
in
self
.
escape
:
escaped_state
=
True
#print("*** TMPL_LIST0 = ***", tmpl_list)
return
tmpl_list
# <- return value stored here
def
GetParenExpr
(
self
,
prepend_str
=
''
,
left_paren
=
'('
,
right_paren
=
')'
):
""" GetParenExpr() is useful for reading in strings
with nested parenthesis and spaces.
This function can read in the entire string:
.trans(0, 10.0*sin(30), 10.0*cos(30))
(Because I was too lazy to write this correctly...)
Spaces are currently stripped out of the expression.
(...unless surrounded by quotes) The string above becomes:
".trans(0,10.0*sin(30),10.0*cos(30))"
Sometimes the caller wants to prepend some text to the beginning
of the expression (which may contain parenthesis). For this
reason, an optional first argument ("prepend_str") can be
provided. By default it is empty.
"""
orig_wordterm
=
self
.
wordterminators
self
.
wordterminators
=
self
.
wordterminators
.
replace
(
left_paren
,
''
)
.
replace
(
right_paren
,
''
)
token
=
self
.
get_token
()
if
((
token
==
''
)
or
(
token
==
self
.
eof
)):
return
prepend_str
expr_str
=
prepend_str
+
token
#if (expr_str.find(left_paren) == -1):
# raise InputError('Error near or before '+self.error_leader()+'\n'
# 'Expected an open-paren (\"'+prepend_str+left_paren+'\") before this point.\n')
# return expr_str
paren_depth
=
expr_str
.
count
(
left_paren
)
-
expr_str
.
count
(
right_paren
)
while
((
len
(
expr_str
)
==
0
)
or
(
paren_depth
>
0
)):
token
=
self
.
get_token
()
if
((
type
(
token
)
is
not
str
)
or
(
token
==
''
)):
raise
InputError
(
'Error near or before '
+
self
.
error_leader
()
+
'
\n
'
'Invalid expression:
\"
'
+
expr_str
+
'
\"
'
)
expr_str
+=
token
paren_depth
=
expr_str
.
count
(
left_paren
)
-
expr_str
.
count
(
right_paren
)
if
(
paren_depth
!=
0
):
raise
InputError
(
'Error near or before '
+
self
.
error_leader
()
+
'
\n
'
'Invalid expression:
\"
'
+
expr_str
+
'
\"
'
)
self
.
wordterminators
=
orig_wordterm
return
expr_str
if
__name__
==
'__main__'
:
if
len
(
sys
.
argv
)
==
1
:
lexer
=
TtreeShlex
()
else
:
file
=
sys
.
argv
[
1
]
lexer
=
TtreeShlex
(
open
(
file
),
file
)
while
1
:
tt
=
lexer
.
get_token
()
if
tt
:
print
(
"Token: "
+
repr
(
tt
))
else
:
break
Event Timeline
Log In to Comment