ast.py
No OneTemporary
Actions

Subscribers

None

File Metadata

Created: Tue, Nov 19, 11:19

ast.py
View Options

	#!/usr/bin/env python
	#
	# Copyright 2007 Neal Norwitz
	# Portions Copyright 2007 Google Inc.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	"""Generate an Abstract Syntax Tree (AST) for C++."""

	__author__ = 'nnorwitz@google.com (Neal Norwitz)'


	# TODO:
	# * Tokens should never be exported, need to convert to Nodes
	# (return types, parameters, etc.)
	# * Handle static class data for templatized classes
	# * Handle casts (both C++ and C-style)
	# * Handle conditions and loops (if/else, switch, for, while/do)
	#
	# TODO much, much later:
	# * Handle #define
	# * exceptions


	try:
	# Python 3.x
	import builtins
	except ImportError:
	# Python 2.x
	import __builtin__ as builtins

	import sys
	import traceback

	from cpp import keywords
	from cpp import tokenize
	from cpp import utils


	if not hasattr(builtins, 'reversed'):
	# Support Python 2.3 and earlier.
	def reversed(seq):
	for i in range(len(seq)-1, -1, -1):
	yield seq[i]

	if not hasattr(builtins, 'next'):
	# Support Python 2.5 and earlier.
	def next(obj):
	return obj.next()


	VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3)

	FUNCTION_NONE = 0x00
	FUNCTION_CONST = 0x01
	FUNCTION_VIRTUAL = 0x02
	FUNCTION_PURE_VIRTUAL = 0x04
	FUNCTION_CTOR = 0x08
	FUNCTION_DTOR = 0x10
	FUNCTION_ATTRIBUTE = 0x20
	FUNCTION_UNKNOWN_ANNOTATION = 0x40
	FUNCTION_THROW = 0x80
	FUNCTION_OVERRIDE = 0x100

	"""
	These are currently unused. Should really handle these properly at some point.

	TYPE_MODIFIER_INLINE = 0x010000
	TYPE_MODIFIER_EXTERN = 0x020000
	TYPE_MODIFIER_STATIC = 0x040000
	TYPE_MODIFIER_CONST = 0x080000
	TYPE_MODIFIER_REGISTER = 0x100000
	TYPE_MODIFIER_VOLATILE = 0x200000
	TYPE_MODIFIER_MUTABLE = 0x400000

	TYPE_MODIFIER_MAP = {
	'inline': TYPE_MODIFIER_INLINE,
	'extern': TYPE_MODIFIER_EXTERN,
	'static': TYPE_MODIFIER_STATIC,
	'const': TYPE_MODIFIER_CONST,
	'register': TYPE_MODIFIER_REGISTER,
	'volatile': TYPE_MODIFIER_VOLATILE,
	'mutable': TYPE_MODIFIER_MUTABLE,
	}
	"""

	_INTERNAL_TOKEN = 'internal'
	_NAMESPACE_POP = 'ns-pop'


	# TODO(nnorwitz): use this as a singleton for templated_types, etc
	# where we don't want to create a new empty dict each time. It is also const.
	class _NullDict(object):
	__contains__ = lambda self: False
	keys = values = items = iterkeys = itervalues = iteritems = lambda self: ()


	# TODO(nnorwitz): move AST nodes into a separate module.
	class Node(object):
	"""Base AST node."""

	def __init__(self, start, end):
	self.start = start
	self.end = end

	def IsDeclaration(self):
	"""Returns bool if this node is a declaration."""
	return False

	def IsDefinition(self):
	"""Returns bool if this node is a definition."""
	return False

	def IsExportable(self):
	"""Returns bool if this node exportable from a header file."""
	return False

	def Requires(self, node):
	"""Does this AST node require the definition of the node passed in?"""
	return False

	def XXX__str__(self):
	return self._StringHelper(self.__class__.__name__, '')

	def _StringHelper(self, name, suffix):
	if not utils.DEBUG:
	return '%s(%s)' % (name, suffix)
	return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix)

	def __repr__(self):
	return str(self)


	class Define(Node):
	def __init__(self, start, end, name, definition):
	Node.__init__(self, start, end)
	self.name = name
	self.definition = definition

	def __str__(self):
	value = '%s %s' % (self.name, self.definition)
	return self._StringHelper(self.__class__.__name__, value)


	class Include(Node):
	def __init__(self, start, end, filename, system):
	Node.__init__(self, start, end)
	self.filename = filename
	self.system = system

	def __str__(self):
	fmt = '"%s"'
	if self.system:
	fmt = '<%s>'
	return self._StringHelper(self.__class__.__name__, fmt % self.filename)


	class Goto(Node):
	def __init__(self, start, end, label):
	Node.__init__(self, start, end)
	self.label = label

	def __str__(self):
	return self._StringHelper(self.__class__.__name__, str(self.label))


	class Expr(Node):
	def __init__(self, start, end, expr):
	Node.__init__(self, start, end)
	self.expr = expr

	def Requires(self, node):
	# TODO(nnorwitz): impl.
	return False

	def __str__(self):
	return self._StringHelper(self.__class__.__name__, str(self.expr))


	class Return(Expr):
	pass


	class Delete(Expr):
	pass


	class Friend(Expr):
	def __init__(self, start, end, expr, namespace):
	Expr.__init__(self, start, end, expr)
	self.namespace = namespace[:]


	class Using(Node):
	def __init__(self, start, end, names):
	Node.__init__(self, start, end)
	self.names = names

	def __str__(self):
	return self._StringHelper(self.__class__.__name__, str(self.names))


	class Parameter(Node):
	def __init__(self, start, end, name, parameter_type, default):
	Node.__init__(self, start, end)
	self.name = name
	self.type = parameter_type
	self.default = default

	def Requires(self, node):
	# TODO(nnorwitz): handle namespaces, etc.
	return self.type.name == node.name

	def __str__(self):
	name = str(self.type)
	suffix = '%s %s' % (name, self.name)
	if self.default:
	suffix += ' = ' + ''.join([d.name for d in self.default])
	return self._StringHelper(self.__class__.__name__, suffix)


	class _GenericDeclaration(Node):
	def __init__(self, start, end, name, namespace):
	Node.__init__(self, start, end)
	self.name = name
	self.namespace = namespace[:]

	def FullName(self):
	prefix = ''
	if self.namespace and self.namespace[-1]:
	prefix = '::'.join(self.namespace) + '::'
	return prefix + self.name

	def _TypeStringHelper(self, suffix):
	if self.namespace:
	names = [n or '<anonymous>' for n in self.namespace]
	suffix += ' in ' + '::'.join(names)
	return self._StringHelper(self.__class__.__name__, suffix)


	# TODO(nnorwitz): merge with Parameter in some way?
	class VariableDeclaration(_GenericDeclaration):
	def __init__(self, start, end, name, var_type, initial_value, namespace):
	_GenericDeclaration.__init__(self, start, end, name, namespace)
	self.type = var_type
	self.initial_value = initial_value

	def Requires(self, node):
	# TODO(nnorwitz): handle namespaces, etc.
	return self.type.name == node.name

	def ToString(self):
	"""Return a string that tries to reconstitute the variable decl."""
	suffix = '%s %s' % (self.type, self.name)
	if self.initial_value:
	suffix += ' = ' + self.initial_value
	return suffix

	def __str__(self):
	return self._StringHelper(self.__class__.__name__, self.ToString())


	class Typedef(_GenericDeclaration):
	def __init__(self, start, end, name, alias, namespace):
	_GenericDeclaration.__init__(self, start, end, name, namespace)
	self.alias = alias

	def IsDefinition(self):
	return True

	def IsExportable(self):
	return True

	def Requires(self, node):
	# TODO(nnorwitz): handle namespaces, etc.
	name = node.name
	for token in self.alias:
	if token is not None and name == token.name:
	return True
	return False

	def __str__(self):
	suffix = '%s, %s' % (self.name, self.alias)
	return self._TypeStringHelper(suffix)


	class _NestedType(_GenericDeclaration):
	def __init__(self, start, end, name, fields, namespace):
	_GenericDeclaration.__init__(self, start, end, name, namespace)
	self.fields = fields

	def IsDefinition(self):
	return True

	def IsExportable(self):
	return True

	def __str__(self):
	suffix = '%s, {%s}' % (self.name, self.fields)
	return self._TypeStringHelper(suffix)


	class Union(_NestedType):
	pass


	class Enum(_NestedType):
	pass


	class Class(_GenericDeclaration):
	def __init__(self, start, end, name, bases, templated_types, body, namespace):
	_GenericDeclaration.__init__(self, start, end, name, namespace)
	self.bases = bases
	self.body = body
	self.templated_types = templated_types

	def IsDeclaration(self):
	return self.bases is None and self.body is None

	def IsDefinition(self):
	return not self.IsDeclaration()

	def IsExportable(self):
	return not self.IsDeclaration()

	def Requires(self, node):
	# TODO(nnorwitz): handle namespaces, etc.
	if self.bases:
	for token_list in self.bases:
	# TODO(nnorwitz): bases are tokens, do name comparison.
	for token in token_list:
	if token.name == node.name:
	return True
	# TODO(nnorwitz): search in body too.
	return False

	def __str__(self):
	name = self.name
	if self.templated_types:
	name += '<%s>' % self.templated_types
	suffix = '%s, %s, %s' % (name, self.bases, self.body)
	return self._TypeStringHelper(suffix)


	class Struct(Class):
	pass


	class Function(_GenericDeclaration):
	def __init__(self, start, end, name, return_type, parameters,
	modifiers, templated_types, body, namespace):
	_GenericDeclaration.__init__(self, start, end, name, namespace)
	converter = TypeConverter(namespace)
	self.return_type = converter.CreateReturnType(return_type)
	self.parameters = converter.ToParameters(parameters)
	self.modifiers = modifiers
	self.body = body
	self.templated_types = templated_types

	def IsDeclaration(self):
	return self.body is None

	def IsDefinition(self):
	return self.body is not None

	def IsExportable(self):
	if self.return_type and 'static' in self.return_type.modifiers:
	return False
	return None not in self.namespace

	def Requires(self, node):
	if self.parameters:
	# TODO(nnorwitz): parameters are tokens, do name comparison.
	for p in self.parameters:
	if p.name == node.name:
	return True
	# TODO(nnorwitz): search in body too.
	return False

	def __str__(self):
	# TODO(nnorwitz): add templated_types.
	suffix = ('%s %s(%s), 0x%02x, %s' %
	(self.return_type, self.name, self.parameters,
	self.modifiers, self.body))
	return self._TypeStringHelper(suffix)


	class Method(Function):
	def __init__(self, start, end, name, in_class, return_type, parameters,
	modifiers, templated_types, body, namespace):
	Function.__init__(self, start, end, name, return_type, parameters,
	modifiers, templated_types, body, namespace)
	# TODO(nnorwitz): in_class could also be a namespace which can
	# mess up finding functions properly.
	self.in_class = in_class


	class Type(_GenericDeclaration):
	"""Type used for any variable (eg class, primitive, struct, etc)."""

	def __init__(self, start, end, name, templated_types, modifiers,
	reference, pointer, array):
	"""
	Args:
	name: str name of main type
	templated_types: [Class (Type?)] template type info between <>
	modifiers: [str] type modifiers (keywords) eg, const, mutable, etc.
	reference, pointer, array: bools
	"""
	_GenericDeclaration.__init__(self, start, end, name, [])
	self.templated_types = templated_types
	if not name and modifiers:
	self.name = modifiers.pop()
	self.modifiers = modifiers
	self.reference = reference
	self.pointer = pointer
	self.array = array

	def __str__(self):
	prefix = ''
	if self.modifiers:
	prefix = ' '.join(self.modifiers) + ' '
	name = str(self.name)
	if self.templated_types:
	name += '<%s>' % self.templated_types
	suffix = prefix + name
	if self.reference:
	suffix += '&'
	if self.pointer:
	suffix += '*'
	if self.array:
	suffix += '[]'
	return self._TypeStringHelper(suffix)

	# By definition, Is* are always False. A Type can only exist in
	# some sort of variable declaration, parameter, or return value.
	def IsDeclaration(self):
	return False

	def IsDefinition(self):
	return False

	def IsExportable(self):
	return False


	class TypeConverter(object):

	def __init__(self, namespace_stack):
	self.namespace_stack = namespace_stack

	def _GetTemplateEnd(self, tokens, start):
	count = 1
	end = start
	while 1:
	token = tokens[end]
	end += 1
	if token.name == '<':
	count += 1
	elif token.name == '>':
	count -= 1
	if count == 0:
	break
	return tokens[start:end-1], end

	def ToType(self, tokens):
	"""Convert [Token,...] to [Class(...), ] useful for base classes.
	For example, code like class Foo : public Bar<x, y> { ... };
	the "Bar<x, y>" portion gets converted to an AST.

	Returns:
	[Class(...), ...]
	"""
	result = []
	name_tokens = []
	reference = pointer = array = False

	def AddType(templated_types):
	# Partition tokens into name and modifier tokens.
	names = []
	modifiers = []
	for t in name_tokens:
	if keywords.IsKeyword(t.name):
	modifiers.append(t.name)
	else:
	names.append(t.name)
	name = ''.join(names)
	if name_tokens:
	result.append(Type(name_tokens[0].start, name_tokens[-1].end,
	name, templated_types, modifiers,
	reference, pointer, array))
	del name_tokens[:]

	i = 0
	end = len(tokens)
	while i < end:
	token = tokens[i]
	if token.name == '<':
	new_tokens, new_end = self._GetTemplateEnd(tokens, i+1)
	AddType(self.ToType(new_tokens))
	# If there is a comma after the template, we need to consume
	# that here otherwise it becomes part of the name.
	i = new_end
	reference = pointer = array = False
	elif token.name == ',':
	AddType([])
	reference = pointer = array = False
	elif token.name == '*':
	pointer = True
	elif token.name == '&':
	reference = True
	elif token.name == '[':
	pointer = True
	elif token.name == ']':
	pass
	else:
	name_tokens.append(token)
	i += 1

	if name_tokens:
	# No '<' in the tokens, just a simple name and no template.
	AddType([])
	return result

	def DeclarationToParts(self, parts, needs_name_removed):
	name = None
	default = []
	if needs_name_removed:
	# Handle default (initial) values properly.
	for i, t in enumerate(parts):
	if t.name == '=':
	default = parts[i+1:]
	name = parts[i-1].name
	if name == ']' and parts[i-2].name == '[':
	name = parts[i-3].name
	i -= 1
	parts = parts[:i-1]
	break
	else:
	if parts[-1].token_type == tokenize.NAME:
	name = parts.pop().name
	else:
	# TODO(nnorwitz): this is a hack that happens for code like
	# Register(Foo<T>); where it thinks this is a function call
	# but it's actually a declaration.
	name = '???'
	modifiers = []
	type_name = []
	other_tokens = []
	templated_types = []
	i = 0
	end = len(parts)
	while i < end:
	p = parts[i]
	if keywords.IsKeyword(p.name):
	modifiers.append(p.name)
	elif p.name == '<':
	templated_tokens, new_end = self._GetTemplateEnd(parts, i+1)
	templated_types = self.ToType(templated_tokens)
	i = new_end - 1
	# Don't add a spurious :: to data members being initialized.
	next_index = i + 1
	if next_index < end and parts[next_index].name == '::':
	i += 1
	elif p.name in ('[', ']', '='):
	# These are handled elsewhere.
	other_tokens.append(p)
	elif p.name not in ('*', '&', '>'):
	# Ensure that names have a space between them.
	if (type_name and type_name[-1].token_type == tokenize.NAME and
	p.token_type == tokenize.NAME):
	type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0))
	type_name.append(p)
	else:
	other_tokens.append(p)
	i += 1
	type_name = ''.join([t.name for t in type_name])
	return name, type_name, templated_types, modifiers, default, other_tokens

	def ToParameters(self, tokens):
	if not tokens:
	return []

	result = []
	name = type_name = ''
	type_modifiers = []
	pointer = reference = array = False
	first_token = None
	default = []

	def AddParameter(end):
	if default:
	del default[0] # Remove flag.
	parts = self.DeclarationToParts(type_modifiers, True)
	(name, type_name, templated_types, modifiers,
	unused_default, unused_other_tokens) = parts
	parameter_type = Type(first_token.start, first_token.end,
	type_name, templated_types, modifiers,
	reference, pointer, array)
	p = Parameter(first_token.start, end, name,
	parameter_type, default)
	result.append(p)

	template_count = 0
	for s in tokens:
	if not first_token:
	first_token = s
	if s.name == '<':
	template_count += 1
	elif s.name == '>':
	template_count -= 1
	if template_count > 0:
	type_modifiers.append(s)
	continue

	if s.name == ',':
	AddParameter(s.start)
	name = type_name = ''
	type_modifiers = []
	pointer = reference = array = False
	first_token = None
	default = []
	elif s.name == '*':
	pointer = True
	elif s.name == '&':
	reference = True
	elif s.name == '[':
	array = True
	elif s.name == ']':
	pass # Just don't add to type_modifiers.
	elif s.name == '=':
	# Got a default value. Add any value (None) as a flag.
	default.append(None)
	elif default:
	default.append(s)
	else:
	type_modifiers.append(s)
	AddParameter(tokens[-1].end)
	return result

	def CreateReturnType(self, return_type_seq):
	if not return_type_seq:
	return None
	start = return_type_seq[0].start
	end = return_type_seq[-1].end
	_, name, templated_types, modifiers, default, other_tokens = \
	self.DeclarationToParts(return_type_seq, False)
	names = [n.name for n in other_tokens]
	reference = '&' in names
	pointer = '*' in names
	array = '[' in names
	return Type(start, end, name, templated_types, modifiers,
	reference, pointer, array)

	def GetTemplateIndices(self, names):
	# names is a list of strings.
	start = names.index('<')
	end = len(names) - 1
	while end > 0:
	if names[end] == '>':
	break
	end -= 1
	return start, end+1

	class AstBuilder(object):
	def __init__(self, token_stream, filename, in_class='', visibility=None,
	namespace_stack=[]):
	self.tokens = token_stream
	self.filename = filename
	# TODO(nnorwitz): use a better data structure (deque) for the queue.
	# Switching directions of the "queue" improved perf by about 25%.
	# Using a deque should be even better since we access from both sides.
	self.token_queue = []
	self.namespace_stack = namespace_stack[:]
	self.in_class = in_class
	if in_class is None:
	self.in_class_name_only = None
	else:
	self.in_class_name_only = in_class.split('::')[-1]
	self.visibility = visibility
	self.in_function = False
	self.current_token = None
	# Keep the state whether we are currently handling a typedef or not.
	self._handling_typedef = False

	self.converter = TypeConverter(self.namespace_stack)

	def HandleError(self, msg, token):
	printable_queue = list(reversed(self.token_queue[-20:]))
	sys.stderr.write('Got %s in %s @ %s %s\n' %
	(msg, self.filename, token, printable_queue))

	def Generate(self):
	while 1:
	token = self._GetNextToken()
	if not token:
	break

	# Get the next token.
	self.current_token = token

	# Dispatch on the next token type.
	if token.token_type == _INTERNAL_TOKEN:
	if token.name == _NAMESPACE_POP:
	self.namespace_stack.pop()
	continue

	try:
	result = self._GenerateOne(token)
	if result is not None:
	yield result
	except:
	self.HandleError('exception', token)
	raise

	def _CreateVariable(self, pos_token, name, type_name, type_modifiers,
	ref_pointer_name_seq, templated_types, value=None):
	reference = '&' in ref_pointer_name_seq
	pointer = '*' in ref_pointer_name_seq
	array = '[' in ref_pointer_name_seq
	var_type = Type(pos_token.start, pos_token.end, type_name,
	templated_types, type_modifiers,
	reference, pointer, array)
	return VariableDeclaration(pos_token.start, pos_token.end,
	name, var_type, value, self.namespace_stack)

	def _GenerateOne(self, token):
	if token.token_type == tokenize.NAME:
	if (keywords.IsKeyword(token.name) and
	not keywords.IsBuiltinType(token.name)):
	method = getattr(self, 'handle_' + token.name)
	return method()
	elif token.name == self.in_class_name_only:
	# The token name is the same as the class, must be a ctor if
	# there is a paren. Otherwise, it's the return type.
	# Peek ahead to get the next token to figure out which.
	next = self._GetNextToken()
	self._AddBackToken(next)
	if next.token_type == tokenize.SYNTAX and next.name == '(':
	return self._GetMethod([token], FUNCTION_CTOR, None, True)
	# Fall through--handle like any other method.

	# Handle data or function declaration/definition.
	syntax = tokenize.SYNTAX
	temp_tokens, last_token = \
	self._GetVarTokensUpTo(syntax, '(', ';', '{', '[')
	temp_tokens.insert(0, token)
	if last_token.name == '(':
	# If there is an assignment before the paren,
	# this is an expression, not a method.
	expr = bool([e for e in temp_tokens if e.name == '='])
	if expr:
	new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	temp_tokens.append(last_token)
	temp_tokens.extend(new_temp)
	last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0)

	if last_token.name == '[':
	# Handle array, this isn't a method, unless it's an operator.
	# TODO(nnorwitz): keep the size somewhere.
	# unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']')
	temp_tokens.append(last_token)
	if temp_tokens[-2].name == 'operator':
	temp_tokens.append(self._GetNextToken())
	else:
	temp_tokens2, last_token = \
	self._GetVarTokensUpTo(tokenize.SYNTAX, ';')
	temp_tokens.extend(temp_tokens2)

	if last_token.name == ';':
	# Handle data, this isn't a method.
	parts = self.converter.DeclarationToParts(temp_tokens, True)
	(name, type_name, templated_types, modifiers, default,
	unused_other_tokens) = parts

	t0 = temp_tokens[0]
	names = [t.name for t in temp_tokens]
	if templated_types:
	start, end = self.converter.GetTemplateIndices(names)
	names = names[:start] + names[end:]
	default = ''.join([t.name for t in default])
	return self._CreateVariable(t0, name, type_name, modifiers,
	names, templated_types, default)
	if last_token.name == '{':
	self._AddBackTokens(temp_tokens[1:])
	self._AddBackToken(last_token)
	method_name = temp_tokens[0].name
	method = getattr(self, 'handle_' + method_name, None)
	if not method:
	# Must be declaring a variable.
	# TODO(nnorwitz): handle the declaration.
	return None
	return method()
	return self._GetMethod(temp_tokens, 0, None, False)
	elif token.token_type == tokenize.SYNTAX:
	if token.name == '~' and self.in_class:
	# Must be a dtor (probably not in method body).
	token = self._GetNextToken()
	# self.in_class can contain A::Name, but the dtor will only
	# be Name. Make sure to compare against the right value.
	if (token.token_type == tokenize.NAME and
	token.name == self.in_class_name_only):
	return self._GetMethod([token], FUNCTION_DTOR, None, True)
	# TODO(nnorwitz): handle a lot more syntax.
	elif token.token_type == tokenize.PREPROCESSOR:
	# TODO(nnorwitz): handle more preprocessor directives.
	# token starts with a #, so remove it and strip whitespace.
	name = token.name[1:].lstrip()
	if name.startswith('include'):
	# Remove "include".
	name = name[7:].strip()
	assert name
	# Handle #include \<newline> "header-on-second-line.h".
	if name.startswith('\\'):
	name = name[1:].strip()
	assert name[0] in '<"', token
	assert name[-1] in '>"', token
	system = name[0] == '<'
	filename = name[1:-1]
	return Include(token.start, token.end, filename, system)
	if name.startswith('define'):
	# Remove "define".
	name = name[6:].strip()
	assert name
	value = ''
	for i, c in enumerate(name):
	if c.isspace():
	value = name[i:].lstrip()
	name = name[:i]
	break
	return Define(token.start, token.end, name, value)
	if name.startswith('if') and name[2:3].isspace():
	condition = name[3:].strip()
	if condition.startswith('0') or condition.startswith('(0)'):
	self._SkipIf0Blocks()
	return None

	def _GetTokensUpTo(self, expected_token_type, expected_token):
	return self._GetVarTokensUpTo(expected_token_type, expected_token)[0]

	def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens):
	last_token = self._GetNextToken()
	tokens = []
	while (last_token.token_type != expected_token_type or
	last_token.name not in expected_tokens):
	tokens.append(last_token)
	last_token = self._GetNextToken()
	return tokens, last_token

	# TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necessary.
	def _IgnoreUpTo(self, token_type, token):
	unused_tokens = self._GetTokensUpTo(token_type, token)

	def _SkipIf0Blocks(self):
	count = 1
	while 1:
	token = self._GetNextToken()
	if token.token_type != tokenize.PREPROCESSOR:
	continue

	name = token.name[1:].lstrip()
	if name.startswith('endif'):
	count -= 1
	if count == 0:
	break
	elif name.startswith('if'):
	count += 1

	def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None):
	if GetNextToken is None:
	GetNextToken = self._GetNextToken
	# Assumes the current token is open_paren and we will consume
	# and return up to the close_paren.
	count = 1
	token = GetNextToken()
	while 1:
	if token.token_type == tokenize.SYNTAX:
	if token.name == open_paren:
	count += 1
	elif token.name == close_paren:
	count -= 1
	if count == 0:
	break
	yield token
	token = GetNextToken()
	yield token

	def _GetParameters(self):
	return self._GetMatchingChar('(', ')')

	def GetScope(self):
	return self._GetMatchingChar('{', '}')

	def _GetNextToken(self):
	if self.token_queue:
	return self.token_queue.pop()
	return next(self.tokens)

	def _AddBackToken(self, token):
	if token.whence == tokenize.WHENCE_STREAM:
	token.whence = tokenize.WHENCE_QUEUE
	self.token_queue.insert(0, token)
	else:
	assert token.whence == tokenize.WHENCE_QUEUE, token
	self.token_queue.append(token)

	def _AddBackTokens(self, tokens):
	if tokens:
	if tokens[-1].whence == tokenize.WHENCE_STREAM:
	for token in tokens:
	token.whence = tokenize.WHENCE_QUEUE
	self.token_queue[:0] = reversed(tokens)
	else:
	assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens
	self.token_queue.extend(reversed(tokens))

	def GetName(self, seq=None):
	"""Returns ([tokens], next_token_info)."""
	GetNextToken = self._GetNextToken
	if seq is not None:
	it = iter(seq)
	GetNextToken = lambda: next(it)
	next_token = GetNextToken()
	tokens = []
	last_token_was_name = False
	while (next_token.token_type == tokenize.NAME or
	(next_token.token_type == tokenize.SYNTAX and
	next_token.name in ('::', '<'))):
	# Two NAMEs in a row means the identifier should terminate.
	# It's probably some sort of variable declaration.
	if last_token_was_name and next_token.token_type == tokenize.NAME:
	break
	last_token_was_name = next_token.token_type == tokenize.NAME
	tokens.append(next_token)
	# Handle templated names.
	if next_token.name == '<':
	tokens.extend(self._GetMatchingChar('<', '>', GetNextToken))
	last_token_was_name = True
	next_token = GetNextToken()
	return tokens, next_token

	def GetMethod(self, modifiers, templated_types):
	return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(')
	assert len(return_type_and_name) >= 1
	return self._GetMethod(return_type_and_name, modifiers, templated_types,
	False)

	def _GetMethod(self, return_type_and_name, modifiers, templated_types,
	get_paren):
	template_portion = None
	if get_paren:
	token = self._GetNextToken()
	assert token.token_type == tokenize.SYNTAX, token
	if token.name == '<':
	# Handle templatized dtors.
	template_portion = [token]
	template_portion.extend(self._GetMatchingChar('<', '>'))
	token = self._GetNextToken()
	assert token.token_type == tokenize.SYNTAX, token
	assert token.name == '(', token

	name = return_type_and_name.pop()
	# Handle templatized ctors.
	if name.name == '>':
	index = 1
	while return_type_and_name[index].name != '<':
	index += 1
	template_portion = return_type_and_name[index:] + [name]
	del return_type_and_name[index:]
	name = return_type_and_name.pop()
	elif name.name == ']':
	rt = return_type_and_name
	assert rt[-1].name == '[', return_type_and_name
	assert rt[-2].name == 'operator', return_type_and_name
	name_seq = return_type_and_name[-2:]
	del return_type_and_name[-2:]
	name = tokenize.Token(tokenize.NAME, 'operator[]',
	name_seq[0].start, name.end)
	# Get the open paren so _GetParameters() below works.
	unused_open_paren = self._GetNextToken()

	# TODO(nnorwitz): store template_portion.
	return_type = return_type_and_name
	indices = name
	if return_type:
	indices = return_type[0]

	# Force ctor for templatized ctors.
	if name.name == self.in_class and not modifiers:
	modifiers \|= FUNCTION_CTOR
	parameters = list(self._GetParameters())
	del parameters[-1] # Remove trailing ')'.

	# Handling operator() is especially weird.
	if name.name == 'operator' and not parameters:
	token = self._GetNextToken()
	assert token.name == '(', token
	parameters = list(self._GetParameters())
	del parameters[-1] # Remove trailing ')'.

	token = self._GetNextToken()
	while token.token_type == tokenize.NAME:
	modifier_token = token
	token = self._GetNextToken()
	if modifier_token.name == 'const':
	modifiers \|= FUNCTION_CONST
	elif modifier_token.name == '__attribute__':
	# TODO(nnorwitz): handle more __attribute__ details.
	modifiers \|= FUNCTION_ATTRIBUTE
	assert token.name == '(', token
	# Consume everything between the (parens).
	unused_tokens = list(self._GetMatchingChar('(', ')'))
	token = self._GetNextToken()
	elif modifier_token.name == 'throw':
	modifiers \|= FUNCTION_THROW
	assert token.name == '(', token
	# Consume everything between the (parens).
	unused_tokens = list(self._GetMatchingChar('(', ')'))
	token = self._GetNextToken()
	elif modifier_token.name == 'override':
	modifiers \|= FUNCTION_OVERRIDE
	elif modifier_token.name == modifier_token.name.upper():
	# HACK(nnorwitz): assume that all upper-case names
	# are some macro we aren't expanding.
	modifiers \|= FUNCTION_UNKNOWN_ANNOTATION
	else:
	self.HandleError('unexpected token', modifier_token)

	assert token.token_type == tokenize.SYNTAX, token
	# Handle ctor initializers.
	if token.name == ':':
	# TODO(nnorwitz): anything else to handle for initializer list?
	while token.name != ';' and token.name != '{':
	token = self._GetNextToken()

	# Handle pointer to functions that are really data but look
	# like method declarations.
	if token.name == '(':
	if parameters[0].name == '*':
	# name contains the return type.
	name = parameters.pop()
	# parameters contains the name of the data.
	modifiers = [p.name for p in parameters]
	# Already at the ( to open the parameter list.
	function_parameters = list(self._GetMatchingChar('(', ')'))
	del function_parameters[-1] # Remove trailing ')'.
	# TODO(nnorwitz): store the function_parameters.
	token = self._GetNextToken()
	assert token.token_type == tokenize.SYNTAX, token
	assert token.name == ';', token
	return self._CreateVariable(indices, name.name, indices.name,
	modifiers, '', None)
	# At this point, we got something like:
	# return_type (type::*name_)(params);
	# This is a data member called name_ that is a function pointer.
	# With this code: void (sq_type::*field_)(string&);
	# We get: name=void return_type=[] parameters=sq_type ... field_
	# TODO(nnorwitz): is return_type always empty?
	# TODO(nnorwitz): this isn't even close to being correct.
	# Just put in something so we don't crash and can move on.
	real_name = parameters[-1]
	modifiers = [p.name for p in self._GetParameters()]
	del modifiers[-1] # Remove trailing ')'.
	return self._CreateVariable(indices, real_name.name, indices.name,
	modifiers, '', None)

	if token.name == '{':
	body = list(self.GetScope())
	del body[-1] # Remove trailing '}'.
	else:
	body = None
	if token.name == '=':
	token = self._GetNextToken()

	if token.name == 'default' or token.name == 'delete':
	# Ignore explicitly defaulted and deleted special members
	# in C++11.
	token = self._GetNextToken()
	else:
	# Handle pure-virtual declarations.
	assert token.token_type == tokenize.CONSTANT, token
	assert token.name == '0', token
	modifiers \|= FUNCTION_PURE_VIRTUAL
	token = self._GetNextToken()

	if token.name == '[':
	# TODO(nnorwitz): store tokens and improve parsing.
	# template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N];
	tokens = list(self._GetMatchingChar('[', ']'))
	token = self._GetNextToken()

	assert token.name == ';', (token, return_type_and_name, parameters)

	# Looks like we got a method, not a function.
	if len(return_type) > 2 and return_type[-1].name == '::':
	return_type, in_class = \
	self._GetReturnTypeAndClassName(return_type)
	return Method(indices.start, indices.end, name.name, in_class,
	return_type, parameters, modifiers, templated_types,
	body, self.namespace_stack)
	return Function(indices.start, indices.end, name.name, return_type,
	parameters, modifiers, templated_types, body,
	self.namespace_stack)

	def _GetReturnTypeAndClassName(self, token_seq):
	# Splitting the return type from the class name in a method
	# can be tricky. For example, Return::Type::Is::Hard::To::Find().
	# Where is the return type and where is the class name?
	# The heuristic used is to pull the last name as the class name.
	# This includes all the templated type info.
	# TODO(nnorwitz): if there is only One name like in the
	# example above, punt and assume the last bit is the class name.

	# Ignore a :: prefix, if exists so we can find the first real name.
	i = 0
	if token_seq[0].name == '::':
	i = 1
	# Ignore a :: suffix, if exists.
	end = len(token_seq) - 1
	if token_seq[end-1].name == '::':
	end -= 1

	# Make a copy of the sequence so we can append a sentinel
	# value. This is required for GetName will has to have some
	# terminating condition beyond the last name.
	seq_copy = token_seq[i:end]
	seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0))
	names = []
	while i < end:
	# Iterate through the sequence parsing out each name.
	new_name, next = self.GetName(seq_copy[i:])
	assert new_name, 'Got empty new_name, next=%s' % next
	# We got a pointer or ref. Add it to the name.
	if next and next.token_type == tokenize.SYNTAX:
	new_name.append(next)
	names.append(new_name)
	i += len(new_name)

	# Now that we have the names, it's time to undo what we did.

	# Remove the sentinel value.
	names[-1].pop()
	# Flatten the token sequence for the return type.
	return_type = [e for seq in names[:-1] for e in seq]
	# The class name is the last name.
	class_name = names[-1]
	return return_type, class_name

	def handle_bool(self):
	pass

	def handle_char(self):
	pass

	def handle_int(self):
	pass

	def handle_long(self):
	pass

	def handle_short(self):
	pass

	def handle_double(self):
	pass

	def handle_float(self):
	pass

	def handle_void(self):
	pass

	def handle_wchar_t(self):
	pass

	def handle_unsigned(self):
	pass

	def handle_signed(self):
	pass

	def _GetNestedType(self, ctor):
	name = None
	name_tokens, token = self.GetName()
	if name_tokens:
	name = ''.join([t.name for t in name_tokens])

	# Handle forward declarations.
	if token.token_type == tokenize.SYNTAX and token.name == ';':
	return ctor(token.start, token.end, name, None,
	self.namespace_stack)

	if token.token_type == tokenize.NAME and self._handling_typedef:
	self._AddBackToken(token)
	return ctor(token.start, token.end, name, None,
	self.namespace_stack)

	# Must be the type declaration.
	fields = list(self._GetMatchingChar('{', '}'))
	del fields[-1] # Remove trailing '}'.
	if token.token_type == tokenize.SYNTAX and token.name == '{':
	next = self._GetNextToken()
	new_type = ctor(token.start, token.end, name, fields,
	self.namespace_stack)
	# A name means this is an anonymous type and the name
	# is the variable declaration.
	if next.token_type != tokenize.NAME:
	return new_type
	name = new_type
	token = next

	# Must be variable declaration using the type prefixed with keyword.
	assert token.token_type == tokenize.NAME, token
	return self._CreateVariable(token, token.name, name, [], '', None)

	def handle_struct(self):
	# Special case the handling typedef/aliasing of structs here.
	# It would be a pain to handle in the class code.
	name_tokens, var_token = self.GetName()
	if name_tokens:
	next_token = self._GetNextToken()
	is_syntax = (var_token.token_type == tokenize.SYNTAX and
	var_token.name[0] in '*&')
	is_variable = (var_token.token_type == tokenize.NAME and
	next_token.name == ';')
	variable = var_token
	if is_syntax and not is_variable:
	variable = next_token
	temp = self._GetNextToken()
	if temp.token_type == tokenize.SYNTAX and temp.name == '(':
	# Handle methods declared to return a struct.
	t0 = name_tokens[0]
	struct = tokenize.Token(tokenize.NAME, 'struct',
	t0.start-7, t0.start-2)
	type_and_name = [struct]
	type_and_name.extend(name_tokens)
	type_and_name.extend((var_token, next_token))
	return self._GetMethod(type_and_name, 0, None, False)
	assert temp.name == ';', (temp, name_tokens, var_token)
	if is_syntax or (is_variable and not self._handling_typedef):
	modifiers = ['struct']
	type_name = ''.join([t.name for t in name_tokens])
	position = name_tokens[0]
	return self._CreateVariable(position, variable.name, type_name,
	modifiers, var_token.name, None)
	name_tokens.extend((var_token, next_token))
	self._AddBackTokens(name_tokens)
	else:
	self._AddBackToken(var_token)
	return self._GetClass(Struct, VISIBILITY_PUBLIC, None)

	def handle_union(self):
	return self._GetNestedType(Union)

	def handle_enum(self):
	token = self._GetNextToken()
	if not (token.token_type == tokenize.NAME and token.name == 'class'):
	self._AddBackToken(token)
	return self._GetNestedType(Enum)

	def handle_auto(self):
	# TODO(nnorwitz): warn about using auto? Probably not since it
	# will be reclaimed and useful for C++0x.
	pass

	def handle_register(self):
	pass

	def handle_const(self):
	pass

	def handle_inline(self):
	pass

	def handle_extern(self):
	pass

	def handle_static(self):
	pass

	def handle_virtual(self):
	# What follows must be a method.
	token = token2 = self._GetNextToken()
	if token.name == 'inline':
	# HACK(nnorwitz): handle inline dtors by ignoring 'inline'.
	token2 = self._GetNextToken()
	if token2.token_type == tokenize.SYNTAX and token2.name == '~':
	return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None)
	assert token.token_type == tokenize.NAME or token.name == '::', token
	return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # )
	return_type_and_name.insert(0, token)
	if token2 is not token:
	return_type_and_name.insert(1, token2)
	return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL,
	None, False)

	def handle_volatile(self):
	pass

	def handle_mutable(self):
	pass

	def handle_public(self):
	assert self.in_class
	self.visibility = VISIBILITY_PUBLIC

	def handle_protected(self):
	assert self.in_class
	self.visibility = VISIBILITY_PROTECTED

	def handle_private(self):
	assert self.in_class
	self.visibility = VISIBILITY_PRIVATE

	def handle_friend(self):
	tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	assert tokens
	t0 = tokens[0]
	return Friend(t0.start, t0.end, tokens, self.namespace_stack)

	def handle_static_cast(self):
	pass

	def handle_const_cast(self):
	pass

	def handle_dynamic_cast(self):
	pass

	def handle_reinterpret_cast(self):
	pass

	def handle_new(self):
	pass

	def handle_delete(self):
	tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	assert tokens
	return Delete(tokens[0].start, tokens[0].end, tokens)

	def handle_typedef(self):
	token = self._GetNextToken()
	if (token.token_type == tokenize.NAME and
	keywords.IsKeyword(token.name)):
	# Token must be struct/enum/union/class.
	method = getattr(self, 'handle_' + token.name)
	self._handling_typedef = True
	tokens = [method()]
	self._handling_typedef = False
	else:
	tokens = [token]

	# Get the remainder of the typedef up to the semi-colon.
	tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';'))

	# TODO(nnorwitz): clean all this up.
	assert tokens
	name = tokens.pop()
	indices = name
	if tokens:
	indices = tokens[0]
	if not indices:
	indices = token
	if name.name == ')':
	# HACK(nnorwitz): Handle pointers to functions "properly".
	if (len(tokens) >= 4 and
	tokens[1].name == '(' and tokens[2].name == '*'):
	tokens.append(name)
	name = tokens[3]
	elif name.name == ']':
	# HACK(nnorwitz): Handle arrays properly.
	if len(tokens) >= 2:
	tokens.append(name)
	name = tokens[1]
	new_type = tokens
	if tokens and isinstance(tokens[0], tokenize.Token):
	new_type = self.converter.ToType(tokens)[0]
	return Typedef(indices.start, indices.end, name.name,
	new_type, self.namespace_stack)

	def handle_typeid(self):
	pass # Not needed yet.

	def handle_typename(self):
	pass # Not needed yet.

	def _GetTemplatedTypes(self):
	result = {}
	tokens = list(self._GetMatchingChar('<', '>'))
	len_tokens = len(tokens) - 1 # Ignore trailing '>'.
	i = 0
	while i < len_tokens:
	key = tokens[i].name
	i += 1
	if keywords.IsKeyword(key) or key == ',':
	continue
	type_name = default = None
	if i < len_tokens:
	i += 1
	if tokens[i-1].name == '=':
	assert i < len_tokens, '%s %s' % (i, tokens)
	default, unused_next_token = self.GetName(tokens[i:])
	i += len(default)
	else:
	if tokens[i-1].name != ',':
	# We got something like: Type variable.
	# Re-adjust the key (variable) and type_name (Type).
	key = tokens[i-1].name
	type_name = tokens[i-2]

	result[key] = (type_name, default)
	return result

	def handle_template(self):
	token = self._GetNextToken()
	assert token.token_type == tokenize.SYNTAX, token
	assert token.name == '<', token
	templated_types = self._GetTemplatedTypes()
	# TODO(nnorwitz): for now, just ignore the template params.
	token = self._GetNextToken()
	if token.token_type == tokenize.NAME:
	if token.name == 'class':
	return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types)
	elif token.name == 'struct':
	return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types)
	elif token.name == 'friend':
	return self.handle_friend()
	self._AddBackToken(token)
	tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';')
	tokens.append(last)
	self._AddBackTokens(tokens)
	if last.name == '(':
	return self.GetMethod(FUNCTION_NONE, templated_types)
	# Must be a variable definition.
	return None

	def handle_true(self):
	pass # Nothing to do.

	def handle_false(self):
	pass # Nothing to do.

	def handle_asm(self):
	pass # Not needed yet.

	def handle_class(self):
	return self._GetClass(Class, VISIBILITY_PRIVATE, None)

	def _GetBases(self):
	# Get base classes.
	bases = []
	while 1:
	token = self._GetNextToken()
	assert token.token_type == tokenize.NAME, token
	# TODO(nnorwitz): store kind of inheritance...maybe.
	if token.name not in ('public', 'protected', 'private'):
	# If inheritance type is not specified, it is private.
	# Just put the token back so we can form a name.
	# TODO(nnorwitz): it would be good to warn about this.
	self._AddBackToken(token)
	else:
	# Check for virtual inheritance.
	token = self._GetNextToken()
	if token.name != 'virtual':
	self._AddBackToken(token)
	else:
	# TODO(nnorwitz): store that we got virtual for this base.
	pass
	base, next_token = self.GetName()
	bases_ast = self.converter.ToType(base)
	assert len(bases_ast) == 1, bases_ast
	bases.append(bases_ast[0])
	assert next_token.token_type == tokenize.SYNTAX, next_token
	if next_token.name == '{':
	token = next_token
	break
	# Support multiple inheritance.
	assert next_token.name == ',', next_token
	return bases, token

	def _GetClass(self, class_type, visibility, templated_types):
	class_name = None
	class_token = self._GetNextToken()
	if class_token.token_type != tokenize.NAME:
	assert class_token.token_type == tokenize.SYNTAX, class_token
	token = class_token
	else:
	# Skip any macro (e.g. storage class specifiers) after the
	# 'class' keyword.
	next_token = self._GetNextToken()
	if next_token.token_type == tokenize.NAME:
	self._AddBackToken(next_token)
	else:
	self._AddBackTokens([class_token, next_token])
	name_tokens, token = self.GetName()
	class_name = ''.join([t.name for t in name_tokens])
	bases = None
	if token.token_type == tokenize.SYNTAX:
	if token.name == ';':
	# Forward declaration.
	return class_type(class_token.start, class_token.end,
	class_name, None, templated_types, None,
	self.namespace_stack)
	if token.name in '*&':
	# Inline forward declaration. Could be method or data.
	name_token = self._GetNextToken()
	next_token = self._GetNextToken()
	if next_token.name == ';':
	# Handle data
	modifiers = ['class']
	return self._CreateVariable(class_token, name_token.name,
	class_name,
	modifiers, token.name, None)
	else:
	# Assume this is a method.
	tokens = (class_token, token, name_token, next_token)
	self._AddBackTokens(tokens)
	return self.GetMethod(FUNCTION_NONE, None)
	if token.name == ':':
	bases, token = self._GetBases()

	body = None
	if token.token_type == tokenize.SYNTAX and token.name == '{':
	assert token.token_type == tokenize.SYNTAX, token
	assert token.name == '{', token

	ast = AstBuilder(self.GetScope(), self.filename, class_name,
	visibility, self.namespace_stack)
	body = list(ast.Generate())

	if not self._handling_typedef:
	token = self._GetNextToken()
	if token.token_type != tokenize.NAME:
	assert token.token_type == tokenize.SYNTAX, token
	assert token.name == ';', token
	else:
	new_class = class_type(class_token.start, class_token.end,
	class_name, bases, None,
	body, self.namespace_stack)

	modifiers = []
	return self._CreateVariable(class_token,
	token.name, new_class,
	modifiers, token.name, None)
	else:
	if not self._handling_typedef:
	self.HandleError('non-typedef token', token)
	self._AddBackToken(token)

	return class_type(class_token.start, class_token.end, class_name,
	bases, templated_types, body, self.namespace_stack)

	def handle_namespace(self):
	token = self._GetNextToken()
	# Support anonymous namespaces.
	name = None
	if token.token_type == tokenize.NAME:
	name = token.name
	token = self._GetNextToken()
	self.namespace_stack.append(name)
	assert token.token_type == tokenize.SYNTAX, token
	# Create an internal token that denotes when the namespace is complete.
	internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP,
	None, None)
	internal_token.whence = token.whence
	if token.name == '=':
	# TODO(nnorwitz): handle aliasing namespaces.
	name, next_token = self.GetName()
	assert next_token.name == ';', next_token
	self._AddBackToken(internal_token)
	else:
	assert token.name == '{', token
	tokens = list(self.GetScope())
	# Replace the trailing } with the internal namespace pop token.
	tokens[-1] = internal_token
	# Handle namespace with nothing in it.
	self._AddBackTokens(tokens)
	return None

	def handle_using(self):
	tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	assert tokens
	return Using(tokens[0].start, tokens[0].end, tokens)

	def handle_explicit(self):
	assert self.in_class
	# Nothing much to do.
	# TODO(nnorwitz): maybe verify the method name == class name.
	# This must be a ctor.
	return self.GetMethod(FUNCTION_CTOR, None)

	def handle_this(self):
	pass # Nothing to do.

	def handle_operator(self):
	# Pull off the next token(s?) and make that part of the method name.
	pass

	def handle_sizeof(self):
	pass

	def handle_case(self):
	pass

	def handle_switch(self):
	pass

	def handle_default(self):
	token = self._GetNextToken()
	assert token.token_type == tokenize.SYNTAX
	assert token.name == ':'

	def handle_if(self):
	pass

	def handle_else(self):
	pass

	def handle_return(self):
	tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	if not tokens:
	return Return(self.current_token.start, self.current_token.end, None)
	return Return(tokens[0].start, tokens[0].end, tokens)

	def handle_goto(self):
	tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';')
	assert len(tokens) == 1, str(tokens)
	return Goto(tokens[0].start, tokens[0].end, tokens[0].name)

	def handle_try(self):
	pass # Not needed yet.

	def handle_catch(self):
	pass # Not needed yet.

	def handle_throw(self):
	pass # Not needed yet.

	def handle_while(self):
	pass

	def handle_do(self):
	pass

	def handle_for(self):
	pass

	def handle_break(self):
	self._IgnoreUpTo(tokenize.SYNTAX, ';')

	def handle_continue(self):
	self._IgnoreUpTo(tokenize.SYNTAX, ';')


	def BuilderFromSource(source, filename):
	"""Utility method that returns an AstBuilder from source code.

	Args:
	source: 'C++ source code'
	filename: 'file1'

	Returns:
	AstBuilder
	"""
	return AstBuilder(tokenize.GetTokens(source), filename)


	def PrintIndentifiers(filename, should_print):
	"""Prints all identifiers for a C++ source file.

	Args:
	filename: 'file1'
	should_print: predicate with signature: bool Function(token)
	"""
	source = utils.ReadFile(filename, False)
	if source is None:
	sys.stderr.write('Unable to find: %s\n' % filename)
	return

	#print('Processing %s' % actual_filename)
	builder = BuilderFromSource(source, filename)
	try:
	for node in builder.Generate():
	if should_print(node):
	print(node.name)
	except KeyboardInterrupt:
	return
	except:
	pass


	def PrintAllIndentifiers(filenames, should_print):
	"""Prints all identifiers for each C++ source file in filenames.

	Args:
	filenames: ['file1', 'file2', ...]
	should_print: predicate with signature: bool Function(token)
	"""
	for path in filenames:
	PrintIndentifiers(path, should_print)


	def main(argv):
	for filename in argv[1:]:
	source = utils.ReadFile(filename)
	if source is None:
	continue

	print('Processing %s' % filename)
	builder = BuilderFromSource(source, filename)
	try:
	entire_ast = filter(None, builder.Generate())
	except KeyboardInterrupt:
	return
	except:
	# Already printed a warning, print the traceback and continue.
	traceback.print_exc()
	else:
	if utils.DEBUG:
	for ast in entire_ast:
	print(ast)


	if __name__ == '__main__':
	main(sys.argv)

ast.pyNo OneTemporaryActions

File Metadata

ast.pyView Options

Event Timeline

ast.py
No OneTemporary
Actions

ast.py
View Options