# HG changeset patch # User Paul Boddie # Date 1337367101 -7200 # Node ID 7bec0014e8711e78c22a1cfd621abb36c161fc9f The compiler package from Python 2.6.8 together with copyright and licensing information. diff -r 000000000000 -r 7bec0014e871 compiler/__init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/__init__.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,29 @@ +"""Package for parsing and compiling Python source code + +There are several functions defined at the top level that are imported +from modules contained in the package. + +parse(buf, mode="exec") -> AST + Converts a string containing Python source code to an abstract + syntax tree (AST). The AST is defined in compiler.ast. + +parseFile(path) -> AST + The same as parse(open(path)) + +walk(ast, visitor, verbose=None) + Does a pre-order walk over the ast using the visitor instance. + See compiler.visitor for details. + +compile(source, filename, mode, flags=None, dont_inherit=None) + Returns a code object. A replacement for the builtin compile() function. + +compileFile(filename) + Generates a .pyc file by compiling filename. +""" +from warnings import warnpy3k +warnpy3k("the compiler package has been removed in Python 3.0", stacklevel=2) +del warnpy3k + +from compiler.transformer import parse, parseFile +from compiler.visitor import walk +from compiler.pycodegen import compile, compileFile diff -r 000000000000 -r 7bec0014e871 compiler/ast.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/ast.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,1358 @@ +"""Python abstract syntax node definitions + +This file is automatically generated by Tools/compiler/astgen.py +""" +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS + +def flatten(seq): + l = [] + for elt in seq: + t = type(elt) + if t is tuple or t is list: + for elt2 in flatten(elt): + l.append(elt2) + else: + l.append(elt) + return l + +def flatten_nodes(seq): + return [n for n in flatten(seq) if isinstance(n, Node)] + +nodes = {} + +class Node: + """Abstract base class for ast nodes.""" + def getChildren(self): + pass # implemented by subclasses + def __iter__(self): + for n in self.getChildren(): + yield n + def asList(self): # for backwards compatibility + return self.getChildren() + def getChildNodes(self): + pass # implemented by subclasses + +class EmptyNode(Node): + pass + +class Expression(Node): + # Expression is an artificial node class to support "eval" + nodes["expression"] = "Expression" + def __init__(self, node): + self.node = node + + def getChildren(self): + return self.node, + + def getChildNodes(self): + return self.node, + + def __repr__(self): + return "Expression(%s)" % (repr(self.node)) + +class Add(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Add((%s, %s))" % (repr(self.left), repr(self.right)) + +class And(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "And(%s)" % (repr(self.nodes),) + +class AssAttr(Node): + def __init__(self, expr, attrname, flags, lineno=None): + self.expr = expr + self.attrname = attrname + self.flags = flags + self.lineno = lineno + + def getChildren(self): + return self.expr, self.attrname, self.flags + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "AssAttr(%s, %s, %s)" % (repr(self.expr), repr(self.attrname), repr(self.flags)) + +class AssList(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "AssList(%s)" % (repr(self.nodes),) + +class AssName(Node): + def __init__(self, name, flags, lineno=None): + self.name = name + self.flags = flags + self.lineno = lineno + + def getChildren(self): + return self.name, self.flags + + def getChildNodes(self): + return () + + def __repr__(self): + return "AssName(%s, %s)" % (repr(self.name), repr(self.flags)) + +class AssTuple(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "AssTuple(%s)" % (repr(self.nodes),) + +class Assert(Node): + def __init__(self, test, fail, lineno=None): + self.test = test + self.fail = fail + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.test) + children.append(self.fail) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.test) + if self.fail is not None: + nodelist.append(self.fail) + return tuple(nodelist) + + def __repr__(self): + return "Assert(%s, %s)" % (repr(self.test), repr(self.fail)) + +class Assign(Node): + def __init__(self, nodes, expr, lineno=None): + self.nodes = nodes + self.expr = expr + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.expr) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + nodelist.append(self.expr) + return tuple(nodelist) + + def __repr__(self): + return "Assign(%s, %s)" % (repr(self.nodes), repr(self.expr)) + +class AugAssign(Node): + def __init__(self, node, op, expr, lineno=None): + self.node = node + self.op = op + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.node, self.op, self.expr + + def getChildNodes(self): + return self.node, self.expr + + def __repr__(self): + return "AugAssign(%s, %s, %s)" % (repr(self.node), repr(self.op), repr(self.expr)) + +class Backquote(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Backquote(%s)" % (repr(self.expr),) + +class Bitand(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitand(%s)" % (repr(self.nodes),) + +class Bitor(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitor(%s)" % (repr(self.nodes),) + +class Bitxor(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Bitxor(%s)" % (repr(self.nodes),) + +class Break(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Break()" + +class CallFunc(Node): + def __init__(self, node, args, star_args = None, dstar_args = None, lineno=None): + self.node = node + self.args = args + self.star_args = star_args + self.dstar_args = dstar_args + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.node) + children.extend(flatten(self.args)) + children.append(self.star_args) + children.append(self.dstar_args) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.node) + nodelist.extend(flatten_nodes(self.args)) + if self.star_args is not None: + nodelist.append(self.star_args) + if self.dstar_args is not None: + nodelist.append(self.dstar_args) + return tuple(nodelist) + + def __repr__(self): + return "CallFunc(%s, %s, %s, %s)" % (repr(self.node), repr(self.args), repr(self.star_args), repr(self.dstar_args)) + +class Class(Node): + def __init__(self, name, bases, doc, code, decorators = None, lineno=None): + self.name = name + self.bases = bases + self.doc = doc + self.code = code + self.decorators = decorators + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.name) + children.extend(flatten(self.bases)) + children.append(self.doc) + children.append(self.code) + children.append(self.decorators) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.bases)) + nodelist.append(self.code) + if self.decorators is not None: + nodelist.append(self.decorators) + return tuple(nodelist) + + def __repr__(self): + return "Class(%s, %s, %s, %s, %s)" % (repr(self.name), repr(self.bases), repr(self.doc), repr(self.code), repr(self.decorators)) + +class Compare(Node): + def __init__(self, expr, ops, lineno=None): + self.expr = expr + self.ops = ops + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.ops)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.ops)) + return tuple(nodelist) + + def __repr__(self): + return "Compare(%s, %s)" % (repr(self.expr), repr(self.ops)) + +class Const(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Const(%s)" % (repr(self.value),) + +class Continue(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Continue()" + +class Decorators(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Decorators(%s)" % (repr(self.nodes),) + +class Dict(Node): + def __init__(self, items, lineno=None): + self.items = items + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.items)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.items)) + return tuple(nodelist) + + def __repr__(self): + return "Dict(%s)" % (repr(self.items),) + +class Discard(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Discard(%s)" % (repr(self.expr),) + +class Div(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Div((%s, %s))" % (repr(self.left), repr(self.right)) + +class Ellipsis(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Ellipsis()" + +class Exec(Node): + def __init__(self, expr, locals, globals, lineno=None): + self.expr = expr + self.locals = locals + self.globals = globals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.locals) + children.append(self.globals) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.locals is not None: + nodelist.append(self.locals) + if self.globals is not None: + nodelist.append(self.globals) + return tuple(nodelist) + + def __repr__(self): + return "Exec(%s, %s, %s)" % (repr(self.expr), repr(self.locals), repr(self.globals)) + +class FloorDiv(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "FloorDiv((%s, %s))" % (repr(self.left), repr(self.right)) + +class For(Node): + def __init__(self, assign, list, body, else_, lineno=None): + self.assign = assign + self.list = list + self.body = body + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.list) + children.append(self.body) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.list) + nodelist.append(self.body) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "For(%s, %s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.body), repr(self.else_)) + +class From(Node): + def __init__(self, modname, names, level, lineno=None): + self.modname = modname + self.names = names + self.level = level + self.lineno = lineno + + def getChildren(self): + return self.modname, self.names, self.level + + def getChildNodes(self): + return () + + def __repr__(self): + return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level)) + +class Function(Node): + def __init__(self, decorators, name, argnames, defaults, flags, doc, code, lineno=None): + self.decorators = decorators + self.name = name + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.doc = doc + self.code = code + self.lineno = lineno + self.varargs = self.kwargs = None + if flags & CO_VARARGS: + self.varargs = 1 + if flags & CO_VARKEYWORDS: + self.kwargs = 1 + + + def getChildren(self): + children = [] + children.append(self.decorators) + children.append(self.name) + children.append(self.argnames) + children.extend(flatten(self.defaults)) + children.append(self.flags) + children.append(self.doc) + children.append(self.code) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + if self.decorators is not None: + nodelist.append(self.decorators) + nodelist.extend(flatten_nodes(self.defaults)) + nodelist.append(self.code) + return tuple(nodelist) + + def __repr__(self): + return "Function(%s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.doc), repr(self.code)) + +class GenExpr(Node): + def __init__(self, code, lineno=None): + self.code = code + self.lineno = lineno + self.argnames = ['.0'] + self.varargs = self.kwargs = None + + + def getChildren(self): + return self.code, + + def getChildNodes(self): + return self.code, + + def __repr__(self): + return "GenExpr(%s)" % (repr(self.code),) + +class GenExprFor(Node): + def __init__(self, assign, iter, ifs, lineno=None): + self.assign = assign + self.iter = iter + self.ifs = ifs + self.lineno = lineno + self.is_outmost = False + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.iter) + children.extend(flatten(self.ifs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.iter) + nodelist.extend(flatten_nodes(self.ifs)) + return tuple(nodelist) + + def __repr__(self): + return "GenExprFor(%s, %s, %s)" % (repr(self.assign), repr(self.iter), repr(self.ifs)) + +class GenExprIf(Node): + def __init__(self, test, lineno=None): + self.test = test + self.lineno = lineno + + def getChildren(self): + return self.test, + + def getChildNodes(self): + return self.test, + + def __repr__(self): + return "GenExprIf(%s)" % (repr(self.test),) + +class GenExprInner(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "GenExprInner(%s, %s)" % (repr(self.expr), repr(self.quals)) + +class Getattr(Node): + def __init__(self, expr, attrname, lineno=None): + self.expr = expr + self.attrname = attrname + self.lineno = lineno + + def getChildren(self): + return self.expr, self.attrname + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Getattr(%s, %s)" % (repr(self.expr), repr(self.attrname)) + +class Global(Node): + def __init__(self, names, lineno=None): + self.names = names + self.lineno = lineno + + def getChildren(self): + return self.names, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Global(%s)" % (repr(self.names),) + +class If(Node): + def __init__(self, tests, else_, lineno=None): + self.tests = tests + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.tests)) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.tests)) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "If(%s, %s)" % (repr(self.tests), repr(self.else_)) + +class IfExp(Node): + def __init__(self, test, then, else_, lineno=None): + self.test = test + self.then = then + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + return self.test, self.then, self.else_ + + def getChildNodes(self): + return self.test, self.then, self.else_ + + def __repr__(self): + return "IfExp(%s, %s, %s)" % (repr(self.test), repr(self.then), repr(self.else_)) + +class Import(Node): + def __init__(self, names, lineno=None): + self.names = names + self.lineno = lineno + + def getChildren(self): + return self.names, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Import(%s)" % (repr(self.names),) + +class Invert(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Invert(%s)" % (repr(self.expr),) + +class Keyword(Node): + def __init__(self, name, expr, lineno=None): + self.name = name + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.name, self.expr + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr)) + +class Lambda(Node): + def __init__(self, argnames, defaults, flags, code, lineno=None): + self.argnames = argnames + self.defaults = defaults + self.flags = flags + self.code = code + self.lineno = lineno + self.varargs = self.kwargs = None + if flags & CO_VARARGS: + self.varargs = 1 + if flags & CO_VARKEYWORDS: + self.kwargs = 1 + + + def getChildren(self): + children = [] + children.append(self.argnames) + children.extend(flatten(self.defaults)) + children.append(self.flags) + children.append(self.code) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.defaults)) + nodelist.append(self.code) + return tuple(nodelist) + + def __repr__(self): + return "Lambda(%s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.code)) + +class LeftShift(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right)) + +class List(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "List(%s)" % (repr(self.nodes),) + +class ListComp(Node): + def __init__(self, expr, quals, lineno=None): + self.expr = expr + self.quals = quals + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.extend(flatten(self.quals)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.quals)) + return tuple(nodelist) + + def __repr__(self): + return "ListComp(%s, %s)" % (repr(self.expr), repr(self.quals)) + +class ListCompFor(Node): + def __init__(self, assign, list, ifs, lineno=None): + self.assign = assign + self.list = list + self.ifs = ifs + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.assign) + children.append(self.list) + children.extend(flatten(self.ifs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.assign) + nodelist.append(self.list) + nodelist.extend(flatten_nodes(self.ifs)) + return tuple(nodelist) + + def __repr__(self): + return "ListCompFor(%s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.ifs)) + +class ListCompIf(Node): + def __init__(self, test, lineno=None): + self.test = test + self.lineno = lineno + + def getChildren(self): + return self.test, + + def getChildNodes(self): + return self.test, + + def __repr__(self): + return "ListCompIf(%s)" % (repr(self.test),) + +class Mod(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Mod((%s, %s))" % (repr(self.left), repr(self.right)) + +class Module(Node): + def __init__(self, doc, node, lineno=None): + self.doc = doc + self.node = node + self.lineno = lineno + + def getChildren(self): + return self.doc, self.node + + def getChildNodes(self): + return self.node, + + def __repr__(self): + return "Module(%s, %s)" % (repr(self.doc), repr(self.node)) + +class Mul(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Mul((%s, %s))" % (repr(self.left), repr(self.right)) + +class Name(Node): + def __init__(self, name, lineno=None): + self.name = name + self.lineno = lineno + + def getChildren(self): + return self.name, + + def getChildNodes(self): + return () + + def __repr__(self): + return "Name(%s)" % (repr(self.name),) + +class Not(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "Not(%s)" % (repr(self.expr),) + +class Or(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Or(%s)" % (repr(self.nodes),) + +class Pass(Node): + def __init__(self, lineno=None): + self.lineno = lineno + + def getChildren(self): + return () + + def getChildNodes(self): + return () + + def __repr__(self): + return "Pass()" + +class Power(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Power((%s, %s))" % (repr(self.left), repr(self.right)) + +class Print(Node): + def __init__(self, nodes, dest, lineno=None): + self.nodes = nodes + self.dest = dest + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.dest) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + if self.dest is not None: + nodelist.append(self.dest) + return tuple(nodelist) + + def __repr__(self): + return "Print(%s, %s)" % (repr(self.nodes), repr(self.dest)) + +class Printnl(Node): + def __init__(self, nodes, dest, lineno=None): + self.nodes = nodes + self.dest = dest + self.lineno = lineno + + def getChildren(self): + children = [] + children.extend(flatten(self.nodes)) + children.append(self.dest) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + if self.dest is not None: + nodelist.append(self.dest) + return tuple(nodelist) + + def __repr__(self): + return "Printnl(%s, %s)" % (repr(self.nodes), repr(self.dest)) + +class Raise(Node): + def __init__(self, expr1, expr2, expr3, lineno=None): + self.expr1 = expr1 + self.expr2 = expr2 + self.expr3 = expr3 + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr1) + children.append(self.expr2) + children.append(self.expr3) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + if self.expr1 is not None: + nodelist.append(self.expr1) + if self.expr2 is not None: + nodelist.append(self.expr2) + if self.expr3 is not None: + nodelist.append(self.expr3) + return tuple(nodelist) + + def __repr__(self): + return "Raise(%s, %s, %s)" % (repr(self.expr1), repr(self.expr2), repr(self.expr3)) + +class Return(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return self.value, + + def __repr__(self): + return "Return(%s)" % (repr(self.value),) + +class RightShift(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "RightShift((%s, %s))" % (repr(self.left), repr(self.right)) + +class Slice(Node): + def __init__(self, expr, flags, lower, upper, lineno=None): + self.expr = expr + self.flags = flags + self.lower = lower + self.upper = upper + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.flags) + children.append(self.lower) + children.append(self.upper) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.lower is not None: + nodelist.append(self.lower) + if self.upper is not None: + nodelist.append(self.upper) + return tuple(nodelist) + + def __repr__(self): + return "Slice(%s, %s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.lower), repr(self.upper)) + +class Sliceobj(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Sliceobj(%s)" % (repr(self.nodes),) + +class Stmt(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Stmt(%s)" % (repr(self.nodes),) + +class Sub(Node): + def __init__(self, leftright, lineno=None): + self.left = leftright[0] + self.right = leftright[1] + self.lineno = lineno + + def getChildren(self): + return self.left, self.right + + def getChildNodes(self): + return self.left, self.right + + def __repr__(self): + return "Sub((%s, %s))" % (repr(self.left), repr(self.right)) + +class Subscript(Node): + def __init__(self, expr, flags, subs, lineno=None): + self.expr = expr + self.flags = flags + self.subs = subs + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.flags) + children.extend(flatten(self.subs)) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + nodelist.extend(flatten_nodes(self.subs)) + return tuple(nodelist) + + def __repr__(self): + return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs)) + +class TryExcept(Node): + def __init__(self, body, handlers, else_, lineno=None): + self.body = body + self.handlers = handlers + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.body) + children.extend(flatten(self.handlers)) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.body) + nodelist.extend(flatten_nodes(self.handlers)) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "TryExcept(%s, %s, %s)" % (repr(self.body), repr(self.handlers), repr(self.else_)) + +class TryFinally(Node): + def __init__(self, body, final, lineno=None): + self.body = body + self.final = final + self.lineno = lineno + + def getChildren(self): + return self.body, self.final + + def getChildNodes(self): + return self.body, self.final + + def __repr__(self): + return "TryFinally(%s, %s)" % (repr(self.body), repr(self.final)) + +class Tuple(Node): + def __init__(self, nodes, lineno=None): + self.nodes = nodes + self.lineno = lineno + + def getChildren(self): + return tuple(flatten(self.nodes)) + + def getChildNodes(self): + nodelist = [] + nodelist.extend(flatten_nodes(self.nodes)) + return tuple(nodelist) + + def __repr__(self): + return "Tuple(%s)" % (repr(self.nodes),) + +class UnaryAdd(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "UnaryAdd(%s)" % (repr(self.expr),) + +class UnarySub(Node): + def __init__(self, expr, lineno=None): + self.expr = expr + self.lineno = lineno + + def getChildren(self): + return self.expr, + + def getChildNodes(self): + return self.expr, + + def __repr__(self): + return "UnarySub(%s)" % (repr(self.expr),) + +class While(Node): + def __init__(self, test, body, else_, lineno=None): + self.test = test + self.body = body + self.else_ = else_ + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.test) + children.append(self.body) + children.append(self.else_) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.test) + nodelist.append(self.body) + if self.else_ is not None: + nodelist.append(self.else_) + return tuple(nodelist) + + def __repr__(self): + return "While(%s, %s, %s)" % (repr(self.test), repr(self.body), repr(self.else_)) + +class With(Node): + def __init__(self, expr, vars, body, lineno=None): + self.expr = expr + self.vars = vars + self.body = body + self.lineno = lineno + + def getChildren(self): + children = [] + children.append(self.expr) + children.append(self.vars) + children.append(self.body) + return tuple(children) + + def getChildNodes(self): + nodelist = [] + nodelist.append(self.expr) + if self.vars is not None: + nodelist.append(self.vars) + nodelist.append(self.body) + return tuple(nodelist) + + def __repr__(self): + return "With(%s, %s, %s)" % (repr(self.expr), repr(self.vars), repr(self.body)) + +class Yield(Node): + def __init__(self, value, lineno=None): + self.value = value + self.lineno = lineno + + def getChildren(self): + return self.value, + + def getChildNodes(self): + return self.value, + + def __repr__(self): + return "Yield(%s)" % (repr(self.value),) + +for name, obj in globals().items(): + if isinstance(obj, type) and issubclass(obj, Node): + nodes[name.lower()] = obj diff -r 000000000000 -r 7bec0014e871 compiler/consts.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/consts.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,22 @@ +# operation flags +OP_ASSIGN = 'OP_ASSIGN' +OP_DELETE = 'OP_DELETE' +OP_APPLY = 'OP_APPLY' + +SC_LOCAL = 1 +SC_GLOBAL = 2 +SC_FREE = 3 +SC_CELL = 4 +SC_UNKNOWN = 5 + +CO_OPTIMIZED = 0x0001 +CO_NEWLOCALS = 0x0002 +CO_VARARGS = 0x0004 +CO_VARKEYWORDS = 0x0008 +CO_NESTED = 0x0010 +CO_GENERATOR = 0x0020 +CO_GENERATOR_ALLOWED = 0 +CO_FUTURE_DIVISION = 0x2000 +CO_FUTURE_ABSIMPORT = 0x4000 +CO_FUTURE_WITH_STATEMENT = 0x8000 +CO_FUTURE_PRINT_FUNCTION = 0x10000 diff -r 000000000000 -r 7bec0014e871 compiler/future.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/future.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,74 @@ +"""Parser for future statements + +""" + +from compiler import ast, walk + +def is_future(stmt): + """Return true if statement is a well-formed future statement""" + if not isinstance(stmt, ast.From): + return 0 + if stmt.modname == "__future__": + return 1 + else: + return 0 + +class FutureParser: + + features = ("nested_scopes", "generators", "division", + "absolute_import", "with_statement", "print_function", + "unicode_literals") + + def __init__(self): + self.found = {} # set + + def visitModule(self, node): + stmt = node.node + for s in stmt.nodes: + if not self.check_stmt(s): + break + + def check_stmt(self, stmt): + if is_future(stmt): + for name, asname in stmt.names: + if name in self.features: + self.found[name] = 1 + else: + raise SyntaxError, \ + "future feature %s is not defined" % name + stmt.valid_future = 1 + return 1 + return 0 + + def get_features(self): + """Return list of features enabled by future statements""" + return self.found.keys() + +class BadFutureParser: + """Check for invalid future statements""" + + def visitFrom(self, node): + if hasattr(node, 'valid_future'): + return + if node.modname != "__future__": + return + raise SyntaxError, "invalid future statement " + repr(node) + +def find_futures(node): + p1 = FutureParser() + p2 = BadFutureParser() + walk(node, p1) + walk(node, p2) + return p1.get_features() + +if __name__ == "__main__": + import sys + from compiler import parseFile, walk + + for file in sys.argv[1:]: + print file + tree = parseFile(file) + v = FutureParser() + walk(tree, v) + print v.found + print diff -r 000000000000 -r 7bec0014e871 compiler/misc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/misc.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,73 @@ + +def flatten(tup): + elts = [] + for elt in tup: + if isinstance(elt, tuple): + elts = elts + flatten(elt) + else: + elts.append(elt) + return elts + +class Set: + def __init__(self): + self.elts = {} + def __len__(self): + return len(self.elts) + def __contains__(self, elt): + return elt in self.elts + def add(self, elt): + self.elts[elt] = elt + def elements(self): + return self.elts.keys() + def has_elt(self, elt): + return elt in self.elts + def remove(self, elt): + del self.elts[elt] + def copy(self): + c = Set() + c.elts.update(self.elts) + return c + +class Stack: + def __init__(self): + self.stack = [] + self.pop = self.stack.pop + def __len__(self): + return len(self.stack) + def push(self, elt): + self.stack.append(elt) + def top(self): + return self.stack[-1] + def __getitem__(self, index): # needed by visitContinue() + return self.stack[index] + +MANGLE_LEN = 256 # magic constant from compile.c + +def mangle(name, klass): + if not name.startswith('__'): + return name + if len(name) + 2 >= MANGLE_LEN: + return name + if name.endswith('__'): + return name + try: + i = 0 + while klass[i] == '_': + i = i + 1 + except IndexError: + return name + klass = klass[i:] + + tlen = len(klass) + len(name) + if tlen > MANGLE_LEN: + klass = klass[:MANGLE_LEN-tlen] + + return "_%s%s" % (klass, name) + +def set_filename(filename, tree): + """Set the filename attribute to filename on every node in tree""" + worklist = [tree] + while worklist: + node = worklist.pop(0) + node.filename = filename + worklist.extend(node.getChildNodes()) diff -r 000000000000 -r 7bec0014e871 compiler/pyassem.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/pyassem.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,818 @@ +"""A flow graph representation for Python bytecode""" + +import dis +import types +import sys + +from compiler import misc +from compiler.consts \ + import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS + +class FlowGraph: + def __init__(self): + self.current = self.entry = Block() + self.exit = Block("exit") + self.blocks = misc.Set() + self.blocks.add(self.entry) + self.blocks.add(self.exit) + + def startBlock(self, block): + if self._debug: + if self.current: + print "end", repr(self.current) + print " next", self.current.next + print " ", self.current.get_children() + print repr(block) + self.current = block + + def nextBlock(self, block=None): + # XXX think we need to specify when there is implicit transfer + # from one block to the next. might be better to represent this + # with explicit JUMP_ABSOLUTE instructions that are optimized + # out when they are unnecessary. + # + # I think this strategy works: each block has a child + # designated as "next" which is returned as the last of the + # children. because the nodes in a graph are emitted in + # reverse post order, the "next" block will always be emitted + # immediately after its parent. + # Worry: maintaining this invariant could be tricky + if block is None: + block = self.newBlock() + + # Note: If the current block ends with an unconditional + # control transfer, then it is incorrect to add an implicit + # transfer to the block graph. The current code requires + # these edges to get the blocks emitted in the right order, + # however. :-( If a client needs to remove these edges, call + # pruneEdges(). + + self.current.addNext(block) + self.startBlock(block) + + def newBlock(self): + b = Block() + self.blocks.add(b) + return b + + def startExitBlock(self): + self.startBlock(self.exit) + + _debug = 0 + + def _enable_debug(self): + self._debug = 1 + + def _disable_debug(self): + self._debug = 0 + + def emit(self, *inst): + if self._debug: + print "\t", inst + if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']: + self.current.addOutEdge(self.exit) + if len(inst) == 2 and isinstance(inst[1], Block): + self.current.addOutEdge(inst[1]) + self.current.emit(inst) + + def getBlocksInOrder(self): + """Return the blocks in reverse postorder + + i.e. each node appears before all of its successors + """ + # XXX make sure every node that doesn't have an explicit next + # is set so that next points to exit + for b in self.blocks.elements(): + if b is self.exit: + continue + if not b.next: + b.addNext(self.exit) + order = dfs_postorder(self.entry, {}) + order.reverse() + self.fixupOrder(order, self.exit) + # hack alert + if not self.exit in order: + order.append(self.exit) + + return order + + def fixupOrder(self, blocks, default_next): + """Fixup bad order introduced by DFS.""" + + # XXX This is a total mess. There must be a better way to get + # the code blocks in the right order. + + self.fixupOrderHonorNext(blocks, default_next) + self.fixupOrderForward(blocks, default_next) + + def fixupOrderHonorNext(self, blocks, default_next): + """Fix one problem with DFS. + + The DFS uses child block, but doesn't know about the special + "next" block. As a result, the DFS can order blocks so that a + block isn't next to the right block for implicit control + transfers. + """ + index = {} + for i in range(len(blocks)): + index[blocks[i]] = i + + for i in range(0, len(blocks) - 1): + b = blocks[i] + n = blocks[i + 1] + if not b.next or b.next[0] == default_next or b.next[0] == n: + continue + # The blocks are in the wrong order. Find the chain of + # blocks to insert where they belong. + cur = b + chain = [] + elt = cur + while elt.next and elt.next[0] != default_next: + chain.append(elt.next[0]) + elt = elt.next[0] + # Now remove the blocks in the chain from the current + # block list, so that they can be re-inserted. + l = [] + for b in chain: + assert index[b] > i + l.append((index[b], b)) + l.sort() + l.reverse() + for j, b in l: + del blocks[index[b]] + # Insert the chain in the proper location + blocks[i:i + 1] = [cur] + chain + # Finally, re-compute the block indexes + for i in range(len(blocks)): + index[blocks[i]] = i + + def fixupOrderForward(self, blocks, default_next): + """Make sure all JUMP_FORWARDs jump forward""" + index = {} + chains = [] + cur = [] + for b in blocks: + index[b] = len(chains) + cur.append(b) + if b.next and b.next[0] == default_next: + chains.append(cur) + cur = [] + chains.append(cur) + + while 1: + constraints = [] + + for i in range(len(chains)): + l = chains[i] + for b in l: + for c in b.get_children(): + if index[c] < i: + forward_p = 0 + for inst in b.insts: + if inst[0] == 'JUMP_FORWARD': + if inst[1] == c: + forward_p = 1 + if not forward_p: + continue + constraints.append((index[c], i)) + + if not constraints: + break + + # XXX just do one for now + # do swaps to get things in the right order + goes_before, a_chain = constraints[0] + assert a_chain > goes_before + c = chains[a_chain] + chains.remove(c) + chains.insert(goes_before, c) + + del blocks[:] + for c in chains: + for b in c: + blocks.append(b) + + def getBlocks(self): + return self.blocks.elements() + + def getRoot(self): + """Return nodes appropriate for use with dominator""" + return self.entry + + def getContainedGraphs(self): + l = [] + for b in self.getBlocks(): + l.extend(b.getContainedGraphs()) + return l + +def dfs_postorder(b, seen): + """Depth-first search of tree rooted at b, return in postorder""" + order = [] + seen[b] = b + for c in b.get_children(): + if c in seen: + continue + order = order + dfs_postorder(c, seen) + order.append(b) + return order + +class Block: + _count = 0 + + def __init__(self, label=''): + self.insts = [] + self.inEdges = misc.Set() + self.outEdges = misc.Set() + self.label = label + self.bid = Block._count + self.next = [] + Block._count = Block._count + 1 + + def __repr__(self): + if self.label: + return "" % (self.label, self.bid) + else: + return "" % (self.bid) + + def __str__(self): + insts = map(str, self.insts) + return "" % (self.label, self.bid, + '\n'.join(insts)) + + def emit(self, inst): + op = inst[0] + if op[:4] == 'JUMP': + self.outEdges.add(inst[1]) + self.insts.append(inst) + + def getInstructions(self): + return self.insts + + def addInEdge(self, block): + self.inEdges.add(block) + + def addOutEdge(self, block): + self.outEdges.add(block) + + def addNext(self, block): + self.next.append(block) + assert len(self.next) == 1, map(str, self.next) + + _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE', + 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP') + + def pruneNext(self): + """Remove bogus edge for unconditional transfers + + Each block has a next edge that accounts for implicit control + transfers, e.g. from a JUMP_IF_FALSE to the block that will be + executed if the test is true. + + These edges must remain for the current assembler code to + work. If they are removed, the dfs_postorder gets things in + weird orders. However, they shouldn't be there for other + purposes, e.g. conversion to SSA form. This method will + remove the next edge when it follows an unconditional control + transfer. + """ + try: + op, arg = self.insts[-1] + except (IndexError, ValueError): + return + if op in self._uncond_transfer: + self.next = [] + + def get_children(self): + if self.next and self.next[0] in self.outEdges: + self.outEdges.remove(self.next[0]) + return self.outEdges.elements() + self.next + + def getContainedGraphs(self): + """Return all graphs contained within this block. + + For example, a MAKE_FUNCTION block will contain a reference to + the graph for the function body. + """ + contained = [] + for inst in self.insts: + if len(inst) == 1: + continue + op = inst[1] + if hasattr(op, 'graph'): + contained.append(op.graph) + return contained + +# flags for code objects + +# the FlowGraph is transformed in place; it exists in one of these states +RAW = "RAW" +FLAT = "FLAT" +CONV = "CONV" +DONE = "DONE" + +class PyFlowGraph(FlowGraph): + super_init = FlowGraph.__init__ + + def __init__(self, name, filename, args=(), optimized=0, klass=None): + self.super_init() + self.name = name + self.filename = filename + self.docstring = None + self.args = args # XXX + self.argcount = getArgCount(args) + self.klass = klass + if optimized: + self.flags = CO_OPTIMIZED | CO_NEWLOCALS + else: + self.flags = 0 + self.consts = [] + self.names = [] + # Free variables found by the symbol table scan, including + # variables used only in nested scopes, are included here. + self.freevars = [] + self.cellvars = [] + # The closure list is used to track the order of cell + # variables and free variables in the resulting code object. + # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both + # kinds of variables. + self.closure = [] + self.varnames = list(args) or [] + for i in range(len(self.varnames)): + var = self.varnames[i] + if isinstance(var, TupleArg): + self.varnames[i] = var.getName() + self.stage = RAW + + def setDocstring(self, doc): + self.docstring = doc + + def setFlag(self, flag): + self.flags = self.flags | flag + if flag == CO_VARARGS: + self.argcount = self.argcount - 1 + + def checkFlag(self, flag): + if self.flags & flag: + return 1 + + def setFreeVars(self, names): + self.freevars = list(names) + + def setCellVars(self, names): + self.cellvars = names + + def getCode(self): + """Get a Python code object""" + assert self.stage == RAW + self.computeStackDepth() + self.flattenGraph() + assert self.stage == FLAT + self.convertArgs() + assert self.stage == CONV + self.makeByteCode() + assert self.stage == DONE + return self.newCodeObject() + + def dump(self, io=None): + if io: + save = sys.stdout + sys.stdout = io + pc = 0 + for t in self.insts: + opname = t[0] + if opname == "SET_LINENO": + print + if len(t) == 1: + print "\t", "%3d" % pc, opname + pc = pc + 1 + else: + print "\t", "%3d" % pc, opname, t[1] + pc = pc + 3 + if io: + sys.stdout = save + + def computeStackDepth(self): + """Compute the max stack depth. + + Approach is to compute the stack effect of each basic block. + Then find the path through the code with the largest total + effect. + """ + depth = {} + exit = None + for b in self.getBlocks(): + depth[b] = findDepth(b.getInstructions()) + + seen = {} + + def max_depth(b, d): + if b in seen: + return d + seen[b] = 1 + d = d + depth[b] + children = b.get_children() + if children: + return max([max_depth(c, d) for c in children]) + else: + if not b.label == "exit": + return max_depth(self.exit, d) + else: + return d + + self.stacksize = max_depth(self.entry, 0) + + def flattenGraph(self): + """Arrange the blocks in order and resolve jumps""" + assert self.stage == RAW + self.insts = insts = [] + pc = 0 + begin = {} + end = {} + for b in self.getBlocksInOrder(): + begin[b] = pc + for inst in b.getInstructions(): + insts.append(inst) + if len(inst) == 1: + pc = pc + 1 + elif inst[0] != "SET_LINENO": + # arg takes 2 bytes + pc = pc + 3 + end[b] = pc + pc = 0 + for i in range(len(insts)): + inst = insts[i] + if len(inst) == 1: + pc = pc + 1 + elif inst[0] != "SET_LINENO": + pc = pc + 3 + opname = inst[0] + if self.hasjrel.has_elt(opname): + oparg = inst[1] + offset = begin[oparg] - pc + insts[i] = opname, offset + elif self.hasjabs.has_elt(opname): + insts[i] = opname, begin[inst[1]] + self.stage = FLAT + + hasjrel = misc.Set() + for i in dis.hasjrel: + hasjrel.add(dis.opname[i]) + hasjabs = misc.Set() + for i in dis.hasjabs: + hasjabs.add(dis.opname[i]) + + def convertArgs(self): + """Convert arguments from symbolic to concrete form""" + assert self.stage == FLAT + self.consts.insert(0, self.docstring) + self.sort_cellvars() + for i in range(len(self.insts)): + t = self.insts[i] + if len(t) == 2: + opname, oparg = t + conv = self._converters.get(opname, None) + if conv: + self.insts[i] = opname, conv(self, oparg) + self.stage = CONV + + def sort_cellvars(self): + """Sort cellvars in the order of varnames and prune from freevars. + """ + cells = {} + for name in self.cellvars: + cells[name] = 1 + self.cellvars = [name for name in self.varnames + if name in cells] + for name in self.cellvars: + del cells[name] + self.cellvars = self.cellvars + cells.keys() + self.closure = self.cellvars + self.freevars + + def _lookupName(self, name, list): + """Return index of name in list, appending if necessary + + This routine uses a list instead of a dictionary, because a + dictionary can't store two different keys if the keys have the + same value but different types, e.g. 2 and 2L. The compiler + must treat these two separately, so it does an explicit type + comparison before comparing the values. + """ + t = type(name) + for i in range(len(list)): + if t == type(list[i]) and list[i] == name: + return i + end = len(list) + list.append(name) + return end + + _converters = {} + def _convert_LOAD_CONST(self, arg): + if hasattr(arg, 'getCode'): + arg = arg.getCode() + return self._lookupName(arg, self.consts) + + def _convert_LOAD_FAST(self, arg): + self._lookupName(arg, self.names) + return self._lookupName(arg, self.varnames) + _convert_STORE_FAST = _convert_LOAD_FAST + _convert_DELETE_FAST = _convert_LOAD_FAST + + def _convert_LOAD_NAME(self, arg): + if self.klass is None: + self._lookupName(arg, self.varnames) + return self._lookupName(arg, self.names) + + def _convert_NAME(self, arg): + if self.klass is None: + self._lookupName(arg, self.varnames) + return self._lookupName(arg, self.names) + _convert_STORE_NAME = _convert_NAME + _convert_DELETE_NAME = _convert_NAME + _convert_IMPORT_NAME = _convert_NAME + _convert_IMPORT_FROM = _convert_NAME + _convert_STORE_ATTR = _convert_NAME + _convert_LOAD_ATTR = _convert_NAME + _convert_DELETE_ATTR = _convert_NAME + _convert_LOAD_GLOBAL = _convert_NAME + _convert_STORE_GLOBAL = _convert_NAME + _convert_DELETE_GLOBAL = _convert_NAME + + def _convert_DEREF(self, arg): + self._lookupName(arg, self.names) + self._lookupName(arg, self.varnames) + return self._lookupName(arg, self.closure) + _convert_LOAD_DEREF = _convert_DEREF + _convert_STORE_DEREF = _convert_DEREF + + def _convert_LOAD_CLOSURE(self, arg): + self._lookupName(arg, self.varnames) + return self._lookupName(arg, self.closure) + + _cmp = list(dis.cmp_op) + def _convert_COMPARE_OP(self, arg): + return self._cmp.index(arg) + + # similarly for other opcodes... + + for name, obj in locals().items(): + if name[:9] == "_convert_": + opname = name[9:] + _converters[opname] = obj + del name, obj, opname + + def makeByteCode(self): + assert self.stage == CONV + self.lnotab = lnotab = LineAddrTable() + for t in self.insts: + opname = t[0] + if len(t) == 1: + lnotab.addCode(self.opnum[opname]) + else: + oparg = t[1] + if opname == "SET_LINENO": + lnotab.nextLine(oparg) + continue + hi, lo = twobyte(oparg) + try: + lnotab.addCode(self.opnum[opname], lo, hi) + except ValueError: + print opname, oparg + print self.opnum[opname], lo, hi + raise + self.stage = DONE + + opnum = {} + for num in range(len(dis.opname)): + opnum[dis.opname[num]] = num + del num + + def newCodeObject(self): + assert self.stage == DONE + if (self.flags & CO_NEWLOCALS) == 0: + nlocals = 0 + else: + nlocals = len(self.varnames) + argcount = self.argcount + if self.flags & CO_VARKEYWORDS: + argcount = argcount - 1 + return types.CodeType(argcount, nlocals, self.stacksize, self.flags, + self.lnotab.getCode(), self.getConsts(), + tuple(self.names), tuple(self.varnames), + self.filename, self.name, self.lnotab.firstline, + self.lnotab.getTable(), tuple(self.freevars), + tuple(self.cellvars)) + + def getConsts(self): + """Return a tuple for the const slot of the code object + + Must convert references to code (MAKE_FUNCTION) to code + objects recursively. + """ + l = [] + for elt in self.consts: + if isinstance(elt, PyFlowGraph): + elt = elt.getCode() + l.append(elt) + return tuple(l) + +def isJump(opname): + if opname[:4] == 'JUMP': + return 1 + +class TupleArg: + """Helper for marking func defs with nested tuples in arglist""" + def __init__(self, count, names): + self.count = count + self.names = names + def __repr__(self): + return "TupleArg(%s, %s)" % (self.count, self.names) + def getName(self): + return ".%d" % self.count + +def getArgCount(args): + argcount = len(args) + if args: + for arg in args: + if isinstance(arg, TupleArg): + numNames = len(misc.flatten(arg.names)) + argcount = argcount - numNames + return argcount + +def twobyte(val): + """Convert an int argument into high and low bytes""" + assert isinstance(val, int) + return divmod(val, 256) + +class LineAddrTable: + """lnotab + + This class builds the lnotab, which is documented in compile.c. + Here's a brief recap: + + For each SET_LINENO instruction after the first one, two bytes are + added to lnotab. (In some cases, multiple two-byte entries are + added.) The first byte is the distance in bytes between the + instruction for the last SET_LINENO and the current SET_LINENO. + The second byte is offset in line numbers. If either offset is + greater than 255, multiple two-byte entries are added -- see + compile.c for the delicate details. + """ + + def __init__(self): + self.code = [] + self.codeOffset = 0 + self.firstline = 0 + self.lastline = 0 + self.lastoff = 0 + self.lnotab = [] + + def addCode(self, *args): + for arg in args: + self.code.append(chr(arg)) + self.codeOffset = self.codeOffset + len(args) + + def nextLine(self, lineno): + if self.firstline == 0: + self.firstline = lineno + self.lastline = lineno + else: + # compute deltas + addr = self.codeOffset - self.lastoff + line = lineno - self.lastline + # Python assumes that lineno always increases with + # increasing bytecode address (lnotab is unsigned char). + # Depending on when SET_LINENO instructions are emitted + # this is not always true. Consider the code: + # a = (1, + # b) + # In the bytecode stream, the assignment to "a" occurs + # after the loading of "b". This works with the C Python + # compiler because it only generates a SET_LINENO instruction + # for the assignment. + if line >= 0: + push = self.lnotab.append + while addr > 255: + push(255); push(0) + addr -= 255 + while line > 255: + push(addr); push(255) + line -= 255 + addr = 0 + if addr > 0 or line > 0: + push(addr); push(line) + self.lastline = lineno + self.lastoff = self.codeOffset + + def getCode(self): + return ''.join(self.code) + + def getTable(self): + return ''.join(map(chr, self.lnotab)) + +class StackDepthTracker: + # XXX 1. need to keep track of stack depth on jumps + # XXX 2. at least partly as a result, this code is broken + + def findDepth(self, insts, debug=0): + depth = 0 + maxDepth = 0 + for i in insts: + opname = i[0] + if debug: + print i, + delta = self.effect.get(opname, None) + if delta is not None: + depth = depth + delta + else: + # now check patterns + for pat, pat_delta in self.patterns: + if opname[:len(pat)] == pat: + delta = pat_delta + depth = depth + delta + break + # if we still haven't found a match + if delta is None: + meth = getattr(self, opname, None) + if meth is not None: + depth = depth + meth(i[1]) + if depth > maxDepth: + maxDepth = depth + if debug: + print depth, maxDepth + return maxDepth + + effect = { + 'POP_TOP': -1, + 'DUP_TOP': 1, + 'LIST_APPEND': -2, + 'SLICE+1': -1, + 'SLICE+2': -1, + 'SLICE+3': -2, + 'STORE_SLICE+0': -1, + 'STORE_SLICE+1': -2, + 'STORE_SLICE+2': -2, + 'STORE_SLICE+3': -3, + 'DELETE_SLICE+0': -1, + 'DELETE_SLICE+1': -2, + 'DELETE_SLICE+2': -2, + 'DELETE_SLICE+3': -3, + 'STORE_SUBSCR': -3, + 'DELETE_SUBSCR': -2, + # PRINT_EXPR? + 'PRINT_ITEM': -1, + 'RETURN_VALUE': -1, + 'YIELD_VALUE': -1, + 'EXEC_STMT': -3, + 'BUILD_CLASS': -2, + 'STORE_NAME': -1, + 'STORE_ATTR': -2, + 'DELETE_ATTR': -1, + 'STORE_GLOBAL': -1, + 'BUILD_MAP': 1, + 'COMPARE_OP': -1, + 'STORE_FAST': -1, + 'IMPORT_STAR': -1, + 'IMPORT_NAME': -1, + 'IMPORT_FROM': 1, + 'LOAD_ATTR': 0, # unlike other loads + # close enough... + 'SETUP_EXCEPT': 3, + 'SETUP_FINALLY': 3, + 'FOR_ITER': 1, + 'WITH_CLEANUP': -1, + } + # use pattern match + patterns = [ + ('BINARY_', -1), + ('LOAD_', 1), + ] + + def UNPACK_SEQUENCE(self, count): + return count-1 + def BUILD_TUPLE(self, count): + return -count+1 + def BUILD_LIST(self, count): + return -count+1 + def CALL_FUNCTION(self, argc): + hi, lo = divmod(argc, 256) + return -(lo + hi * 2) + def CALL_FUNCTION_VAR(self, argc): + return self.CALL_FUNCTION(argc)-1 + def CALL_FUNCTION_KW(self, argc): + return self.CALL_FUNCTION(argc)-1 + def CALL_FUNCTION_VAR_KW(self, argc): + return self.CALL_FUNCTION(argc)-2 + def MAKE_FUNCTION(self, argc): + return -argc + def MAKE_CLOSURE(self, argc): + # XXX need to account for free variables too! + return -argc + def BUILD_SLICE(self, argc): + if argc == 2: + return -1 + elif argc == 3: + return -2 + def DUP_TOPX(self, argc): + return argc + +findDepth = StackDepthTracker().findDepth diff -r 000000000000 -r 7bec0014e871 compiler/pycodegen.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/pycodegen.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,1530 @@ +import imp +import os +import marshal +import struct +import sys +from cStringIO import StringIO + +from compiler import ast, parse, walk, syntax +from compiler import pyassem, misc, future, symbols +from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL +from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, + CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION, + CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION) +from compiler.pyassem import TupleArg + +# XXX The version-specific code can go, since this code only works with 2.x. +# Do we have Python 1.x or Python 2.x? +try: + VERSION = sys.version_info[0] +except AttributeError: + VERSION = 1 + +callfunc_opcode_info = { + # (Have *args, Have **args) : opcode + (0,0) : "CALL_FUNCTION", + (1,0) : "CALL_FUNCTION_VAR", + (0,1) : "CALL_FUNCTION_KW", + (1,1) : "CALL_FUNCTION_VAR_KW", +} + +LOOP = 1 +EXCEPT = 2 +TRY_FINALLY = 3 +END_FINALLY = 4 + +def compileFile(filename, display=0): + f = open(filename, 'U') + buf = f.read() + f.close() + mod = Module(buf, filename) + try: + mod.compile(display) + except SyntaxError: + raise + else: + f = open(filename + "c", "wb") + mod.dump(f) + f.close() + +def compile(source, filename, mode, flags=None, dont_inherit=None): + """Replacement for builtin compile() function""" + if flags is not None or dont_inherit is not None: + raise RuntimeError, "not implemented yet" + + if mode == "single": + gen = Interactive(source, filename) + elif mode == "exec": + gen = Module(source, filename) + elif mode == "eval": + gen = Expression(source, filename) + else: + raise ValueError("compile() 3rd arg must be 'exec' or " + "'eval' or 'single'") + gen.compile() + return gen.code + +class AbstractCompileMode: + + mode = None # defined by subclass + + def __init__(self, source, filename): + self.source = source + self.filename = filename + self.code = None + + def _get_tree(self): + tree = parse(self.source, self.mode) + misc.set_filename(self.filename, tree) + syntax.check(tree) + return tree + + def compile(self): + pass # implemented by subclass + + def getCode(self): + return self.code + +class Expression(AbstractCompileMode): + + mode = "eval" + + def compile(self): + tree = self._get_tree() + gen = ExpressionCodeGenerator(tree) + self.code = gen.getCode() + +class Interactive(AbstractCompileMode): + + mode = "single" + + def compile(self): + tree = self._get_tree() + gen = InteractiveCodeGenerator(tree) + self.code = gen.getCode() + +class Module(AbstractCompileMode): + + mode = "exec" + + def compile(self, display=0): + tree = self._get_tree() + gen = ModuleCodeGenerator(tree) + if display: + import pprint + print pprint.pprint(tree) + self.code = gen.getCode() + + def dump(self, f): + f.write(self.getPycHeader()) + marshal.dump(self.code, f) + + MAGIC = imp.get_magic() + + def getPycHeader(self): + # compile.c uses marshal to write a long directly, with + # calling the interface that would also generate a 1-byte code + # to indicate the type of the value. simplest way to get the + # same effect is to call marshal and then skip the code. + mtime = os.path.getmtime(self.filename) + mtime = struct.pack(' 0: + top = top - 1 + kind, loop_block = self.setups[top] + if kind == LOOP: + break + if kind != LOOP: + raise SyntaxError, "'continue' outside loop (%s, %d)" % \ + (node.filename, node.lineno) + self.emit('CONTINUE_LOOP', loop_block) + self.nextBlock() + elif kind == END_FINALLY: + msg = "'continue' not allowed inside 'finally' clause (%s, %d)" + raise SyntaxError, msg % (node.filename, node.lineno) + + def visitTest(self, node, jump): + end = self.newBlock() + for child in node.nodes[:-1]: + self.visit(child) + self.emit(jump, end) + self.nextBlock() + self.emit('POP_TOP') + self.visit(node.nodes[-1]) + self.nextBlock(end) + + def visitAnd(self, node): + self.visitTest(node, 'JUMP_IF_FALSE') + + def visitOr(self, node): + self.visitTest(node, 'JUMP_IF_TRUE') + + def visitIfExp(self, node): + endblock = self.newBlock() + elseblock = self.newBlock() + self.visit(node.test) + self.emit('JUMP_IF_FALSE', elseblock) + self.emit('POP_TOP') + self.visit(node.then) + self.emit('JUMP_FORWARD', endblock) + self.nextBlock(elseblock) + self.emit('POP_TOP') + self.visit(node.else_) + self.nextBlock(endblock) + + def visitCompare(self, node): + self.visit(node.expr) + cleanup = self.newBlock() + for op, code in node.ops[:-1]: + self.visit(code) + self.emit('DUP_TOP') + self.emit('ROT_THREE') + self.emit('COMPARE_OP', op) + self.emit('JUMP_IF_FALSE', cleanup) + self.nextBlock() + self.emit('POP_TOP') + # now do the last comparison + if node.ops: + op, code = node.ops[-1] + self.visit(code) + self.emit('COMPARE_OP', op) + if len(node.ops) > 1: + end = self.newBlock() + self.emit('JUMP_FORWARD', end) + self.startBlock(cleanup) + self.emit('ROT_TWO') + self.emit('POP_TOP') + self.nextBlock(end) + + # list comprehensions + __list_count = 0 + + def visitListComp(self, node): + self.set_lineno(node) + # setup list + tmpname = "$list%d" % self.__list_count + self.__list_count = self.__list_count + 1 + self.emit('BUILD_LIST', 0) + self.emit('DUP_TOP') + self._implicitNameOp('STORE', tmpname) + + stack = [] + for i, for_ in zip(range(len(node.quals)), node.quals): + start, anchor = self.visit(for_) + cont = None + for if_ in for_.ifs: + if cont is None: + cont = self.newBlock() + self.visit(if_, cont) + stack.insert(0, (start, cont, anchor)) + + self._implicitNameOp('LOAD', tmpname) + self.visit(node.expr) + self.emit('LIST_APPEND') + + for start, cont, anchor in stack: + if cont: + skip_one = self.newBlock() + self.emit('JUMP_FORWARD', skip_one) + self.startBlock(cont) + self.emit('POP_TOP') + self.nextBlock(skip_one) + self.emit('JUMP_ABSOLUTE', start) + self.startBlock(anchor) + self._implicitNameOp('DELETE', tmpname) + + self.__list_count = self.__list_count - 1 + + def visitListCompFor(self, node): + start = self.newBlock() + anchor = self.newBlock() + + self.visit(node.list) + self.emit('GET_ITER') + self.nextBlock(start) + self.set_lineno(node, force=True) + self.emit('FOR_ITER', anchor) + self.nextBlock() + self.visit(node.assign) + return start, anchor + + def visitListCompIf(self, node, branch): + self.set_lineno(node, force=True) + self.visit(node.test) + self.emit('JUMP_IF_FALSE', branch) + self.newBlock() + self.emit('POP_TOP') + + def _makeClosure(self, gen, args): + frees = gen.scope.get_free_vars() + if frees: + for name in frees: + self.emit('LOAD_CLOSURE', name) + self.emit('BUILD_TUPLE', len(frees)) + self.emit('LOAD_CONST', gen) + self.emit('MAKE_CLOSURE', args) + else: + self.emit('LOAD_CONST', gen) + self.emit('MAKE_FUNCTION', args) + + def visitGenExpr(self, node): + gen = GenExprCodeGenerator(node, self.scopes, self.class_name, + self.get_module()) + walk(node.code, gen) + gen.finish() + self.set_lineno(node) + self._makeClosure(gen, 0) + # precomputation of outmost iterable + self.visit(node.code.quals[0].iter) + self.emit('GET_ITER') + self.emit('CALL_FUNCTION', 1) + + def visitGenExprInner(self, node): + self.set_lineno(node) + # setup list + + stack = [] + for i, for_ in zip(range(len(node.quals)), node.quals): + start, anchor, end = self.visit(for_) + cont = None + for if_ in for_.ifs: + if cont is None: + cont = self.newBlock() + self.visit(if_, cont) + stack.insert(0, (start, cont, anchor, end)) + + self.visit(node.expr) + self.emit('YIELD_VALUE') + self.emit('POP_TOP') + + for start, cont, anchor, end in stack: + if cont: + skip_one = self.newBlock() + self.emit('JUMP_FORWARD', skip_one) + self.startBlock(cont) + self.emit('POP_TOP') + self.nextBlock(skip_one) + self.emit('JUMP_ABSOLUTE', start) + self.startBlock(anchor) + self.emit('POP_BLOCK') + self.setups.pop() + self.startBlock(end) + + self.emit('LOAD_CONST', None) + + def visitGenExprFor(self, node): + start = self.newBlock() + anchor = self.newBlock() + end = self.newBlock() + + self.setups.push((LOOP, start)) + self.emit('SETUP_LOOP', end) + + if node.is_outmost: + self.loadName('.0') + else: + self.visit(node.iter) + self.emit('GET_ITER') + + self.nextBlock(start) + self.set_lineno(node, force=True) + self.emit('FOR_ITER', anchor) + self.nextBlock() + self.visit(node.assign) + return start, anchor, end + + def visitGenExprIf(self, node, branch): + self.set_lineno(node, force=True) + self.visit(node.test) + self.emit('JUMP_IF_FALSE', branch) + self.newBlock() + self.emit('POP_TOP') + + # exception related + + def visitAssert(self, node): + # XXX would be interesting to implement this via a + # transformation of the AST before this stage + if __debug__: + end = self.newBlock() + self.set_lineno(node) + # XXX AssertionError appears to be special case -- it is always + # loaded as a global even if there is a local name. I guess this + # is a sort of renaming op. + self.nextBlock() + self.visit(node.test) + self.emit('JUMP_IF_TRUE', end) + self.nextBlock() + self.emit('POP_TOP') + self.emit('LOAD_GLOBAL', 'AssertionError') + if node.fail: + self.visit(node.fail) + self.emit('RAISE_VARARGS', 2) + else: + self.emit('RAISE_VARARGS', 1) + self.nextBlock(end) + self.emit('POP_TOP') + + def visitRaise(self, node): + self.set_lineno(node) + n = 0 + if node.expr1: + self.visit(node.expr1) + n = n + 1 + if node.expr2: + self.visit(node.expr2) + n = n + 1 + if node.expr3: + self.visit(node.expr3) + n = n + 1 + self.emit('RAISE_VARARGS', n) + + def visitTryExcept(self, node): + body = self.newBlock() + handlers = self.newBlock() + end = self.newBlock() + if node.else_: + lElse = self.newBlock() + else: + lElse = end + self.set_lineno(node) + self.emit('SETUP_EXCEPT', handlers) + self.nextBlock(body) + self.setups.push((EXCEPT, body)) + self.visit(node.body) + self.emit('POP_BLOCK') + self.setups.pop() + self.emit('JUMP_FORWARD', lElse) + self.startBlock(handlers) + + last = len(node.handlers) - 1 + for i in range(len(node.handlers)): + expr, target, body = node.handlers[i] + self.set_lineno(expr) + if expr: + self.emit('DUP_TOP') + self.visit(expr) + self.emit('COMPARE_OP', 'exception match') + next = self.newBlock() + self.emit('JUMP_IF_FALSE', next) + self.nextBlock() + self.emit('POP_TOP') + self.emit('POP_TOP') + if target: + self.visit(target) + else: + self.emit('POP_TOP') + self.emit('POP_TOP') + self.visit(body) + self.emit('JUMP_FORWARD', end) + if expr: + self.nextBlock(next) + else: + self.nextBlock() + if expr: # XXX + self.emit('POP_TOP') + self.emit('END_FINALLY') + if node.else_: + self.nextBlock(lElse) + self.visit(node.else_) + self.nextBlock(end) + + def visitTryFinally(self, node): + body = self.newBlock() + final = self.newBlock() + self.set_lineno(node) + self.emit('SETUP_FINALLY', final) + self.nextBlock(body) + self.setups.push((TRY_FINALLY, body)) + self.visit(node.body) + self.emit('POP_BLOCK') + self.setups.pop() + self.emit('LOAD_CONST', None) + self.nextBlock(final) + self.setups.push((END_FINALLY, final)) + self.visit(node.final) + self.emit('END_FINALLY') + self.setups.pop() + + __with_count = 0 + + def visitWith(self, node): + body = self.newBlock() + final = self.newBlock() + valuevar = "$value%d" % self.__with_count + self.__with_count += 1 + self.set_lineno(node) + self.visit(node.expr) + self.emit('DUP_TOP') + self.emit('LOAD_ATTR', '__exit__') + self.emit('ROT_TWO') + self.emit('LOAD_ATTR', '__enter__') + self.emit('CALL_FUNCTION', 0) + if node.vars is None: + self.emit('POP_TOP') + else: + self._implicitNameOp('STORE', valuevar) + self.emit('SETUP_FINALLY', final) + self.nextBlock(body) + self.setups.push((TRY_FINALLY, body)) + if node.vars is not None: + self._implicitNameOp('LOAD', valuevar) + self._implicitNameOp('DELETE', valuevar) + self.visit(node.vars) + self.visit(node.body) + self.emit('POP_BLOCK') + self.setups.pop() + self.emit('LOAD_CONST', None) + self.nextBlock(final) + self.setups.push((END_FINALLY, final)) + self.emit('WITH_CLEANUP') + self.emit('END_FINALLY') + self.setups.pop() + self.__with_count -= 1 + + # misc + + def visitDiscard(self, node): + self.set_lineno(node) + self.visit(node.expr) + self.emit('POP_TOP') + + def visitConst(self, node): + self.emit('LOAD_CONST', node.value) + + def visitKeyword(self, node): + self.emit('LOAD_CONST', node.name) + self.visit(node.expr) + + def visitGlobal(self, node): + # no code to generate + pass + + def visitName(self, node): + self.set_lineno(node) + self.loadName(node.name) + + def visitPass(self, node): + self.set_lineno(node) + + def visitImport(self, node): + self.set_lineno(node) + level = 0 if self.graph.checkFlag(CO_FUTURE_ABSIMPORT) else -1 + for name, alias in node.names: + if VERSION > 1: + self.emit('LOAD_CONST', level) + self.emit('LOAD_CONST', None) + self.emit('IMPORT_NAME', name) + mod = name.split(".")[0] + if alias: + self._resolveDots(name) + self.storeName(alias) + else: + self.storeName(mod) + + def visitFrom(self, node): + self.set_lineno(node) + level = node.level + if level == 0 and not self.graph.checkFlag(CO_FUTURE_ABSIMPORT): + level = -1 + fromlist = tuple(name for (name, alias) in node.names) + if VERSION > 1: + self.emit('LOAD_CONST', level) + self.emit('LOAD_CONST', fromlist) + self.emit('IMPORT_NAME', node.modname) + for name, alias in node.names: + if VERSION > 1: + if name == '*': + self.namespace = 0 + self.emit('IMPORT_STAR') + # There can only be one name w/ from ... import * + assert len(node.names) == 1 + return + else: + self.emit('IMPORT_FROM', name) + self._resolveDots(name) + self.storeName(alias or name) + else: + self.emit('IMPORT_FROM', name) + self.emit('POP_TOP') + + def _resolveDots(self, name): + elts = name.split(".") + if len(elts) == 1: + return + for elt in elts[1:]: + self.emit('LOAD_ATTR', elt) + + def visitGetattr(self, node): + self.visit(node.expr) + self.emit('LOAD_ATTR', self.mangle(node.attrname)) + + # next five implement assignments + + def visitAssign(self, node): + self.set_lineno(node) + self.visit(node.expr) + dups = len(node.nodes) - 1 + for i in range(len(node.nodes)): + elt = node.nodes[i] + if i < dups: + self.emit('DUP_TOP') + if isinstance(elt, ast.Node): + self.visit(elt) + + def visitAssName(self, node): + if node.flags == 'OP_ASSIGN': + self.storeName(node.name) + elif node.flags == 'OP_DELETE': + self.set_lineno(node) + self.delName(node.name) + else: + print "oops", node.flags + + def visitAssAttr(self, node): + self.visit(node.expr) + if node.flags == 'OP_ASSIGN': + self.emit('STORE_ATTR', self.mangle(node.attrname)) + elif node.flags == 'OP_DELETE': + self.emit('DELETE_ATTR', self.mangle(node.attrname)) + else: + print "warning: unexpected flags:", node.flags + print node + + def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'): + if findOp(node) != 'OP_DELETE': + self.emit(op, len(node.nodes)) + for child in node.nodes: + self.visit(child) + + if VERSION > 1: + visitAssTuple = _visitAssSequence + visitAssList = _visitAssSequence + else: + def visitAssTuple(self, node): + self._visitAssSequence(node, 'UNPACK_TUPLE') + + def visitAssList(self, node): + self._visitAssSequence(node, 'UNPACK_LIST') + + # augmented assignment + + def visitAugAssign(self, node): + self.set_lineno(node) + aug_node = wrap_aug(node.node) + self.visit(aug_node, "load") + self.visit(node.expr) + self.emit(self._augmented_opcode[node.op]) + self.visit(aug_node, "store") + + _augmented_opcode = { + '+=' : 'INPLACE_ADD', + '-=' : 'INPLACE_SUBTRACT', + '*=' : 'INPLACE_MULTIPLY', + '/=' : 'INPLACE_DIVIDE', + '//=': 'INPLACE_FLOOR_DIVIDE', + '%=' : 'INPLACE_MODULO', + '**=': 'INPLACE_POWER', + '>>=': 'INPLACE_RSHIFT', + '<<=': 'INPLACE_LSHIFT', + '&=' : 'INPLACE_AND', + '^=' : 'INPLACE_XOR', + '|=' : 'INPLACE_OR', + } + + def visitAugName(self, node, mode): + if mode == "load": + self.loadName(node.name) + elif mode == "store": + self.storeName(node.name) + + def visitAugGetattr(self, node, mode): + if mode == "load": + self.visit(node.expr) + self.emit('DUP_TOP') + self.emit('LOAD_ATTR', self.mangle(node.attrname)) + elif mode == "store": + self.emit('ROT_TWO') + self.emit('STORE_ATTR', self.mangle(node.attrname)) + + def visitAugSlice(self, node, mode): + if mode == "load": + self.visitSlice(node, 1) + elif mode == "store": + slice = 0 + if node.lower: + slice = slice | 1 + if node.upper: + slice = slice | 2 + if slice == 0: + self.emit('ROT_TWO') + elif slice == 3: + self.emit('ROT_FOUR') + else: + self.emit('ROT_THREE') + self.emit('STORE_SLICE+%d' % slice) + + def visitAugSubscript(self, node, mode): + if mode == "load": + self.visitSubscript(node, 1) + elif mode == "store": + self.emit('ROT_THREE') + self.emit('STORE_SUBSCR') + + def visitExec(self, node): + self.visit(node.expr) + if node.locals is None: + self.emit('LOAD_CONST', None) + else: + self.visit(node.locals) + if node.globals is None: + self.emit('DUP_TOP') + else: + self.visit(node.globals) + self.emit('EXEC_STMT') + + def visitCallFunc(self, node): + pos = 0 + kw = 0 + self.set_lineno(node) + self.visit(node.node) + for arg in node.args: + self.visit(arg) + if isinstance(arg, ast.Keyword): + kw = kw + 1 + else: + pos = pos + 1 + if node.star_args is not None: + self.visit(node.star_args) + if node.dstar_args is not None: + self.visit(node.dstar_args) + have_star = node.star_args is not None + have_dstar = node.dstar_args is not None + opcode = callfunc_opcode_info[have_star, have_dstar] + self.emit(opcode, kw << 8 | pos) + + def visitPrint(self, node, newline=0): + self.set_lineno(node) + if node.dest: + self.visit(node.dest) + for child in node.nodes: + if node.dest: + self.emit('DUP_TOP') + self.visit(child) + if node.dest: + self.emit('ROT_TWO') + self.emit('PRINT_ITEM_TO') + else: + self.emit('PRINT_ITEM') + if node.dest and not newline: + self.emit('POP_TOP') + + def visitPrintnl(self, node): + self.visitPrint(node, newline=1) + if node.dest: + self.emit('PRINT_NEWLINE_TO') + else: + self.emit('PRINT_NEWLINE') + + def visitReturn(self, node): + self.set_lineno(node) + self.visit(node.value) + self.emit('RETURN_VALUE') + + def visitYield(self, node): + self.set_lineno(node) + self.visit(node.value) + self.emit('YIELD_VALUE') + + # slice and subscript stuff + + def visitSlice(self, node, aug_flag=None): + # aug_flag is used by visitAugSlice + self.visit(node.expr) + slice = 0 + if node.lower: + self.visit(node.lower) + slice = slice | 1 + if node.upper: + self.visit(node.upper) + slice = slice | 2 + if aug_flag: + if slice == 0: + self.emit('DUP_TOP') + elif slice == 3: + self.emit('DUP_TOPX', 3) + else: + self.emit('DUP_TOPX', 2) + if node.flags == 'OP_APPLY': + self.emit('SLICE+%d' % slice) + elif node.flags == 'OP_ASSIGN': + self.emit('STORE_SLICE+%d' % slice) + elif node.flags == 'OP_DELETE': + self.emit('DELETE_SLICE+%d' % slice) + else: + print "weird slice", node.flags + raise + + def visitSubscript(self, node, aug_flag=None): + self.visit(node.expr) + for sub in node.subs: + self.visit(sub) + if len(node.subs) > 1: + self.emit('BUILD_TUPLE', len(node.subs)) + if aug_flag: + self.emit('DUP_TOPX', 2) + if node.flags == 'OP_APPLY': + self.emit('BINARY_SUBSCR') + elif node.flags == 'OP_ASSIGN': + self.emit('STORE_SUBSCR') + elif node.flags == 'OP_DELETE': + self.emit('DELETE_SUBSCR') + + # binary ops + + def binaryOp(self, node, op): + self.visit(node.left) + self.visit(node.right) + self.emit(op) + + def visitAdd(self, node): + return self.binaryOp(node, 'BINARY_ADD') + + def visitSub(self, node): + return self.binaryOp(node, 'BINARY_SUBTRACT') + + def visitMul(self, node): + return self.binaryOp(node, 'BINARY_MULTIPLY') + + def visitDiv(self, node): + return self.binaryOp(node, self._div_op) + + def visitFloorDiv(self, node): + return self.binaryOp(node, 'BINARY_FLOOR_DIVIDE') + + def visitMod(self, node): + return self.binaryOp(node, 'BINARY_MODULO') + + def visitPower(self, node): + return self.binaryOp(node, 'BINARY_POWER') + + def visitLeftShift(self, node): + return self.binaryOp(node, 'BINARY_LSHIFT') + + def visitRightShift(self, node): + return self.binaryOp(node, 'BINARY_RSHIFT') + + # unary ops + + def unaryOp(self, node, op): + self.visit(node.expr) + self.emit(op) + + def visitInvert(self, node): + return self.unaryOp(node, 'UNARY_INVERT') + + def visitUnarySub(self, node): + return self.unaryOp(node, 'UNARY_NEGATIVE') + + def visitUnaryAdd(self, node): + return self.unaryOp(node, 'UNARY_POSITIVE') + + def visitUnaryInvert(self, node): + return self.unaryOp(node, 'UNARY_INVERT') + + def visitNot(self, node): + return self.unaryOp(node, 'UNARY_NOT') + + def visitBackquote(self, node): + return self.unaryOp(node, 'UNARY_CONVERT') + + # bit ops + + def bitOp(self, nodes, op): + self.visit(nodes[0]) + for node in nodes[1:]: + self.visit(node) + self.emit(op) + + def visitBitand(self, node): + return self.bitOp(node.nodes, 'BINARY_AND') + + def visitBitor(self, node): + return self.bitOp(node.nodes, 'BINARY_OR') + + def visitBitxor(self, node): + return self.bitOp(node.nodes, 'BINARY_XOR') + + # object constructors + + def visitEllipsis(self, node): + self.emit('LOAD_CONST', Ellipsis) + + def visitTuple(self, node): + self.set_lineno(node) + for elt in node.nodes: + self.visit(elt) + self.emit('BUILD_TUPLE', len(node.nodes)) + + def visitList(self, node): + self.set_lineno(node) + for elt in node.nodes: + self.visit(elt) + self.emit('BUILD_LIST', len(node.nodes)) + + def visitSliceobj(self, node): + for child in node.nodes: + self.visit(child) + self.emit('BUILD_SLICE', len(node.nodes)) + + def visitDict(self, node): + self.set_lineno(node) + self.emit('BUILD_MAP', 0) + for k, v in node.items: + self.emit('DUP_TOP') + self.visit(k) + self.visit(v) + self.emit('ROT_THREE') + self.emit('STORE_SUBSCR') + +class NestedScopeMixin: + """Defines initClass() for nested scoping (Python 2.2-compatible)""" + def initClass(self): + self.__class__.NameFinder = LocalNameFinder + self.__class__.FunctionGen = FunctionCodeGenerator + self.__class__.ClassGen = ClassCodeGenerator + +class ModuleCodeGenerator(NestedScopeMixin, CodeGenerator): + __super_init = CodeGenerator.__init__ + + scopes = None + + def __init__(self, tree): + self.graph = pyassem.PyFlowGraph("", tree.filename) + self.futures = future.find_futures(tree) + self.__super_init() + walk(tree, self) + + def get_module(self): + return self + +class ExpressionCodeGenerator(NestedScopeMixin, CodeGenerator): + __super_init = CodeGenerator.__init__ + + scopes = None + futures = () + + def __init__(self, tree): + self.graph = pyassem.PyFlowGraph("", tree.filename) + self.__super_init() + walk(tree, self) + + def get_module(self): + return self + +class InteractiveCodeGenerator(NestedScopeMixin, CodeGenerator): + + __super_init = CodeGenerator.__init__ + + scopes = None + futures = () + + def __init__(self, tree): + self.graph = pyassem.PyFlowGraph("", tree.filename) + self.__super_init() + self.set_lineno(tree) + walk(tree, self) + self.emit('RETURN_VALUE') + + def get_module(self): + return self + + def visitDiscard(self, node): + # XXX Discard means it's an expression. Perhaps this is a bad + # name. + self.visit(node.expr) + self.emit('PRINT_EXPR') + +class AbstractFunctionCode: + optimized = 1 + lambdaCount = 0 + + def __init__(self, func, scopes, isLambda, class_name, mod): + self.class_name = class_name + self.module = mod + if isLambda: + klass = FunctionCodeGenerator + name = "" % klass.lambdaCount + klass.lambdaCount = klass.lambdaCount + 1 + else: + name = func.name + + args, hasTupleArg = generateArgList(func.argnames) + self.graph = pyassem.PyFlowGraph(name, func.filename, args, + optimized=1) + self.isLambda = isLambda + self.super_init() + + if not isLambda and func.doc: + self.setDocstring(func.doc) + + lnf = walk(func.code, self.NameFinder(args), verbose=0) + self.locals.push(lnf.getLocals()) + if func.varargs: + self.graph.setFlag(CO_VARARGS) + if func.kwargs: + self.graph.setFlag(CO_VARKEYWORDS) + self.set_lineno(func) + if hasTupleArg: + self.generateArgUnpack(func.argnames) + + def get_module(self): + return self.module + + def finish(self): + self.graph.startExitBlock() + if not self.isLambda: + self.emit('LOAD_CONST', None) + self.emit('RETURN_VALUE') + + def generateArgUnpack(self, args): + for i in range(len(args)): + arg = args[i] + if isinstance(arg, tuple): + self.emit('LOAD_FAST', '.%d' % (i * 2)) + self.unpackSequence(arg) + + def unpackSequence(self, tup): + if VERSION > 1: + self.emit('UNPACK_SEQUENCE', len(tup)) + else: + self.emit('UNPACK_TUPLE', len(tup)) + for elt in tup: + if isinstance(elt, tuple): + self.unpackSequence(elt) + else: + self._nameOp('STORE', elt) + + unpackTuple = unpackSequence + +class FunctionCodeGenerator(NestedScopeMixin, AbstractFunctionCode, + CodeGenerator): + super_init = CodeGenerator.__init__ # call be other init + scopes = None + + __super_init = AbstractFunctionCode.__init__ + + def __init__(self, func, scopes, isLambda, class_name, mod): + self.scopes = scopes + self.scope = scopes[func] + self.__super_init(func, scopes, isLambda, class_name, mod) + self.graph.setFreeVars(self.scope.get_free_vars()) + self.graph.setCellVars(self.scope.get_cell_vars()) + if self.scope.generator is not None: + self.graph.setFlag(CO_GENERATOR) + +class GenExprCodeGenerator(NestedScopeMixin, AbstractFunctionCode, + CodeGenerator): + super_init = CodeGenerator.__init__ # call be other init + scopes = None + + __super_init = AbstractFunctionCode.__init__ + + def __init__(self, gexp, scopes, class_name, mod): + self.scopes = scopes + self.scope = scopes[gexp] + self.__super_init(gexp, scopes, 1, class_name, mod) + self.graph.setFreeVars(self.scope.get_free_vars()) + self.graph.setCellVars(self.scope.get_cell_vars()) + self.graph.setFlag(CO_GENERATOR) + +class AbstractClassCode: + + def __init__(self, klass, scopes, module): + self.class_name = klass.name + self.module = module + self.graph = pyassem.PyFlowGraph(klass.name, klass.filename, + optimized=0, klass=1) + self.super_init() + lnf = walk(klass.code, self.NameFinder(), verbose=0) + self.locals.push(lnf.getLocals()) + self.graph.setFlag(CO_NEWLOCALS) + if klass.doc: + self.setDocstring(klass.doc) + + def get_module(self): + return self.module + + def finish(self): + self.graph.startExitBlock() + self.emit('LOAD_LOCALS') + self.emit('RETURN_VALUE') + +class ClassCodeGenerator(NestedScopeMixin, AbstractClassCode, CodeGenerator): + super_init = CodeGenerator.__init__ + scopes = None + + __super_init = AbstractClassCode.__init__ + + def __init__(self, klass, scopes, module): + self.scopes = scopes + self.scope = scopes[klass] + self.__super_init(klass, scopes, module) + self.graph.setFreeVars(self.scope.get_free_vars()) + self.graph.setCellVars(self.scope.get_cell_vars()) + self.set_lineno(klass) + self.emit("LOAD_GLOBAL", "__name__") + self.storeName("__module__") + if klass.doc: + self.emit("LOAD_CONST", klass.doc) + self.storeName('__doc__') + +def generateArgList(arglist): + """Generate an arg list marking TupleArgs""" + args = [] + extra = [] + count = 0 + for i in range(len(arglist)): + elt = arglist[i] + if isinstance(elt, str): + args.append(elt) + elif isinstance(elt, tuple): + args.append(TupleArg(i * 2, elt)) + extra.extend(misc.flatten(elt)) + count = count + 1 + else: + raise ValueError, "unexpect argument type:", elt + return args + extra, count + +def findOp(node): + """Find the op (DELETE, LOAD, STORE) in an AssTuple tree""" + v = OpFinder() + walk(node, v, verbose=0) + return v.op + +class OpFinder: + def __init__(self): + self.op = None + def visitAssName(self, node): + if self.op is None: + self.op = node.flags + elif self.op != node.flags: + raise ValueError, "mixed ops in stmt" + visitAssAttr = visitAssName + visitSubscript = visitAssName + +class Delegator: + """Base class to support delegation for augmented assignment nodes + + To generator code for augmented assignments, we use the following + wrapper classes. In visitAugAssign, the left-hand expression node + is visited twice. The first time the visit uses the normal method + for that node . The second time the visit uses a different method + that generates the appropriate code to perform the assignment. + These delegator classes wrap the original AST nodes in order to + support the variant visit methods. + """ + def __init__(self, obj): + self.obj = obj + + def __getattr__(self, attr): + return getattr(self.obj, attr) + +class AugGetattr(Delegator): + pass + +class AugName(Delegator): + pass + +class AugSlice(Delegator): + pass + +class AugSubscript(Delegator): + pass + +wrapper = { + ast.Getattr: AugGetattr, + ast.Name: AugName, + ast.Slice: AugSlice, + ast.Subscript: AugSubscript, + } + +def wrap_aug(node): + return wrapper[node.__class__](node) + +if __name__ == "__main__": + for file in sys.argv[1:]: + compileFile(file) diff -r 000000000000 -r 7bec0014e871 compiler/symbols.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/symbols.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,461 @@ +"""Module symbol-table generator""" + +from compiler import ast +from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN +from compiler.misc import mangle +import types + + +import sys + +MANGLE_LEN = 256 + +class Scope: + # XXX how much information do I need about each name? + def __init__(self, name, module, klass=None): + self.name = name + self.module = module + self.defs = {} + self.uses = {} + self.globals = {} + self.params = {} + self.frees = {} + self.cells = {} + self.children = [] + # nested is true if the class could contain free variables, + # i.e. if it is nested within another function. + self.nested = None + self.generator = None + self.klass = None + if klass is not None: + for i in range(len(klass)): + if klass[i] != '_': + self.klass = klass[i:] + break + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.name) + + def mangle(self, name): + if self.klass is None: + return name + return mangle(name, self.klass) + + def add_def(self, name): + self.defs[self.mangle(name)] = 1 + + def add_use(self, name): + self.uses[self.mangle(name)] = 1 + + def add_global(self, name): + name = self.mangle(name) + if name in self.uses or name in self.defs: + pass # XXX warn about global following def/use + if name in self.params: + raise SyntaxError, "%s in %s is global and parameter" % \ + (name, self.name) + self.globals[name] = 1 + self.module.add_def(name) + + def add_param(self, name): + name = self.mangle(name) + self.defs[name] = 1 + self.params[name] = 1 + + def get_names(self): + d = {} + d.update(self.defs) + d.update(self.uses) + d.update(self.globals) + return d.keys() + + def add_child(self, child): + self.children.append(child) + + def get_children(self): + return self.children + + def DEBUG(self): + print >> sys.stderr, self.name, self.nested and "nested" or "" + print >> sys.stderr, "\tglobals: ", self.globals + print >> sys.stderr, "\tcells: ", self.cells + print >> sys.stderr, "\tdefs: ", self.defs + print >> sys.stderr, "\tuses: ", self.uses + print >> sys.stderr, "\tfrees:", self.frees + + def check_name(self, name): + """Return scope of name. + + The scope of a name could be LOCAL, GLOBAL, FREE, or CELL. + """ + if name in self.globals: + return SC_GLOBAL + if name in self.cells: + return SC_CELL + if name in self.defs: + return SC_LOCAL + if self.nested and (name in self.frees or name in self.uses): + return SC_FREE + if self.nested: + return SC_UNKNOWN + else: + return SC_GLOBAL + + def get_free_vars(self): + if not self.nested: + return () + free = {} + free.update(self.frees) + for name in self.uses.keys(): + if name not in self.defs and name not in self.globals: + free[name] = 1 + return free.keys() + + def handle_children(self): + for child in self.children: + frees = child.get_free_vars() + globals = self.add_frees(frees) + for name in globals: + child.force_global(name) + + def force_global(self, name): + """Force name to be global in scope. + + Some child of the current node had a free reference to name. + When the child was processed, it was labelled a free + variable. Now that all its enclosing scope have been + processed, the name is known to be a global or builtin. So + walk back down the child chain and set the name to be global + rather than free. + + Be careful to stop if a child does not think the name is + free. + """ + self.globals[name] = 1 + if name in self.frees: + del self.frees[name] + for child in self.children: + if child.check_name(name) == SC_FREE: + child.force_global(name) + + def add_frees(self, names): + """Process list of free vars from nested scope. + + Returns a list of names that are either 1) declared global in the + parent or 2) undefined in a top-level parent. In either case, + the nested scope should treat them as globals. + """ + child_globals = [] + for name in names: + sc = self.check_name(name) + if self.nested: + if sc == SC_UNKNOWN or sc == SC_FREE \ + or isinstance(self, ClassScope): + self.frees[name] = 1 + elif sc == SC_GLOBAL: + child_globals.append(name) + elif isinstance(self, FunctionScope) and sc == SC_LOCAL: + self.cells[name] = 1 + elif sc != SC_CELL: + child_globals.append(name) + else: + if sc == SC_LOCAL: + self.cells[name] = 1 + elif sc != SC_CELL: + child_globals.append(name) + return child_globals + + def get_cell_vars(self): + return self.cells.keys() + +class ModuleScope(Scope): + __super_init = Scope.__init__ + + def __init__(self): + self.__super_init("global", self) + +class FunctionScope(Scope): + pass + +class GenExprScope(Scope): + __super_init = Scope.__init__ + + __counter = 1 + + def __init__(self, module, klass=None): + i = self.__counter + self.__counter += 1 + self.__super_init("generator expression<%d>"%i, module, klass) + self.add_param('.0') + + def get_names(self): + keys = Scope.get_names(self) + return keys + +class LambdaScope(FunctionScope): + __super_init = Scope.__init__ + + __counter = 1 + + def __init__(self, module, klass=None): + i = self.__counter + self.__counter += 1 + self.__super_init("lambda.%d" % i, module, klass) + +class ClassScope(Scope): + __super_init = Scope.__init__ + + def __init__(self, name, module): + self.__super_init(name, module, name) + +class SymbolVisitor: + def __init__(self): + self.scopes = {} + self.klass = None + + # node that define new scopes + + def visitModule(self, node): + scope = self.module = self.scopes[node] = ModuleScope() + self.visit(node.node, scope) + + visitExpression = visitModule + + def visitFunction(self, node, parent): + if node.decorators: + self.visit(node.decorators, parent) + parent.add_def(node.name) + for n in node.defaults: + self.visit(n, parent) + scope = FunctionScope(node.name, self.module, self.klass) + if parent.nested or isinstance(parent, FunctionScope): + scope.nested = 1 + self.scopes[node] = scope + self._do_args(scope, node.argnames) + self.visit(node.code, scope) + self.handle_free_vars(scope, parent) + + def visitGenExpr(self, node, parent): + scope = GenExprScope(self.module, self.klass); + if parent.nested or isinstance(parent, FunctionScope) \ + or isinstance(parent, GenExprScope): + scope.nested = 1 + + self.scopes[node] = scope + self.visit(node.code, scope) + + self.handle_free_vars(scope, parent) + + def visitGenExprInner(self, node, scope): + for genfor in node.quals: + self.visit(genfor, scope) + + self.visit(node.expr, scope) + + def visitGenExprFor(self, node, scope): + self.visit(node.assign, scope, 1) + self.visit(node.iter, scope) + for if_ in node.ifs: + self.visit(if_, scope) + + def visitGenExprIf(self, node, scope): + self.visit(node.test, scope) + + def visitLambda(self, node, parent, assign=0): + # Lambda is an expression, so it could appear in an expression + # context where assign is passed. The transformer should catch + # any code that has a lambda on the left-hand side. + assert not assign + + for n in node.defaults: + self.visit(n, parent) + scope = LambdaScope(self.module, self.klass) + if parent.nested or isinstance(parent, FunctionScope): + scope.nested = 1 + self.scopes[node] = scope + self._do_args(scope, node.argnames) + self.visit(node.code, scope) + self.handle_free_vars(scope, parent) + + def _do_args(self, scope, args): + for name in args: + if type(name) == types.TupleType: + self._do_args(scope, name) + else: + scope.add_param(name) + + def handle_free_vars(self, scope, parent): + parent.add_child(scope) + scope.handle_children() + + def visitClass(self, node, parent): + parent.add_def(node.name) + for n in node.bases: + self.visit(n, parent) + scope = ClassScope(node.name, self.module) + if parent.nested or isinstance(parent, FunctionScope): + scope.nested = 1 + if node.doc is not None: + scope.add_def('__doc__') + scope.add_def('__module__') + self.scopes[node] = scope + prev = self.klass + self.klass = node.name + self.visit(node.code, scope) + self.klass = prev + self.handle_free_vars(scope, parent) + + # name can be a def or a use + + # XXX a few calls and nodes expect a third "assign" arg that is + # true if the name is being used as an assignment. only + # expressions contained within statements may have the assign arg. + + def visitName(self, node, scope, assign=0): + if assign: + scope.add_def(node.name) + else: + scope.add_use(node.name) + + # operations that bind new names + + def visitFor(self, node, scope): + self.visit(node.assign, scope, 1) + self.visit(node.list, scope) + self.visit(node.body, scope) + if node.else_: + self.visit(node.else_, scope) + + def visitFrom(self, node, scope): + for name, asname in node.names: + if name == "*": + continue + scope.add_def(asname or name) + + def visitImport(self, node, scope): + for name, asname in node.names: + i = name.find(".") + if i > -1: + name = name[:i] + scope.add_def(asname or name) + + def visitGlobal(self, node, scope): + for name in node.names: + scope.add_global(name) + + def visitAssign(self, node, scope): + """Propagate assignment flag down to child nodes. + + The Assign node doesn't itself contains the variables being + assigned to. Instead, the children in node.nodes are visited + with the assign flag set to true. When the names occur in + those nodes, they are marked as defs. + + Some names that occur in an assignment target are not bound by + the assignment, e.g. a name occurring inside a slice. The + visitor handles these nodes specially; they do not propagate + the assign flag to their children. + """ + for n in node.nodes: + self.visit(n, scope, 1) + self.visit(node.expr, scope) + + def visitAssName(self, node, scope, assign=1): + scope.add_def(node.name) + + def visitAssAttr(self, node, scope, assign=0): + self.visit(node.expr, scope, 0) + + def visitSubscript(self, node, scope, assign=0): + self.visit(node.expr, scope, 0) + for n in node.subs: + self.visit(n, scope, 0) + + def visitSlice(self, node, scope, assign=0): + self.visit(node.expr, scope, 0) + if node.lower: + self.visit(node.lower, scope, 0) + if node.upper: + self.visit(node.upper, scope, 0) + + def visitAugAssign(self, node, scope): + # If the LHS is a name, then this counts as assignment. + # Otherwise, it's just use. + self.visit(node.node, scope) + if isinstance(node.node, ast.Name): + self.visit(node.node, scope, 1) # XXX worry about this + self.visit(node.expr, scope) + + # prune if statements if tests are false + + _const_types = types.StringType, types.IntType, types.FloatType + + def visitIf(self, node, scope): + for test, body in node.tests: + if isinstance(test, ast.Const): + if type(test.value) in self._const_types: + if not test.value: + continue + self.visit(test, scope) + self.visit(body, scope) + if node.else_: + self.visit(node.else_, scope) + + # a yield statement signals a generator + + def visitYield(self, node, scope): + scope.generator = 1 + self.visit(node.value, scope) + +def list_eq(l1, l2): + return sorted(l1) == sorted(l2) + +if __name__ == "__main__": + import sys + from compiler import parseFile, walk + import symtable + + def get_names(syms): + return [s for s in [s.get_name() for s in syms.get_symbols()] + if not (s.startswith('_[') or s.startswith('.'))] + + for file in sys.argv[1:]: + print file + f = open(file) + buf = f.read() + f.close() + syms = symtable.symtable(buf, file, "exec") + mod_names = get_names(syms) + tree = parseFile(file) + s = SymbolVisitor() + walk(tree, s) + + # compare module-level symbols + names2 = s.scopes[tree].get_names() + + if not list_eq(mod_names, names2): + print + print "oops", file + print sorted(mod_names) + print sorted(names2) + sys.exit(-1) + + d = {} + d.update(s.scopes) + del d[tree] + scopes = d.values() + del d + + for s in syms.get_symbols(): + if s.is_namespace(): + l = [sc for sc in scopes + if sc.name == s.get_name()] + if len(l) > 1: + print "skipping", s.get_name() + else: + if not list_eq(get_names(s.get_namespace()), + l[0].get_names()): + print s.get_name() + print sorted(get_names(s.get_namespace())) + print sorted(l[0].get_names()) + sys.exit(-1) diff -r 000000000000 -r 7bec0014e871 compiler/syntax.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/syntax.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,46 @@ +"""Check for errs in the AST. + +The Python parser does not catch all syntax errors. Others, like +assignments with invalid targets, are caught in the code generation +phase. + +The compiler package catches some errors in the transformer module. +But it seems clearer to write checkers that use the AST to detect +errors. +""" + +from compiler import ast, walk + +def check(tree, multi=None): + v = SyntaxErrorChecker(multi) + walk(tree, v) + return v.errors + +class SyntaxErrorChecker: + """A visitor to find syntax errors in the AST.""" + + def __init__(self, multi=None): + """Create new visitor object. + + If optional argument multi is not None, then print messages + for each error rather than raising a SyntaxError for the + first. + """ + self.multi = multi + self.errors = 0 + + def error(self, node, msg): + self.errors = self.errors + 1 + if self.multi is not None: + print "%s:%s: %s" % (node.filename, node.lineno, msg) + else: + raise SyntaxError, "%s (%s:%s)" % (msg, node.filename, node.lineno) + + def visitAssign(self, node): + # the transformer module handles many of these + pass +## for target in node.nodes: +## if isinstance(target, ast.AssList): +## if target.lineno is None: +## target.lineno = node.lineno +## self.error(target, "can't assign to list comprehension") diff -r 000000000000 -r 7bec0014e871 compiler/transformer.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/transformer.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,1499 @@ +"""Parse tree transformation module. + +Transforms Python source code into an abstract syntax tree (AST) +defined in the ast module. + +The simplest ways to invoke this module are via parse and parseFile. +parse(buf) -> AST +parseFile(path) -> AST +""" + +# Original version written by Greg Stein (gstein@lyra.org) +# and Bill Tutt (rassilon@lima.mudlib.org) +# February 1997. +# +# Modifications and improvements for Python 2.0 by Jeremy Hylton and +# Mark Hammond +# +# Some fixes to try to have correct line number on almost all nodes +# (except Module, Discard and Stmt) added by Sylvain Thenault +# +# Portions of this file are: +# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved. +# +# This module is provided under a BSD-ish license. See +# http://www.opensource.org/licenses/bsd-license.html +# and replace OWNER, ORGANIZATION, and YEAR as appropriate. + +from compiler.ast import * +import parser +import symbol +import token + +class WalkerError(StandardError): + pass + +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS +from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY + +def parseFile(path): + f = open(path, "U") + # XXX The parser API tolerates files without a trailing newline, + # but not strings without a trailing newline. Always add an extra + # newline to the file contents, since we're going through the string + # version of the API. + src = f.read() + "\n" + f.close() + return parse(src) + +def parse(buf, mode="exec"): + if mode == "exec" or mode == "single": + return Transformer().parsesuite(buf) + elif mode == "eval": + return Transformer().parseexpr(buf) + else: + raise ValueError("compile() arg 3 must be" + " 'exec' or 'eval' or 'single'") + +def asList(nodes): + l = [] + for item in nodes: + if hasattr(item, "asList"): + l.append(item.asList()) + else: + if type(item) is type( (None, None) ): + l.append(tuple(asList(item))) + elif type(item) is type( [] ): + l.append(asList(item)) + else: + l.append(item) + return l + +def extractLineNo(ast): + if not isinstance(ast[1], tuple): + # get a terminal node + return ast[2] + for child in ast[1:]: + if isinstance(child, tuple): + lineno = extractLineNo(child) + if lineno is not None: + return lineno + +def Node(*args): + kind = args[0] + if kind in nodes: + try: + return nodes[kind](*args[1:]) + except TypeError: + print nodes[kind], len(args), args + raise + else: + raise WalkerError, "Can't find appropriate Node type: %s" % str(args) + #return apply(ast.Node, args) + +class Transformer: + """Utility object for transforming Python parse trees. + + Exposes the following methods: + tree = transform(ast_tree) + tree = parsesuite(text) + tree = parseexpr(text) + tree = parsefile(fileob | filename) + """ + + def __init__(self): + self._dispatch = {} + for value, name in symbol.sym_name.items(): + if hasattr(self, name): + self._dispatch[value] = getattr(self, name) + self._dispatch[token.NEWLINE] = self.com_NEWLINE + self._atom_dispatch = {token.LPAR: self.atom_lpar, + token.LSQB: self.atom_lsqb, + token.LBRACE: self.atom_lbrace, + token.BACKQUOTE: self.atom_backquote, + token.NUMBER: self.atom_number, + token.STRING: self.atom_string, + token.NAME: self.atom_name, + } + self.encoding = None + + def transform(self, tree): + """Transform an AST into a modified parse tree.""" + if not (isinstance(tree, tuple) or isinstance(tree, list)): + tree = parser.st2tuple(tree, line_info=1) + return self.compile_node(tree) + + def parsesuite(self, text): + """Return a modified parse tree for the given suite text.""" + return self.transform(parser.suite(text)) + + def parseexpr(self, text): + """Return a modified parse tree for the given expression text.""" + return self.transform(parser.expr(text)) + + def parsefile(self, file): + """Return a modified parse tree for the contents of the given file.""" + if type(file) == type(''): + file = open(file) + return self.parsesuite(file.read()) + + # -------------------------------------------------------------- + # + # PRIVATE METHODS + # + + def compile_node(self, node): + ### emit a line-number node? + n = node[0] + + if n == symbol.encoding_decl: + self.encoding = node[2] + node = node[1] + n = node[0] + + if n == symbol.single_input: + return self.single_input(node[1:]) + if n == symbol.file_input: + return self.file_input(node[1:]) + if n == symbol.eval_input: + return self.eval_input(node[1:]) + if n == symbol.lambdef: + return self.lambdef(node[1:]) + if n == symbol.funcdef: + return self.funcdef(node[1:]) + if n == symbol.classdef: + return self.classdef(node[1:]) + + raise WalkerError, ('unexpected node type', n) + + def single_input(self, node): + ### do we want to do anything about being "interactive" ? + + # NEWLINE | simple_stmt | compound_stmt NEWLINE + n = node[0][0] + if n != token.NEWLINE: + return self.com_stmt(node[0]) + + return Pass() + + def file_input(self, nodelist): + doc = self.get_docstring(nodelist, symbol.file_input) + if doc is not None: + i = 1 + else: + i = 0 + stmts = [] + for node in nodelist[i:]: + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: + self.com_append_stmt(stmts, node) + return Module(doc, Stmt(stmts)) + + def eval_input(self, nodelist): + # from the built-in function input() + ### is this sufficient? + return Expression(self.com_node(nodelist[0])) + + def decorator_name(self, nodelist): + listlen = len(nodelist) + assert listlen >= 1 and listlen % 2 == 1 + + item = self.atom_name(nodelist) + i = 1 + while i < listlen: + assert nodelist[i][0] == token.DOT + assert nodelist[i + 1][0] == token.NAME + item = Getattr(item, nodelist[i + 1][1]) + i += 2 + + return item + + def decorator(self, nodelist): + # '@' dotted_name [ '(' [arglist] ')' ] + assert len(nodelist) in (3, 5, 6) + assert nodelist[0][0] == token.AT + assert nodelist[-1][0] == token.NEWLINE + + assert nodelist[1][0] == symbol.dotted_name + funcname = self.decorator_name(nodelist[1][1:]) + + if len(nodelist) > 3: + assert nodelist[2][0] == token.LPAR + expr = self.com_call_function(funcname, nodelist[3]) + else: + expr = funcname + + return expr + + def decorators(self, nodelist): + # decorators: decorator ([NEWLINE] decorator)* NEWLINE + items = [] + for dec_nodelist in nodelist: + assert dec_nodelist[0] == symbol.decorator + items.append(self.decorator(dec_nodelist[1:])) + return Decorators(items) + + def decorated(self, nodelist): + assert nodelist[0][0] == symbol.decorators + if nodelist[1][0] == symbol.funcdef: + n = [nodelist[0]] + list(nodelist[1][1:]) + return self.funcdef(n) + elif nodelist[1][0] == symbol.classdef: + decorators = self.decorators(nodelist[0][1:]) + cls = self.classdef(nodelist[1][1:]) + cls.decorators = decorators + return cls + raise WalkerError() + + def funcdef(self, nodelist): + # -6 -5 -4 -3 -2 -1 + # funcdef: [decorators] 'def' NAME parameters ':' suite + # parameters: '(' [varargslist] ')' + + if len(nodelist) == 6: + assert nodelist[0][0] == symbol.decorators + decorators = self.decorators(nodelist[0][1:]) + else: + assert len(nodelist) == 5 + decorators = None + + lineno = nodelist[-4][2] + name = nodelist[-4][1] + args = nodelist[-3][2] + + if args[0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(args[1:]) + else: + names = defaults = () + flags = 0 + doc = self.get_docstring(nodelist[-1]) + + # code for function + code = self.com_node(nodelist[-1]) + + if doc is not None: + assert isinstance(code, Stmt) + assert isinstance(code.nodes[0], Discard) + del code.nodes[0] + return Function(decorators, name, names, defaults, flags, doc, code, + lineno=lineno) + + def lambdef(self, nodelist): + # lambdef: 'lambda' [varargslist] ':' test + if nodelist[2][0] == symbol.varargslist: + names, defaults, flags = self.com_arglist(nodelist[2][1:]) + else: + names = defaults = () + flags = 0 + + # code for lambda + code = self.com_node(nodelist[-1]) + + return Lambda(names, defaults, flags, code, lineno=nodelist[1][2]) + old_lambdef = lambdef + + def classdef(self, nodelist): + # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite + + name = nodelist[1][1] + doc = self.get_docstring(nodelist[-1]) + if nodelist[2][0] == token.COLON: + bases = [] + elif nodelist[3][0] == token.RPAR: + bases = [] + else: + bases = self.com_bases(nodelist[3]) + + # code for class + code = self.com_node(nodelist[-1]) + + if doc is not None: + assert isinstance(code, Stmt) + assert isinstance(code.nodes[0], Discard) + del code.nodes[0] + + return Class(name, bases, doc, code, lineno=nodelist[1][2]) + + def stmt(self, nodelist): + return self.com_stmt(nodelist[0]) + + small_stmt = stmt + flow_stmt = stmt + compound_stmt = stmt + + def simple_stmt(self, nodelist): + # small_stmt (';' small_stmt)* [';'] NEWLINE + stmts = [] + for i in range(0, len(nodelist), 2): + self.com_append_stmt(stmts, nodelist[i]) + return Stmt(stmts) + + def parameters(self, nodelist): + raise WalkerError + + def varargslist(self, nodelist): + raise WalkerError + + def fpdef(self, nodelist): + raise WalkerError + + def fplist(self, nodelist): + raise WalkerError + + def dotted_name(self, nodelist): + raise WalkerError + + def comp_op(self, nodelist): + raise WalkerError + + def trailer(self, nodelist): + raise WalkerError + + def sliceop(self, nodelist): + raise WalkerError + + def argument(self, nodelist): + raise WalkerError + + # -------------------------------------------------------------- + # + # STATEMENT NODES (invoked by com_node()) + # + + def expr_stmt(self, nodelist): + # augassign testlist | testlist ('=' testlist)* + en = nodelist[-1] + exprNode = self.lookup_node(en)(en[1:]) + if len(nodelist) == 1: + return Discard(exprNode, lineno=exprNode.lineno) + if nodelist[1][0] == token.EQUAL: + nodesl = [] + for i in range(0, len(nodelist) - 2, 2): + nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN)) + return Assign(nodesl, exprNode, lineno=nodelist[1][2]) + else: + lval = self.com_augassign(nodelist[0]) + op = self.com_augassign_op(nodelist[1]) + return AugAssign(lval, op[1], exprNode, lineno=op[2]) + raise WalkerError, "can't get here" + + def print_stmt(self, nodelist): + # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ]) + items = [] + if len(nodelist) == 1: + start = 1 + dest = None + elif nodelist[1][0] == token.RIGHTSHIFT: + assert len(nodelist) == 3 \ + or nodelist[3][0] == token.COMMA + dest = self.com_node(nodelist[2]) + start = 4 + else: + dest = None + start = 1 + for i in range(start, len(nodelist), 2): + items.append(self.com_node(nodelist[i])) + if nodelist[-1][0] == token.COMMA: + return Print(items, dest, lineno=nodelist[0][2]) + return Printnl(items, dest, lineno=nodelist[0][2]) + + def del_stmt(self, nodelist): + return self.com_assign(nodelist[1], OP_DELETE) + + def pass_stmt(self, nodelist): + return Pass(lineno=nodelist[0][2]) + + def break_stmt(self, nodelist): + return Break(lineno=nodelist[0][2]) + + def continue_stmt(self, nodelist): + return Continue(lineno=nodelist[0][2]) + + def return_stmt(self, nodelist): + # return: [testlist] + if len(nodelist) < 2: + return Return(Const(None), lineno=nodelist[0][2]) + return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2]) + + def yield_stmt(self, nodelist): + expr = self.com_node(nodelist[0]) + return Discard(expr, lineno=expr.lineno) + + def yield_expr(self, nodelist): + if len(nodelist) > 1: + value = self.com_node(nodelist[1]) + else: + value = Const(None) + return Yield(value, lineno=nodelist[0][2]) + + def raise_stmt(self, nodelist): + # raise: [test [',' test [',' test]]] + if len(nodelist) > 5: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + if len(nodelist) > 3: + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + if len(nodelist) > 1: + expr1 = self.com_node(nodelist[1]) + else: + expr1 = None + return Raise(expr1, expr2, expr3, lineno=nodelist[0][2]) + + def import_stmt(self, nodelist): + # import_stmt: import_name | import_from + assert len(nodelist) == 1 + return self.com_node(nodelist[0]) + + def import_name(self, nodelist): + # import_name: 'import' dotted_as_names + return Import(self.com_dotted_as_names(nodelist[1]), + lineno=nodelist[0][2]) + + def import_from(self, nodelist): + # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' | + # '(' import_as_names ')' | import_as_names) + assert nodelist[0][1] == 'from' + idx = 1 + while nodelist[idx][1] == '.': + idx += 1 + level = idx - 1 + if nodelist[idx][0] == symbol.dotted_name: + fromname = self.com_dotted_name(nodelist[idx]) + idx += 1 + else: + fromname = "" + assert nodelist[idx][1] == 'import' + if nodelist[idx + 1][0] == token.STAR: + return From(fromname, [('*', None)], level, + lineno=nodelist[0][2]) + else: + node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)] + return From(fromname, self.com_import_as_names(node), level, + lineno=nodelist[0][2]) + + def global_stmt(self, nodelist): + # global: NAME (',' NAME)* + names = [] + for i in range(1, len(nodelist), 2): + names.append(nodelist[i][1]) + return Global(names, lineno=nodelist[0][2]) + + def exec_stmt(self, nodelist): + # exec_stmt: 'exec' expr ['in' expr [',' expr]] + expr1 = self.com_node(nodelist[1]) + if len(nodelist) >= 4: + expr2 = self.com_node(nodelist[3]) + if len(nodelist) >= 6: + expr3 = self.com_node(nodelist[5]) + else: + expr3 = None + else: + expr2 = expr3 = None + + return Exec(expr1, expr2, expr3, lineno=nodelist[0][2]) + + def assert_stmt(self, nodelist): + # 'assert': test, [',' test] + expr1 = self.com_node(nodelist[1]) + if (len(nodelist) == 4): + expr2 = self.com_node(nodelist[3]) + else: + expr2 = None + return Assert(expr1, expr2, lineno=nodelist[0][2]) + + def if_stmt(self, nodelist): + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite] + tests = [] + for i in range(0, len(nodelist) - 3, 4): + testNode = self.com_node(nodelist[i + 1]) + suiteNode = self.com_node(nodelist[i + 3]) + tests.append((testNode, suiteNode)) + + if len(nodelist) % 4 == 3: + elseNode = self.com_node(nodelist[-1]) +## elseNode.lineno = nodelist[-1][1][2] + else: + elseNode = None + return If(tests, elseNode, lineno=nodelist[0][2]) + + def while_stmt(self, nodelist): + # 'while' test ':' suite ['else' ':' suite] + + testNode = self.com_node(nodelist[1]) + bodyNode = self.com_node(nodelist[3]) + + if len(nodelist) > 4: + elseNode = self.com_node(nodelist[6]) + else: + elseNode = None + + return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2]) + + def for_stmt(self, nodelist): + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] + + assignNode = self.com_assign(nodelist[1], OP_ASSIGN) + listNode = self.com_node(nodelist[3]) + bodyNode = self.com_node(nodelist[5]) + + if len(nodelist) > 8: + elseNode = self.com_node(nodelist[8]) + else: + elseNode = None + + return For(assignNode, listNode, bodyNode, elseNode, + lineno=nodelist[0][2]) + + def try_stmt(self, nodelist): + return self.com_try_except_finally(nodelist) + + def with_stmt(self, nodelist): + return self.com_with(nodelist) + + def with_var(self, nodelist): + return self.com_with_var(nodelist) + + def suite(self, nodelist): + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT + if len(nodelist) == 1: + return self.com_stmt(nodelist[0]) + + stmts = [] + for node in nodelist: + if node[0] == symbol.stmt: + self.com_append_stmt(stmts, node) + return Stmt(stmts) + + # -------------------------------------------------------------- + # + # EXPRESSION NODES (invoked by com_node()) + # + + def testlist(self, nodelist): + # testlist: expr (',' expr)* [','] + # testlist_safe: test [(',' test)+ [',']] + # exprlist: expr (',' expr)* [','] + return self.com_binary(Tuple, nodelist) + + testlist_safe = testlist # XXX + testlist1 = testlist + exprlist = testlist + + def testlist_gexp(self, nodelist): + if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: + test = self.com_node(nodelist[0]) + return self.com_generator_expression(test, nodelist[1]) + return self.testlist(nodelist) + + def test(self, nodelist): + # or_test ['if' or_test 'else' test] | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + then = self.com_node(nodelist[0]) + if len(nodelist) > 1: + assert len(nodelist) == 5 + assert nodelist[1][1] == 'if' + assert nodelist[3][1] == 'else' + test = self.com_node(nodelist[2]) + else_ = self.com_node(nodelist[4]) + return IfExp(test, then, else_, lineno=nodelist[1][2]) + return then + + def or_test(self, nodelist): + # and_test ('or' and_test)* | lambdef + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef: + return self.lambdef(nodelist[0]) + return self.com_binary(Or, nodelist) + old_test = or_test + + def and_test(self, nodelist): + # not_test ('and' not_test)* + return self.com_binary(And, nodelist) + + def not_test(self, nodelist): + # 'not' not_test | comparison + result = self.com_node(nodelist[-1]) + if len(nodelist) == 2: + return Not(result, lineno=nodelist[0][2]) + return result + + def comparison(self, nodelist): + # comparison: expr (comp_op expr)* + node = self.com_node(nodelist[0]) + if len(nodelist) == 1: + return node + + results = [] + for i in range(2, len(nodelist), 2): + nl = nodelist[i-1] + + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' + # | 'in' | 'not' 'in' | 'is' | 'is' 'not' + n = nl[1] + if n[0] == token.NAME: + type = n[1] + if len(nl) == 3: + if type == 'not': + type = 'not in' + else: + type = 'is not' + else: + type = _cmp_types[n[0]] + + lineno = nl[1][2] + results.append((type, self.com_node(nodelist[i]))) + + # we need a special "compare" node so that we can distinguish + # 3 < x < 5 from (3 < x) < 5 + # the two have very different semantics and results (note that the + # latter form is always true) + + return Compare(node, results, lineno=lineno) + + def expr(self, nodelist): + # xor_expr ('|' xor_expr)* + return self.com_binary(Bitor, nodelist) + + def xor_expr(self, nodelist): + # xor_expr ('^' xor_expr)* + return self.com_binary(Bitxor, nodelist) + + def and_expr(self, nodelist): + # xor_expr ('&' xor_expr)* + return self.com_binary(Bitand, nodelist) + + def shift_expr(self, nodelist): + # shift_expr ('<<'|'>>' shift_expr)* + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.LEFTSHIFT: + node = LeftShift([node, right], lineno=nodelist[1][2]) + elif nodelist[i-1][0] == token.RIGHTSHIFT: + node = RightShift([node, right], lineno=nodelist[1][2]) + else: + raise ValueError, "unexpected token: %s" % nodelist[i-1][0] + return node + + def arith_expr(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + if nodelist[i-1][0] == token.PLUS: + node = Add([node, right], lineno=nodelist[1][2]) + elif nodelist[i-1][0] == token.MINUS: + node = Sub([node, right], lineno=nodelist[1][2]) + else: + raise ValueError, "unexpected token: %s" % nodelist[i-1][0] + return node + + def term(self, nodelist): + node = self.com_node(nodelist[0]) + for i in range(2, len(nodelist), 2): + right = self.com_node(nodelist[i]) + t = nodelist[i-1][0] + if t == token.STAR: + node = Mul([node, right]) + elif t == token.SLASH: + node = Div([node, right]) + elif t == token.PERCENT: + node = Mod([node, right]) + elif t == token.DOUBLESLASH: + node = FloorDiv([node, right]) + else: + raise ValueError, "unexpected token: %s" % t + node.lineno = nodelist[1][2] + return node + + def factor(self, nodelist): + elt = nodelist[0] + t = elt[0] + node = self.lookup_node(nodelist[-1])(nodelist[-1][1:]) + # need to handle (unary op)constant here... + if t == token.PLUS: + return UnaryAdd(node, lineno=elt[2]) + elif t == token.MINUS: + return UnarySub(node, lineno=elt[2]) + elif t == token.TILDE: + node = Invert(node, lineno=elt[2]) + return node + + def power(self, nodelist): + # power: atom trailer* ('**' factor)* + node = self.com_node(nodelist[0]) + for i in range(1, len(nodelist)): + elt = nodelist[i] + if elt[0] == token.DOUBLESTAR: + return Power([node, self.com_node(nodelist[i+1])], + lineno=elt[2]) + + node = self.com_apply_trailer(node, elt) + + return node + + def atom(self, nodelist): + return self._atom_dispatch[nodelist[0][0]](nodelist) + + def atom_lpar(self, nodelist): + if nodelist[1][0] == token.RPAR: + return Tuple((), lineno=nodelist[0][2]) + return self.com_node(nodelist[1]) + + def atom_lsqb(self, nodelist): + if nodelist[1][0] == token.RSQB: + return List((), lineno=nodelist[0][2]) + return self.com_list_constructor(nodelist[1]) + + def atom_lbrace(self, nodelist): + if nodelist[1][0] == token.RBRACE: + return Dict((), lineno=nodelist[0][2]) + return self.com_dictmaker(nodelist[1]) + + def atom_backquote(self, nodelist): + return Backquote(self.com_node(nodelist[1])) + + def atom_number(self, nodelist): + ### need to verify this matches compile.c + k = eval(nodelist[0][1]) + return Const(k, lineno=nodelist[0][2]) + + def decode_literal(self, lit): + if self.encoding: + # this is particularly fragile & a bit of a + # hack... changes in compile.c:parsestr and + # tokenizer.c must be reflected here. + if self.encoding not in ['utf-8', 'iso-8859-1']: + lit = unicode(lit, 'utf-8').encode(self.encoding) + return eval("# coding: %s\n%s" % (self.encoding, lit)) + else: + return eval(lit) + + def atom_string(self, nodelist): + k = '' + for node in nodelist: + k += self.decode_literal(node[1]) + return Const(k, lineno=nodelist[0][2]) + + def atom_name(self, nodelist): + return Name(nodelist[0][1], lineno=nodelist[0][2]) + + # -------------------------------------------------------------- + # + # INTERNAL PARSING UTILITIES + # + + # The use of com_node() introduces a lot of extra stack frames, + # enough to cause a stack overflow compiling test.test_parser with + # the standard interpreter recursionlimit. The com_node() is a + # convenience function that hides the dispatch details, but comes + # at a very high cost. It is more efficient to dispatch directly + # in the callers. In these cases, use lookup_node() and call the + # dispatched node directly. + + def lookup_node(self, node): + return self._dispatch[node[0]] + + def com_node(self, node): + # Note: compile.c has handling in com_node for del_stmt, pass_stmt, + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt, + # and compound_stmt. + # We'll just dispatch them. + return self._dispatch[node[0]](node[1:]) + + def com_NEWLINE(self, *args): + # A ';' at the end of a line can make a NEWLINE token appear + # here, Render it harmless. (genc discards ('discard', + # ('const', xxxx)) Nodes) + return Discard(Const(None)) + + def com_arglist(self, nodelist): + # varargslist: + # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) + # | fpdef ['=' test] (',' fpdef ['=' test])* [','] + # fpdef: NAME | '(' fplist ')' + # fplist: fpdef (',' fpdef)* [','] + names = [] + defaults = [] + flags = 0 + + i = 0 + while i < len(nodelist): + node = nodelist[i] + if node[0] == token.STAR or node[0] == token.DOUBLESTAR: + if node[0] == token.STAR: + node = nodelist[i+1] + if node[0] == token.NAME: + names.append(node[1]) + flags = flags | CO_VARARGS + i = i + 3 + + if i < len(nodelist): + # should be DOUBLESTAR + t = nodelist[i][0] + if t == token.DOUBLESTAR: + node = nodelist[i+1] + else: + raise ValueError, "unexpected token: %s" % t + names.append(node[1]) + flags = flags | CO_VARKEYWORDS + + break + + # fpdef: NAME | '(' fplist ')' + names.append(self.com_fpdef(node)) + + i = i + 1 + if i < len(nodelist) and nodelist[i][0] == token.EQUAL: + defaults.append(self.com_node(nodelist[i + 1])) + i = i + 2 + elif len(defaults): + # we have already seen an argument with default, but here + # came one without + raise SyntaxError, "non-default argument follows default argument" + + # skip the comma + i = i + 1 + + return names, defaults, flags + + def com_fpdef(self, node): + # fpdef: NAME | '(' fplist ')' + if node[1][0] == token.LPAR: + return self.com_fplist(node[2]) + return node[1][1] + + def com_fplist(self, node): + # fplist: fpdef (',' fpdef)* [','] + if len(node) == 2: + return self.com_fpdef(node[1]) + list = [] + for i in range(1, len(node), 2): + list.append(self.com_fpdef(node[i])) + return tuple(list) + + def com_dotted_name(self, node): + # String together the dotted names and return the string + name = "" + for n in node: + if type(n) == type(()) and n[0] == 1: + name = name + n[1] + '.' + return name[:-1] + + def com_dotted_as_name(self, node): + assert node[0] == symbol.dotted_as_name + node = node[1:] + dot = self.com_dotted_name(node[0][1:]) + if len(node) == 1: + return dot, None + assert node[1][1] == 'as' + assert node[2][0] == token.NAME + return dot, node[2][1] + + def com_dotted_as_names(self, node): + assert node[0] == symbol.dotted_as_names + node = node[1:] + names = [self.com_dotted_as_name(node[0])] + for i in range(2, len(node), 2): + names.append(self.com_dotted_as_name(node[i])) + return names + + def com_import_as_name(self, node): + assert node[0] == symbol.import_as_name + node = node[1:] + assert node[0][0] == token.NAME + if len(node) == 1: + return node[0][1], None + assert node[1][1] == 'as', node + assert node[2][0] == token.NAME + return node[0][1], node[2][1] + + def com_import_as_names(self, node): + assert node[0] == symbol.import_as_names + node = node[1:] + names = [self.com_import_as_name(node[0])] + for i in range(2, len(node), 2): + names.append(self.com_import_as_name(node[i])) + return names + + def com_bases(self, node): + bases = [] + for i in range(1, len(node), 2): + bases.append(self.com_node(node[i])) + return bases + + def com_try_except_finally(self, nodelist): + # ('try' ':' suite + # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite] + # | 'finally' ':' suite)) + + if nodelist[3][0] == token.NAME: + # first clause is a finally clause: only try-finally + return TryFinally(self.com_node(nodelist[2]), + self.com_node(nodelist[5]), + lineno=nodelist[0][2]) + + #tryexcept: [TryNode, [except_clauses], elseNode)] + clauses = [] + elseNode = None + finallyNode = None + for i in range(3, len(nodelist), 3): + node = nodelist[i] + if node[0] == symbol.except_clause: + # except_clause: 'except' [expr [(',' | 'as') expr]] */ + if len(node) > 2: + expr1 = self.com_node(node[2]) + if len(node) > 4: + expr2 = self.com_assign(node[4], OP_ASSIGN) + else: + expr2 = None + else: + expr1 = expr2 = None + clauses.append((expr1, expr2, self.com_node(nodelist[i+2]))) + + if node[0] == token.NAME: + if node[1] == 'else': + elseNode = self.com_node(nodelist[i+2]) + elif node[1] == 'finally': + finallyNode = self.com_node(nodelist[i+2]) + try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode, + lineno=nodelist[0][2]) + if finallyNode: + return TryFinally(try_except, finallyNode, lineno=nodelist[0][2]) + else: + return try_except + + def com_with(self, nodelist): + # with_stmt: 'with' expr [with_var] ':' suite + expr = self.com_node(nodelist[1]) + body = self.com_node(nodelist[-1]) + if nodelist[2][0] == token.COLON: + var = None + else: + var = self.com_assign(nodelist[2][2], OP_ASSIGN) + return With(expr, var, body, lineno=nodelist[0][2]) + + def com_with_var(self, nodelist): + # with_var: 'as' expr + return self.com_node(nodelist[1]) + + def com_augassign_op(self, node): + assert node[0] == symbol.augassign + return node[1] + + def com_augassign(self, node): + """Return node suitable for lvalue of augmented assignment + + Names, slices, and attributes are the only allowable nodes. + """ + l = self.com_node(node) + if l.__class__ in (Name, Slice, Subscript, Getattr): + return l + raise SyntaxError, "can't assign to %s" % l.__class__.__name__ + + def com_assign(self, node, assigning): + # return a node suitable for use as an "lvalue" + # loop to avoid trivial recursion + while 1: + t = node[0] + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp): + if len(node) > 2: + return self.com_assign_tuple(node, assigning) + node = node[1] + elif t in _assign_types: + if len(node) > 2: + raise SyntaxError, "can't assign to operator" + node = node[1] + elif t == symbol.power: + if node[1][0] != symbol.atom: + raise SyntaxError, "can't assign to operator" + if len(node) > 2: + primary = self.com_node(node[1]) + for i in range(2, len(node)-1): + ch = node[i] + if ch[0] == token.DOUBLESTAR: + raise SyntaxError, "can't assign to operator" + primary = self.com_apply_trailer(primary, ch) + return self.com_assign_trailer(primary, node[-1], + assigning) + node = node[1] + elif t == symbol.atom: + t = node[1][0] + if t == token.LPAR: + node = node[2] + if node[0] == token.RPAR: + raise SyntaxError, "can't assign to ()" + elif t == token.LSQB: + node = node[2] + if node[0] == token.RSQB: + raise SyntaxError, "can't assign to []" + return self.com_assign_list(node, assigning) + elif t == token.NAME: + return self.com_assign_name(node[1], assigning) + else: + raise SyntaxError, "can't assign to literal" + else: + raise SyntaxError, "bad assignment (%s)" % t + + def com_assign_tuple(self, node, assigning): + assigns = [] + for i in range(1, len(node), 2): + assigns.append(self.com_assign(node[i], assigning)) + return AssTuple(assigns, lineno=extractLineNo(node)) + + def com_assign_list(self, node, assigning): + assigns = [] + for i in range(1, len(node), 2): + if i + 1 < len(node): + if node[i + 1][0] == symbol.list_for: + raise SyntaxError, "can't assign to list comprehension" + assert node[i + 1][0] == token.COMMA, node[i + 1] + assigns.append(self.com_assign(node[i], assigning)) + return AssList(assigns, lineno=extractLineNo(node)) + + def com_assign_name(self, node, assigning): + return AssName(node[1], assigning, lineno=node[2]) + + def com_assign_trailer(self, primary, node, assigning): + t = node[1][0] + if t == token.DOT: + return self.com_assign_attr(primary, node[2], assigning) + if t == token.LSQB: + return self.com_subscriptlist(primary, node[2], assigning) + if t == token.LPAR: + raise SyntaxError, "can't assign to function call" + raise SyntaxError, "unknown trailer type: %s" % t + + def com_assign_attr(self, primary, node, assigning): + return AssAttr(primary, node[1], assigning, lineno=node[-1]) + + def com_binary(self, constructor, nodelist): + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])." + l = len(nodelist) + if l == 1: + n = nodelist[0] + return self.lookup_node(n)(n[1:]) + items = [] + for i in range(0, l, 2): + n = nodelist[i] + items.append(self.lookup_node(n)(n[1:])) + return constructor(items, lineno=extractLineNo(nodelist)) + + def com_stmt(self, node): + result = self.lookup_node(node)(node[1:]) + assert result is not None + if isinstance(result, Stmt): + return result + return Stmt([result]) + + def com_append_stmt(self, stmts, node): + result = self.lookup_node(node)(node[1:]) + assert result is not None + if isinstance(result, Stmt): + stmts.extend(result.nodes) + else: + stmts.append(result) + + if hasattr(symbol, 'list_for'): + def com_list_constructor(self, nodelist): + # listmaker: test ( list_for | (',' test)* [','] ) + values = [] + for i in range(1, len(nodelist)): + if nodelist[i][0] == symbol.list_for: + assert len(nodelist[i:]) == 1 + return self.com_list_comprehension(values[0], + nodelist[i]) + elif nodelist[i][0] == token.COMMA: + continue + values.append(self.com_node(nodelist[i])) + return List(values, lineno=values[0].lineno) + + def com_list_comprehension(self, expr, node): + # list_iter: list_for | list_if + # list_for: 'for' exprlist 'in' testlist [list_iter] + # list_if: 'if' test [list_iter] + + # XXX should raise SyntaxError for assignment + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + listNode = self.com_node(node[4]) + newfor = ListCompFor(assignNode, listNode, []) + newfor.lineno = node[1][2] + fors.append(newfor) + if len(node) == 5: + node = None + else: + node = self.com_list_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = ListCompIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + else: + node = self.com_list_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected list comprehension element: %s %d" + % (node, lineno)) + return ListComp(expr, fors, lineno=lineno) + + def com_list_iter(self, node): + assert node[0] == symbol.list_iter + return node[1] + else: + def com_list_constructor(self, nodelist): + values = [] + for i in range(1, len(nodelist), 2): + values.append(self.com_node(nodelist[i])) + return List(values, lineno=values[0].lineno) + + if hasattr(symbol, 'gen_for'): + def com_generator_expression(self, expr, node): + # gen_iter: gen_for | gen_if + # gen_for: 'for' exprlist 'in' test [gen_iter] + # gen_if: 'if' test [gen_iter] + + lineno = node[1][2] + fors = [] + while node: + t = node[1][1] + if t == 'for': + assignNode = self.com_assign(node[2], OP_ASSIGN) + genNode = self.com_node(node[4]) + newfor = GenExprFor(assignNode, genNode, [], + lineno=node[1][2]) + fors.append(newfor) + if (len(node)) == 5: + node = None + else: + node = self.com_gen_iter(node[5]) + elif t == 'if': + test = self.com_node(node[2]) + newif = GenExprIf(test, lineno=node[1][2]) + newfor.ifs.append(newif) + if len(node) == 3: + node = None + else: + node = self.com_gen_iter(node[3]) + else: + raise SyntaxError, \ + ("unexpected generator expression element: %s %d" + % (node, lineno)) + fors[0].is_outmost = True + return GenExpr(GenExprInner(expr, fors), lineno=lineno) + + def com_gen_iter(self, node): + assert node[0] == symbol.gen_iter + return node[1] + + def com_dictmaker(self, nodelist): + # dictmaker: test ':' test (',' test ':' value)* [','] + items = [] + for i in range(1, len(nodelist), 4): + items.append((self.com_node(nodelist[i]), + self.com_node(nodelist[i+2]))) + return Dict(items, lineno=items[0][0].lineno) + + def com_apply_trailer(self, primaryNode, nodelist): + t = nodelist[1][0] + if t == token.LPAR: + return self.com_call_function(primaryNode, nodelist[2]) + if t == token.DOT: + return self.com_select_member(primaryNode, nodelist[2]) + if t == token.LSQB: + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY) + + raise SyntaxError, 'unknown node type: %s' % t + + def com_select_member(self, primaryNode, nodelist): + if nodelist[0] != token.NAME: + raise SyntaxError, "member must be a name" + return Getattr(primaryNode, nodelist[1], lineno=nodelist[2]) + + def com_call_function(self, primaryNode, nodelist): + if nodelist[0] == token.RPAR: + return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist)) + args = [] + kw = 0 + star_node = dstar_node = None + len_nodelist = len(nodelist) + i = 1 + while i < len_nodelist: + node = nodelist[i] + + if node[0]==token.STAR: + if star_node is not None: + raise SyntaxError, 'already have the varargs indentifier' + star_node = self.com_node(nodelist[i+1]) + i = i + 3 + continue + elif node[0]==token.DOUBLESTAR: + if dstar_node is not None: + raise SyntaxError, 'already have the kwargs indentifier' + dstar_node = self.com_node(nodelist[i+1]) + i = i + 3 + continue + + # positional or named parameters + kw, result = self.com_argument(node, kw, star_node) + + if len_nodelist != 2 and isinstance(result, GenExpr) \ + and len(node) == 3 and node[2][0] == symbol.gen_for: + # allow f(x for x in y), but reject f(x for x in y, 1) + # should use f((x for x in y), 1) instead of f(x for x in y, 1) + raise SyntaxError, 'generator expression needs parenthesis' + + args.append(result) + i = i + 2 + + return CallFunc(primaryNode, args, star_node, dstar_node, + lineno=extractLineNo(nodelist)) + + def com_argument(self, nodelist, kw, star_node): + if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for: + test = self.com_node(nodelist[1]) + return 0, self.com_generator_expression(test, nodelist[2]) + if len(nodelist) == 2: + if kw: + raise SyntaxError, "non-keyword arg after keyword arg" + if star_node: + raise SyntaxError, "only named arguments may follow *expression" + return 0, self.com_node(nodelist[1]) + result = self.com_node(nodelist[3]) + n = nodelist[1] + while len(n) == 2 and n[0] != token.NAME: + n = n[1] + if n[0] != token.NAME: + raise SyntaxError, "keyword can't be an expression (%s)"%n[0] + node = Keyword(n[1], result, lineno=n[2]) + return 1, node + + def com_subscriptlist(self, primary, nodelist, assigning): + # slicing: simple_slicing | extended_slicing + # simple_slicing: primary "[" short_slice "]" + # extended_slicing: primary "[" slice_list "]" + # slice_list: slice_item ("," slice_item)* [","] + + # backwards compat slice for '[i:j]' + if len(nodelist) == 2: + sub = nodelist[1] + if (sub[1][0] == token.COLON or \ + (len(sub) > 2 and sub[2][0] == token.COLON)) and \ + sub[-1][0] != symbol.sliceop: + return self.com_slice(primary, sub, assigning) + + subscripts = [] + for i in range(1, len(nodelist), 2): + subscripts.append(self.com_subscript(nodelist[i])) + return Subscript(primary, assigning, subscripts, + lineno=extractLineNo(nodelist)) + + def com_subscript(self, node): + # slice_item: expression | proper_slice | ellipsis + ch = node[1] + t = ch[0] + if t == token.DOT and node[2][0] == token.DOT: + return Ellipsis() + if t == token.COLON or len(node) > 2: + return self.com_sliceobj(node) + return self.com_node(ch) + + def com_sliceobj(self, node): + # proper_slice: short_slice | long_slice + # short_slice: [lower_bound] ":" [upper_bound] + # long_slice: short_slice ":" [stride] + # lower_bound: expression + # upper_bound: expression + # stride: expression + # + # Note: a stride may be further slicing... + + items = [] + + if node[1][0] == token.COLON: + items.append(Const(None)) + i = 2 + else: + items.append(self.com_node(node[1])) + # i == 2 is a COLON + i = 3 + + if i < len(node) and node[i][0] == symbol.test: + items.append(self.com_node(node[i])) + i = i + 1 + else: + items.append(Const(None)) + + # a short_slice has been built. look for long_slice now by looking + # for strides... + for j in range(i, len(node)): + ch = node[j] + if len(ch) == 2: + items.append(Const(None)) + else: + items.append(self.com_node(ch[2])) + return Sliceobj(items, lineno=extractLineNo(node)) + + def com_slice(self, primary, node, assigning): + # short_slice: [lower_bound] ":" [upper_bound] + lower = upper = None + if len(node) == 3: + if node[1][0] == token.COLON: + upper = self.com_node(node[2]) + else: + lower = self.com_node(node[1]) + elif len(node) == 4: + lower = self.com_node(node[1]) + upper = self.com_node(node[3]) + return Slice(primary, assigning, lower, upper, + lineno=extractLineNo(node)) + + def get_docstring(self, node, n=None): + if n is None: + n = node[0] + node = node[1:] + if n == symbol.suite: + if len(node) == 1: + return self.get_docstring(node[0]) + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.file_input: + for sub in node: + if sub[0] == symbol.stmt: + return self.get_docstring(sub) + return None + if n == symbol.atom: + if node[0][0] == token.STRING: + s = '' + for t in node: + s = s + eval(t[1]) + return s + return None + if n == symbol.stmt or n == symbol.simple_stmt \ + or n == symbol.small_stmt: + return self.get_docstring(node[0]) + if n in _doc_nodes and len(node) == 1: + return self.get_docstring(node[0]) + return None + + +_doc_nodes = [ + symbol.expr_stmt, + symbol.testlist, + symbol.testlist_safe, + symbol.test, + symbol.or_test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + ] + +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '==' +# | 'in' | 'not' 'in' | 'is' | 'is' 'not' +_cmp_types = { + token.LESS : '<', + token.GREATER : '>', + token.EQEQUAL : '==', + token.EQUAL : '==', + token.LESSEQUAL : '<=', + token.GREATEREQUAL : '>=', + token.NOTEQUAL : '!=', + } + +_legal_node_types = [ + symbol.funcdef, + symbol.classdef, + symbol.stmt, + symbol.small_stmt, + symbol.flow_stmt, + symbol.simple_stmt, + symbol.compound_stmt, + symbol.expr_stmt, + symbol.print_stmt, + symbol.del_stmt, + symbol.pass_stmt, + symbol.break_stmt, + symbol.continue_stmt, + symbol.return_stmt, + symbol.raise_stmt, + symbol.import_stmt, + symbol.global_stmt, + symbol.exec_stmt, + symbol.assert_stmt, + symbol.if_stmt, + symbol.while_stmt, + symbol.for_stmt, + symbol.try_stmt, + symbol.with_stmt, + symbol.suite, + symbol.testlist, + symbol.testlist_safe, + symbol.test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.exprlist, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + symbol.power, + symbol.atom, + ] + +if hasattr(symbol, 'yield_stmt'): + _legal_node_types.append(symbol.yield_stmt) +if hasattr(symbol, 'yield_expr'): + _legal_node_types.append(symbol.yield_expr) + +_assign_types = [ + symbol.test, + symbol.or_test, + symbol.and_test, + symbol.not_test, + symbol.comparison, + symbol.expr, + symbol.xor_expr, + symbol.and_expr, + symbol.shift_expr, + symbol.arith_expr, + symbol.term, + symbol.factor, + ] + +_names = {} +for k, v in symbol.sym_name.items(): + _names[k] = v +for k, v in token.tok_name.items(): + _names[k] = v + +def debug_tree(tree): + l = [] + for elt in tree: + if isinstance(elt, int): + l.append(_names.get(elt, elt)) + elif isinstance(elt, str): + l.append(elt) + else: + l.append(debug_tree(elt)) + return l diff -r 000000000000 -r 7bec0014e871 compiler/visitor.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/compiler/visitor.py Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,113 @@ +from compiler import ast + +# XXX should probably rename ASTVisitor to ASTWalker +# XXX can it be made even more generic? + +class ASTVisitor: + """Performs a depth-first walk of the AST + + The ASTVisitor will walk the AST, performing either a preorder or + postorder traversal depending on which method is called. + + methods: + preorder(tree, visitor) + postorder(tree, visitor) + tree: an instance of ast.Node + visitor: an instance with visitXXX methods + + The ASTVisitor is responsible for walking over the tree in the + correct order. For each node, it checks the visitor argument for + a method named 'visitNodeType' where NodeType is the name of the + node's class, e.g. Class. If the method exists, it is called + with the node as its sole argument. + + The visitor method for a particular node type can control how + child nodes are visited during a preorder walk. (It can't control + the order during a postorder walk, because it is called _after_ + the walk has occurred.) The ASTVisitor modifies the visitor + argument by adding a visit method to the visitor; this method can + be used to visit a child node of arbitrary type. + """ + + VERBOSE = 0 + + def __init__(self): + self.node = None + self._cache = {} + + def default(self, node, *args): + for child in node.getChildNodes(): + self.dispatch(child, *args) + + def dispatch(self, node, *args): + self.node = node + klass = node.__class__ + meth = self._cache.get(klass, None) + if meth is None: + className = klass.__name__ + meth = getattr(self.visitor, 'visit' + className, self.default) + self._cache[klass] = meth +## if self.VERBOSE > 0: +## className = klass.__name__ +## if self.VERBOSE == 1: +## if meth == 0: +## print "dispatch", className +## else: +## print "dispatch", className, (meth and meth.__name__ or '') + return meth(node, *args) + + def preorder(self, tree, visitor, *args): + """Do preorder walk of tree using visitor""" + self.visitor = visitor + visitor.visit = self.dispatch + self.dispatch(tree, *args) # XXX *args make sense? + +class ExampleASTVisitor(ASTVisitor): + """Prints examples of the nodes that aren't visited + + This visitor-driver is only useful for development, when it's + helpful to develop a visitor incrementally, and get feedback on what + you still have to do. + """ + examples = {} + + def dispatch(self, node, *args): + self.node = node + meth = self._cache.get(node.__class__, None) + className = node.__class__.__name__ + if meth is None: + meth = getattr(self.visitor, 'visit' + className, 0) + self._cache[node.__class__] = meth + if self.VERBOSE > 1: + print "dispatch", className, (meth and meth.__name__ or '') + if meth: + meth(node, *args) + elif self.VERBOSE > 0: + klass = node.__class__ + if klass not in self.examples: + self.examples[klass] = klass + print + print self.visitor + print klass + for attr in dir(node): + if attr[0] != '_': + print "\t", "%-12.12s" % attr, getattr(node, attr) + print + return self.default(node, *args) + +# XXX this is an API change + +_walker = ASTVisitor +def walk(tree, visitor, walker=None, verbose=None): + if walker is None: + walker = _walker() + if verbose is not None: + walker.VERBOSE = verbose + walker.preorder(tree, visitor) + return walker.visitor + +def dumpNode(node): + print node.__class__ + for attr in dir(node): + if attr[0] != '_': + print "\t", "%-10.10s" % attr, getattr(node, attr) diff -r 000000000000 -r 7bec0014e871 docs/COPYRIGHT --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/docs/COPYRIGHT Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,24 @@ +Copyright information from Python 2.6.8 +--------------------------------------- + +Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012 +Python Software Foundation. +All rights reserved. + +Copyright (c) 2000 BeOpen.com. +All rights reserved. + +Copyright (c) 1995-2001 Corporation for National Research Initiatives. +All rights reserved. + +Copyright (c) 1991-1995 Stichting Mathematisch Centrum. +All rights reserved. + + +License information +------------------- + +See the file "LICENSE" for information on the history of this +software, terms & conditions for usage, and a DISCLAIMER OF ALL +WARRANTIES. diff -r 000000000000 -r 7bec0014e871 docs/LICENSE --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/docs/LICENSE Fri May 18 20:51:41 2012 +0200 @@ -0,0 +1,281 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.2 2.1.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2.1 2.2 2002 PSF yes + 2.2.2 2.2.1 2002 PSF yes + 2.2.3 2.2.2 2003 PSF yes + 2.3 2.2.2 2002-2003 PSF yes + 2.3.1 2.3 2002-2003 PSF yes + 2.3.2 2.3.1 2002-2003 PSF yes + 2.3.3 2.3.2 2002-2003 PSF yes + 2.3.4 2.3.3 2004 PSF yes + 2.3.5 2.3.4 2005 PSF yes + 2.4 2.3 2004 PSF yes + 2.4.1 2.4 2005 PSF yes + 2.4.2 2.4.1 2005 PSF yes + 2.4.3 2.4.2 2006 PSF yes + 2.4.4 2.4.3 2006 PSF yes + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 2.6.6 2.6.5 2010 PSF yes + 2.6.7 2.6.6 2011 PSF yes + 2.6.8 2.6.7 2012 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 +Python Software Foundation; All Rights Reserved" are retained in Python alone or +in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.