1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/compiler/transformer.py Fri May 18 23:26:30 2012 +0200
1.3 @@ -0,0 +1,1535 @@
1.4 +"""Parse tree transformation module.
1.5 +
1.6 +Transforms Python source code into an abstract syntax tree (AST)
1.7 +defined in the ast module.
1.8 +
1.9 +The simplest ways to invoke this module are via parse and parseFile.
1.10 +parse(buf) -> AST
1.11 +parseFile(path) -> AST
1.12 +"""
1.13 +
1.14 +# Original version written by Greg Stein (gstein@lyra.org)
1.15 +# and Bill Tutt (rassilon@lima.mudlib.org)
1.16 +# February 1997.
1.17 +#
1.18 +# Modifications and improvements for Python 2.0 by Jeremy Hylton and
1.19 +# Mark Hammond
1.20 +#
1.21 +# Some fixes to try to have correct line number on almost all nodes
1.22 +# (except Module, Discard and Stmt) added by Sylvain Thenault
1.23 +#
1.24 +# Portions of this file are:
1.25 +# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
1.26 +#
1.27 +# This module is provided under a BSD-ish license. See
1.28 +# http://www.opensource.org/licenses/bsd-license.html
1.29 +# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
1.30 +
1.31 +from compiler.ast import *
1.32 +import parser
1.33 +import symbol
1.34 +import token
1.35 +
1.36 +class WalkerError(StandardError):
1.37 + pass
1.38 +
1.39 +from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
1.40 +from compiler.consts import OP_ASSIGN, OP_DELETE, OP_APPLY
1.41 +
1.42 +def parseFile(path):
1.43 + f = open(path, "U")
1.44 + # XXX The parser API tolerates files without a trailing newline,
1.45 + # but not strings without a trailing newline. Always add an extra
1.46 + # newline to the file contents, since we're going through the string
1.47 + # version of the API.
1.48 + src = f.read() + "\n"
1.49 + f.close()
1.50 + return parse(src)
1.51 +
1.52 +def parse(buf, mode="exec"):
1.53 + if mode == "exec" or mode == "single":
1.54 + return Transformer().parsesuite(buf)
1.55 + elif mode == "eval":
1.56 + return Transformer().parseexpr(buf)
1.57 + else:
1.58 + raise ValueError("compile() arg 3 must be"
1.59 + " 'exec' or 'eval' or 'single'")
1.60 +
1.61 +def asList(nodes):
1.62 + l = []
1.63 + for item in nodes:
1.64 + if hasattr(item, "asList"):
1.65 + l.append(item.asList())
1.66 + else:
1.67 + if type(item) is type( (None, None) ):
1.68 + l.append(tuple(asList(item)))
1.69 + elif type(item) is type( [] ):
1.70 + l.append(asList(item))
1.71 + else:
1.72 + l.append(item)
1.73 + return l
1.74 +
1.75 +def extractLineNo(ast):
1.76 + if not isinstance(ast[1], tuple):
1.77 + # get a terminal node
1.78 + return ast[2]
1.79 + for child in ast[1:]:
1.80 + if isinstance(child, tuple):
1.81 + lineno = extractLineNo(child)
1.82 + if lineno is not None:
1.83 + return lineno
1.84 +
1.85 +def Node(*args):
1.86 + kind = args[0]
1.87 + if kind in nodes:
1.88 + try:
1.89 + return nodes[kind](*args[1:])
1.90 + except TypeError:
1.91 + print nodes[kind], len(args), args
1.92 + raise
1.93 + else:
1.94 + raise WalkerError, "Can't find appropriate Node type: %s" % str(args)
1.95 + #return apply(ast.Node, args)
1.96 +
1.97 +class Transformer:
1.98 + """Utility object for transforming Python parse trees.
1.99 +
1.100 + Exposes the following methods:
1.101 + tree = transform(ast_tree)
1.102 + tree = parsesuite(text)
1.103 + tree = parseexpr(text)
1.104 + tree = parsefile(fileob | filename)
1.105 + """
1.106 +
1.107 + def __init__(self):
1.108 + self._dispatch = {}
1.109 + for value, name in symbol.sym_name.items():
1.110 + if hasattr(self, name):
1.111 + self._dispatch[value] = getattr(self, name)
1.112 + self._dispatch[token.NEWLINE] = self.com_NEWLINE
1.113 + self._atom_dispatch = {token.LPAR: self.atom_lpar,
1.114 + token.LSQB: self.atom_lsqb,
1.115 + token.LBRACE: self.atom_lbrace,
1.116 + token.BACKQUOTE: self.atom_backquote,
1.117 + token.NUMBER: self.atom_number,
1.118 + token.STRING: self.atom_string,
1.119 + token.NAME: self.atom_name,
1.120 + }
1.121 + self.encoding = None
1.122 +
1.123 + def transform(self, tree):
1.124 + """Transform an AST into a modified parse tree."""
1.125 + if not (isinstance(tree, tuple) or isinstance(tree, list)):
1.126 + tree = parser.st2tuple(tree, line_info=1)
1.127 + return self.compile_node(tree)
1.128 +
1.129 + def parsesuite(self, text):
1.130 + """Return a modified parse tree for the given suite text."""
1.131 + return self.transform(parser.suite(text))
1.132 +
1.133 + def parseexpr(self, text):
1.134 + """Return a modified parse tree for the given expression text."""
1.135 + return self.transform(parser.expr(text))
1.136 +
1.137 + def parsefile(self, file):
1.138 + """Return a modified parse tree for the contents of the given file."""
1.139 + if type(file) == type(''):
1.140 + file = open(file)
1.141 + return self.parsesuite(file.read())
1.142 +
1.143 + # --------------------------------------------------------------
1.144 + #
1.145 + # PRIVATE METHODS
1.146 + #
1.147 +
1.148 + def compile_node(self, node):
1.149 + ### emit a line-number node?
1.150 + n = node[0]
1.151 +
1.152 + if n == symbol.encoding_decl:
1.153 + self.encoding = node[2]
1.154 + node = node[1]
1.155 + n = node[0]
1.156 +
1.157 + if n == symbol.single_input:
1.158 + return self.single_input(node[1:])
1.159 + if n == symbol.file_input:
1.160 + return self.file_input(node[1:])
1.161 + if n == symbol.eval_input:
1.162 + return self.eval_input(node[1:])
1.163 + if n == symbol.lambdef:
1.164 + return self.lambdef(node[1:])
1.165 + if n == symbol.funcdef:
1.166 + return self.funcdef(node[1:])
1.167 + if n == symbol.classdef:
1.168 + return self.classdef(node[1:])
1.169 +
1.170 + raise WalkerError, ('unexpected node type', n)
1.171 +
1.172 + def single_input(self, node):
1.173 + ### do we want to do anything about being "interactive" ?
1.174 +
1.175 + # NEWLINE | simple_stmt | compound_stmt NEWLINE
1.176 + n = node[0][0]
1.177 + if n != token.NEWLINE:
1.178 + return self.com_stmt(node[0])
1.179 +
1.180 + return Pass()
1.181 +
1.182 + def file_input(self, nodelist):
1.183 + doc = self.get_docstring(nodelist, symbol.file_input)
1.184 + if doc is not None:
1.185 + i = 1
1.186 + else:
1.187 + i = 0
1.188 + stmts = []
1.189 + for node in nodelist[i:]:
1.190 + if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
1.191 + self.com_append_stmt(stmts, node)
1.192 + return Module(doc, Stmt(stmts))
1.193 +
1.194 + def eval_input(self, nodelist):
1.195 + # from the built-in function input()
1.196 + ### is this sufficient?
1.197 + return Expression(self.com_node(nodelist[0]))
1.198 +
1.199 + def decorator_name(self, nodelist):
1.200 + listlen = len(nodelist)
1.201 + assert listlen >= 1 and listlen % 2 == 1
1.202 +
1.203 + item = self.atom_name(nodelist)
1.204 + i = 1
1.205 + while i < listlen:
1.206 + assert nodelist[i][0] == token.DOT
1.207 + assert nodelist[i + 1][0] == token.NAME
1.208 + item = Getattr(item, nodelist[i + 1][1])
1.209 + i += 2
1.210 +
1.211 + return item
1.212 +
1.213 + def decorator(self, nodelist):
1.214 + # '@' dotted_name [ '(' [arglist] ')' ]
1.215 + assert len(nodelist) in (3, 5, 6)
1.216 + assert nodelist[0][0] == token.AT
1.217 + assert nodelist[-1][0] == token.NEWLINE
1.218 +
1.219 + assert nodelist[1][0] == symbol.dotted_name
1.220 + funcname = self.decorator_name(nodelist[1][1:])
1.221 +
1.222 + if len(nodelist) > 3:
1.223 + assert nodelist[2][0] == token.LPAR
1.224 + expr = self.com_call_function(funcname, nodelist[3])
1.225 + else:
1.226 + expr = funcname
1.227 +
1.228 + return expr
1.229 +
1.230 + def decorators(self, nodelist):
1.231 + # decorators: decorator ([NEWLINE] decorator)* NEWLINE
1.232 + items = []
1.233 + for dec_nodelist in nodelist:
1.234 + assert dec_nodelist[0] == symbol.decorator
1.235 + items.append(self.decorator(dec_nodelist[1:]))
1.236 + return Decorators(items)
1.237 +
1.238 + def decorated(self, nodelist):
1.239 + assert nodelist[0][0] == symbol.decorators
1.240 + if nodelist[1][0] == symbol.funcdef:
1.241 + n = [nodelist[0]] + list(nodelist[1][1:])
1.242 + return self.funcdef(n)
1.243 + elif nodelist[1][0] == symbol.classdef:
1.244 + decorators = self.decorators(nodelist[0][1:])
1.245 + cls = self.classdef(nodelist[1][1:])
1.246 + cls.decorators = decorators
1.247 + return cls
1.248 + raise WalkerError()
1.249 +
1.250 + def funcdef(self, nodelist):
1.251 + # -6 -5 -4 -3 -2 -1
1.252 + # funcdef: [decorators] 'def' NAME parameters ':' suite
1.253 + # parameters: '(' [varargslist] ')'
1.254 +
1.255 + if len(nodelist) == 6:
1.256 + assert nodelist[0][0] == symbol.decorators
1.257 + decorators = self.decorators(nodelist[0][1:])
1.258 + else:
1.259 + assert len(nodelist) == 5
1.260 + decorators = None
1.261 +
1.262 + lineno = nodelist[-4][2]
1.263 + name = nodelist[-4][1]
1.264 + args = nodelist[-3][2]
1.265 +
1.266 + if args[0] == symbol.varargslist:
1.267 + names, defaults, flags = self.com_arglist(args[1:])
1.268 + else:
1.269 + names = defaults = ()
1.270 + flags = 0
1.271 + doc = self.get_docstring(nodelist[-1])
1.272 +
1.273 + # code for function
1.274 + code = self.com_node(nodelist[-1])
1.275 +
1.276 + if doc is not None:
1.277 + assert isinstance(code, Stmt)
1.278 + assert isinstance(code.nodes[0], Discard)
1.279 + del code.nodes[0]
1.280 + return Function(decorators, name, names, defaults, flags, doc, code,
1.281 + lineno=lineno)
1.282 +
1.283 + def lambdef(self, nodelist):
1.284 + # lambdef: 'lambda' [varargslist] ':' test
1.285 + if nodelist[2][0] == symbol.varargslist:
1.286 + names, defaults, flags = self.com_arglist(nodelist[2][1:])
1.287 + else:
1.288 + names = defaults = ()
1.289 + flags = 0
1.290 +
1.291 + # code for lambda
1.292 + code = self.com_node(nodelist[-1])
1.293 +
1.294 + return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
1.295 + old_lambdef = lambdef
1.296 +
1.297 + def classdef(self, nodelist):
1.298 + # classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
1.299 +
1.300 + name = nodelist[1][1]
1.301 + doc = self.get_docstring(nodelist[-1])
1.302 + if nodelist[2][0] == token.COLON:
1.303 + bases = []
1.304 + elif nodelist[3][0] == token.RPAR:
1.305 + bases = []
1.306 + else:
1.307 + bases = self.com_bases(nodelist[3])
1.308 +
1.309 + # code for class
1.310 + code = self.com_node(nodelist[-1])
1.311 +
1.312 + if doc is not None:
1.313 + assert isinstance(code, Stmt)
1.314 + assert isinstance(code.nodes[0], Discard)
1.315 + del code.nodes[0]
1.316 +
1.317 + return Class(name, bases, doc, code, lineno=nodelist[1][2])
1.318 +
1.319 + def stmt(self, nodelist):
1.320 + return self.com_stmt(nodelist[0])
1.321 +
1.322 + small_stmt = stmt
1.323 + flow_stmt = stmt
1.324 + compound_stmt = stmt
1.325 +
1.326 + def simple_stmt(self, nodelist):
1.327 + # small_stmt (';' small_stmt)* [';'] NEWLINE
1.328 + stmts = []
1.329 + for i in range(0, len(nodelist), 2):
1.330 + self.com_append_stmt(stmts, nodelist[i])
1.331 + return Stmt(stmts)
1.332 +
1.333 + def parameters(self, nodelist):
1.334 + raise WalkerError
1.335 +
1.336 + def varargslist(self, nodelist):
1.337 + raise WalkerError
1.338 +
1.339 + def fpdef(self, nodelist):
1.340 + raise WalkerError
1.341 +
1.342 + def fplist(self, nodelist):
1.343 + raise WalkerError
1.344 +
1.345 + def dotted_name(self, nodelist):
1.346 + raise WalkerError
1.347 +
1.348 + def comp_op(self, nodelist):
1.349 + raise WalkerError
1.350 +
1.351 + def trailer(self, nodelist):
1.352 + raise WalkerError
1.353 +
1.354 + def sliceop(self, nodelist):
1.355 + raise WalkerError
1.356 +
1.357 + def argument(self, nodelist):
1.358 + raise WalkerError
1.359 +
1.360 + # --------------------------------------------------------------
1.361 + #
1.362 + # STATEMENT NODES (invoked by com_node())
1.363 + #
1.364 +
1.365 + def expr_stmt(self, nodelist):
1.366 + # augassign testlist | testlist ('=' testlist)*
1.367 + en = nodelist[-1]
1.368 + exprNode = self.lookup_node(en)(en[1:])
1.369 + if len(nodelist) == 1:
1.370 + return Discard(exprNode, lineno=exprNode.lineno)
1.371 + if nodelist[1][0] == token.EQUAL:
1.372 + nodesl = []
1.373 + for i in range(0, len(nodelist) - 2, 2):
1.374 + nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
1.375 + return Assign(nodesl, exprNode, lineno=nodelist[1][2])
1.376 + else:
1.377 + lval = self.com_augassign(nodelist[0])
1.378 + op = self.com_augassign_op(nodelist[1])
1.379 + return AugAssign(lval, op[1], exprNode, lineno=op[2])
1.380 + raise WalkerError, "can't get here"
1.381 +
1.382 + def print_stmt(self, nodelist):
1.383 + # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
1.384 + items = []
1.385 + if len(nodelist) == 1:
1.386 + start = 1
1.387 + dest = None
1.388 + elif nodelist[1][0] == token.RIGHTSHIFT:
1.389 + assert len(nodelist) == 3 \
1.390 + or nodelist[3][0] == token.COMMA
1.391 + dest = self.com_node(nodelist[2])
1.392 + start = 4
1.393 + else:
1.394 + dest = None
1.395 + start = 1
1.396 + for i in range(start, len(nodelist), 2):
1.397 + items.append(self.com_node(nodelist[i]))
1.398 + if nodelist[-1][0] == token.COMMA:
1.399 + return Print(items, dest, lineno=nodelist[0][2])
1.400 + return Printnl(items, dest, lineno=nodelist[0][2])
1.401 +
1.402 + def del_stmt(self, nodelist):
1.403 + return self.com_assign(nodelist[1], OP_DELETE)
1.404 +
1.405 + def pass_stmt(self, nodelist):
1.406 + return Pass(lineno=nodelist[0][2])
1.407 +
1.408 + def break_stmt(self, nodelist):
1.409 + return Break(lineno=nodelist[0][2])
1.410 +
1.411 + def continue_stmt(self, nodelist):
1.412 + return Continue(lineno=nodelist[0][2])
1.413 +
1.414 + def return_stmt(self, nodelist):
1.415 + # return: [testlist]
1.416 + if len(nodelist) < 2:
1.417 + return Return(Const(None), lineno=nodelist[0][2])
1.418 + return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
1.419 +
1.420 + def yield_stmt(self, nodelist):
1.421 + expr = self.com_node(nodelist[0])
1.422 + return Discard(expr, lineno=expr.lineno)
1.423 +
1.424 + def yield_expr(self, nodelist):
1.425 + if len(nodelist) > 1:
1.426 + value = self.com_node(nodelist[1])
1.427 + else:
1.428 + value = Const(None)
1.429 + return Yield(value, lineno=nodelist[0][2])
1.430 +
1.431 + def raise_stmt(self, nodelist):
1.432 + # raise: [test [',' test [',' test]]]
1.433 + if len(nodelist) > 5:
1.434 + expr3 = self.com_node(nodelist[5])
1.435 + else:
1.436 + expr3 = None
1.437 + if len(nodelist) > 3:
1.438 + expr2 = self.com_node(nodelist[3])
1.439 + else:
1.440 + expr2 = None
1.441 + if len(nodelist) > 1:
1.442 + expr1 = self.com_node(nodelist[1])
1.443 + else:
1.444 + expr1 = None
1.445 + return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
1.446 +
1.447 + def import_stmt(self, nodelist):
1.448 + # import_stmt: import_name | import_from
1.449 + assert len(nodelist) == 1
1.450 + return self.com_node(nodelist[0])
1.451 +
1.452 + def import_name(self, nodelist):
1.453 + # import_name: 'import' dotted_as_names
1.454 + return Import(self.com_dotted_as_names(nodelist[1]),
1.455 + lineno=nodelist[0][2])
1.456 +
1.457 + def import_from(self, nodelist):
1.458 + # import_from: 'from' ('.'* dotted_name | '.') 'import' ('*' |
1.459 + # '(' import_as_names ')' | import_as_names)
1.460 + assert nodelist[0][1] == 'from'
1.461 + idx = 1
1.462 + while nodelist[idx][1] == '.':
1.463 + idx += 1
1.464 + level = idx - 1
1.465 + if nodelist[idx][0] == symbol.dotted_name:
1.466 + fromname = self.com_dotted_name(nodelist[idx])
1.467 + idx += 1
1.468 + else:
1.469 + fromname = ""
1.470 + assert nodelist[idx][1] == 'import'
1.471 + if nodelist[idx + 1][0] == token.STAR:
1.472 + return From(fromname, [('*', None)], level,
1.473 + lineno=nodelist[0][2])
1.474 + else:
1.475 + node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]
1.476 + return From(fromname, self.com_import_as_names(node), level,
1.477 + lineno=nodelist[0][2])
1.478 +
1.479 + def global_stmt(self, nodelist):
1.480 + # global: NAME (',' NAME)*
1.481 + names = []
1.482 + for i in range(1, len(nodelist), 2):
1.483 + names.append(nodelist[i][1])
1.484 + return Global(names, lineno=nodelist[0][2])
1.485 +
1.486 + def exec_stmt(self, nodelist):
1.487 + # exec_stmt: 'exec' expr ['in' expr [',' expr]]
1.488 + expr1 = self.com_node(nodelist[1])
1.489 + if len(nodelist) >= 4:
1.490 + expr2 = self.com_node(nodelist[3])
1.491 + if len(nodelist) >= 6:
1.492 + expr3 = self.com_node(nodelist[5])
1.493 + else:
1.494 + expr3 = None
1.495 + else:
1.496 + expr2 = expr3 = None
1.497 +
1.498 + return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
1.499 +
1.500 + def assert_stmt(self, nodelist):
1.501 + # 'assert': test, [',' test]
1.502 + expr1 = self.com_node(nodelist[1])
1.503 + if (len(nodelist) == 4):
1.504 + expr2 = self.com_node(nodelist[3])
1.505 + else:
1.506 + expr2 = None
1.507 + return Assert(expr1, expr2, lineno=nodelist[0][2])
1.508 +
1.509 + def if_stmt(self, nodelist):
1.510 + # if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
1.511 + tests = []
1.512 + for i in range(0, len(nodelist) - 3, 4):
1.513 + testNode = self.com_node(nodelist[i + 1])
1.514 + suiteNode = self.com_node(nodelist[i + 3])
1.515 + tests.append((testNode, suiteNode))
1.516 +
1.517 + if len(nodelist) % 4 == 3:
1.518 + elseNode = self.com_node(nodelist[-1])
1.519 +## elseNode.lineno = nodelist[-1][1][2]
1.520 + else:
1.521 + elseNode = None
1.522 + return If(tests, elseNode, lineno=nodelist[0][2])
1.523 +
1.524 + def while_stmt(self, nodelist):
1.525 + # 'while' test ':' suite ['else' ':' suite]
1.526 +
1.527 + testNode = self.com_node(nodelist[1])
1.528 + bodyNode = self.com_node(nodelist[3])
1.529 +
1.530 + if len(nodelist) > 4:
1.531 + elseNode = self.com_node(nodelist[6])
1.532 + else:
1.533 + elseNode = None
1.534 +
1.535 + return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
1.536 +
1.537 + def for_stmt(self, nodelist):
1.538 + # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
1.539 +
1.540 + assignNode = self.com_assign(nodelist[1], OP_ASSIGN)
1.541 + listNode = self.com_node(nodelist[3])
1.542 + bodyNode = self.com_node(nodelist[5])
1.543 +
1.544 + if len(nodelist) > 8:
1.545 + elseNode = self.com_node(nodelist[8])
1.546 + else:
1.547 + elseNode = None
1.548 +
1.549 + return For(assignNode, listNode, bodyNode, elseNode,
1.550 + lineno=nodelist[0][2])
1.551 +
1.552 + def try_stmt(self, nodelist):
1.553 + return self.com_try_except_finally(nodelist)
1.554 +
1.555 + def with_stmt(self, nodelist):
1.556 + return self.com_with(nodelist)
1.557 +
1.558 + def with_var(self, nodelist):
1.559 + return self.com_with_var(nodelist)
1.560 +
1.561 + def suite(self, nodelist):
1.562 + # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
1.563 + if len(nodelist) == 1:
1.564 + return self.com_stmt(nodelist[0])
1.565 +
1.566 + stmts = []
1.567 + for node in nodelist:
1.568 + if node[0] == symbol.stmt:
1.569 + self.com_append_stmt(stmts, node)
1.570 + return Stmt(stmts)
1.571 +
1.572 + # --------------------------------------------------------------
1.573 + #
1.574 + # EXPRESSION NODES (invoked by com_node())
1.575 + #
1.576 +
1.577 + def testlist(self, nodelist):
1.578 + # testlist: expr (',' expr)* [',']
1.579 + # testlist_safe: test [(',' test)+ [',']]
1.580 + # exprlist: expr (',' expr)* [',']
1.581 + return self.com_binary(Tuple, nodelist)
1.582 +
1.583 + testlist_safe = testlist # XXX
1.584 + testlist1 = testlist
1.585 + exprlist = testlist
1.586 +
1.587 + def testlist_comp(self, nodelist):
1.588 + # test ( comp_for | (',' test)* [','] )
1.589 + assert nodelist[0][0] == symbol.test
1.590 + if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for:
1.591 + test = self.com_node(nodelist[0])
1.592 + return self.com_generator_expression(test, nodelist[1])
1.593 + return self.testlist(nodelist)
1.594 +
1.595 + def test(self, nodelist):
1.596 + # or_test ['if' or_test 'else' test] | lambdef
1.597 + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.598 + return self.lambdef(nodelist[0])
1.599 + then = self.com_node(nodelist[0])
1.600 + if len(nodelist) > 1:
1.601 + assert len(nodelist) == 5
1.602 + assert nodelist[1][1] == 'if'
1.603 + assert nodelist[3][1] == 'else'
1.604 + test = self.com_node(nodelist[2])
1.605 + else_ = self.com_node(nodelist[4])
1.606 + return IfExp(test, then, else_, lineno=nodelist[1][2])
1.607 + return then
1.608 +
1.609 + def or_test(self, nodelist):
1.610 + # and_test ('or' and_test)* | lambdef
1.611 + if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.612 + return self.lambdef(nodelist[0])
1.613 + return self.com_binary(Or, nodelist)
1.614 + old_test = or_test
1.615 +
1.616 + def and_test(self, nodelist):
1.617 + # not_test ('and' not_test)*
1.618 + return self.com_binary(And, nodelist)
1.619 +
1.620 + def not_test(self, nodelist):
1.621 + # 'not' not_test | comparison
1.622 + result = self.com_node(nodelist[-1])
1.623 + if len(nodelist) == 2:
1.624 + return Not(result, lineno=nodelist[0][2])
1.625 + return result
1.626 +
1.627 + def comparison(self, nodelist):
1.628 + # comparison: expr (comp_op expr)*
1.629 + node = self.com_node(nodelist[0])
1.630 + if len(nodelist) == 1:
1.631 + return node
1.632 +
1.633 + results = []
1.634 + for i in range(2, len(nodelist), 2):
1.635 + nl = nodelist[i-1]
1.636 +
1.637 + # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.638 + # | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.639 + n = nl[1]
1.640 + if n[0] == token.NAME:
1.641 + type = n[1]
1.642 + if len(nl) == 3:
1.643 + if type == 'not':
1.644 + type = 'not in'
1.645 + else:
1.646 + type = 'is not'
1.647 + else:
1.648 + type = _cmp_types[n[0]]
1.649 +
1.650 + lineno = nl[1][2]
1.651 + results.append((type, self.com_node(nodelist[i])))
1.652 +
1.653 + # we need a special "compare" node so that we can distinguish
1.654 + # 3 < x < 5 from (3 < x) < 5
1.655 + # the two have very different semantics and results (note that the
1.656 + # latter form is always true)
1.657 +
1.658 + return Compare(node, results, lineno=lineno)
1.659 +
1.660 + def expr(self, nodelist):
1.661 + # xor_expr ('|' xor_expr)*
1.662 + return self.com_binary(Bitor, nodelist)
1.663 +
1.664 + def xor_expr(self, nodelist):
1.665 + # xor_expr ('^' xor_expr)*
1.666 + return self.com_binary(Bitxor, nodelist)
1.667 +
1.668 + def and_expr(self, nodelist):
1.669 + # xor_expr ('&' xor_expr)*
1.670 + return self.com_binary(Bitand, nodelist)
1.671 +
1.672 + def shift_expr(self, nodelist):
1.673 + # shift_expr ('<<'|'>>' shift_expr)*
1.674 + node = self.com_node(nodelist[0])
1.675 + for i in range(2, len(nodelist), 2):
1.676 + right = self.com_node(nodelist[i])
1.677 + if nodelist[i-1][0] == token.LEFTSHIFT:
1.678 + node = LeftShift([node, right], lineno=nodelist[1][2])
1.679 + elif nodelist[i-1][0] == token.RIGHTSHIFT:
1.680 + node = RightShift([node, right], lineno=nodelist[1][2])
1.681 + else:
1.682 + raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.683 + return node
1.684 +
1.685 + def arith_expr(self, nodelist):
1.686 + node = self.com_node(nodelist[0])
1.687 + for i in range(2, len(nodelist), 2):
1.688 + right = self.com_node(nodelist[i])
1.689 + if nodelist[i-1][0] == token.PLUS:
1.690 + node = Add([node, right], lineno=nodelist[1][2])
1.691 + elif nodelist[i-1][0] == token.MINUS:
1.692 + node = Sub([node, right], lineno=nodelist[1][2])
1.693 + else:
1.694 + raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.695 + return node
1.696 +
1.697 + def term(self, nodelist):
1.698 + node = self.com_node(nodelist[0])
1.699 + for i in range(2, len(nodelist), 2):
1.700 + right = self.com_node(nodelist[i])
1.701 + t = nodelist[i-1][0]
1.702 + if t == token.STAR:
1.703 + node = Mul([node, right])
1.704 + elif t == token.SLASH:
1.705 + node = Div([node, right])
1.706 + elif t == token.PERCENT:
1.707 + node = Mod([node, right])
1.708 + elif t == token.DOUBLESLASH:
1.709 + node = FloorDiv([node, right])
1.710 + else:
1.711 + raise ValueError, "unexpected token: %s" % t
1.712 + node.lineno = nodelist[1][2]
1.713 + return node
1.714 +
1.715 + def factor(self, nodelist):
1.716 + elt = nodelist[0]
1.717 + t = elt[0]
1.718 + node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
1.719 + # need to handle (unary op)constant here...
1.720 + if t == token.PLUS:
1.721 + return UnaryAdd(node, lineno=elt[2])
1.722 + elif t == token.MINUS:
1.723 + return UnarySub(node, lineno=elt[2])
1.724 + elif t == token.TILDE:
1.725 + node = Invert(node, lineno=elt[2])
1.726 + return node
1.727 +
1.728 + def power(self, nodelist):
1.729 + # power: atom trailer* ('**' factor)*
1.730 + node = self.com_node(nodelist[0])
1.731 + for i in range(1, len(nodelist)):
1.732 + elt = nodelist[i]
1.733 + if elt[0] == token.DOUBLESTAR:
1.734 + return Power([node, self.com_node(nodelist[i+1])],
1.735 + lineno=elt[2])
1.736 +
1.737 + node = self.com_apply_trailer(node, elt)
1.738 +
1.739 + return node
1.740 +
1.741 + def atom(self, nodelist):
1.742 + return self._atom_dispatch[nodelist[0][0]](nodelist)
1.743 +
1.744 + def atom_lpar(self, nodelist):
1.745 + if nodelist[1][0] == token.RPAR:
1.746 + return Tuple((), lineno=nodelist[0][2])
1.747 + return self.com_node(nodelist[1])
1.748 +
1.749 + def atom_lsqb(self, nodelist):
1.750 + if nodelist[1][0] == token.RSQB:
1.751 + return List((), lineno=nodelist[0][2])
1.752 + return self.com_list_constructor(nodelist[1])
1.753 +
1.754 + def atom_lbrace(self, nodelist):
1.755 + if nodelist[1][0] == token.RBRACE:
1.756 + return Dict((), lineno=nodelist[0][2])
1.757 + return self.com_dictorsetmaker(nodelist[1])
1.758 +
1.759 + def atom_backquote(self, nodelist):
1.760 + return Backquote(self.com_node(nodelist[1]))
1.761 +
1.762 + def atom_number(self, nodelist):
1.763 + ### need to verify this matches compile.c
1.764 + k = eval(nodelist[0][1])
1.765 + return Const(k, lineno=nodelist[0][2])
1.766 +
1.767 + def decode_literal(self, lit):
1.768 + if self.encoding:
1.769 + # this is particularly fragile & a bit of a
1.770 + # hack... changes in compile.c:parsestr and
1.771 + # tokenizer.c must be reflected here.
1.772 + if self.encoding not in ['utf-8', 'iso-8859-1']:
1.773 + lit = unicode(lit, 'utf-8').encode(self.encoding)
1.774 + return eval("# coding: %s\n%s" % (self.encoding, lit))
1.775 + else:
1.776 + return eval(lit)
1.777 +
1.778 + def atom_string(self, nodelist):
1.779 + k = ''
1.780 + for node in nodelist:
1.781 + k += self.decode_literal(node[1])
1.782 + return Const(k, lineno=nodelist[0][2])
1.783 +
1.784 + def atom_name(self, nodelist):
1.785 + return Name(nodelist[0][1], lineno=nodelist[0][2])
1.786 +
1.787 + # --------------------------------------------------------------
1.788 + #
1.789 + # INTERNAL PARSING UTILITIES
1.790 + #
1.791 +
1.792 + # The use of com_node() introduces a lot of extra stack frames,
1.793 + # enough to cause a stack overflow compiling test.test_parser with
1.794 + # the standard interpreter recursionlimit. The com_node() is a
1.795 + # convenience function that hides the dispatch details, but comes
1.796 + # at a very high cost. It is more efficient to dispatch directly
1.797 + # in the callers. In these cases, use lookup_node() and call the
1.798 + # dispatched node directly.
1.799 +
1.800 + def lookup_node(self, node):
1.801 + return self._dispatch[node[0]]
1.802 +
1.803 + def com_node(self, node):
1.804 + # Note: compile.c has handling in com_node for del_stmt, pass_stmt,
1.805 + # break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
1.806 + # and compound_stmt.
1.807 + # We'll just dispatch them.
1.808 + return self._dispatch[node[0]](node[1:])
1.809 +
1.810 + def com_NEWLINE(self, *args):
1.811 + # A ';' at the end of a line can make a NEWLINE token appear
1.812 + # here, Render it harmless. (genc discards ('discard',
1.813 + # ('const', xxxx)) Nodes)
1.814 + return Discard(Const(None))
1.815 +
1.816 + def com_arglist(self, nodelist):
1.817 + # varargslist:
1.818 + # (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME)
1.819 + # | fpdef ['=' test] (',' fpdef ['=' test])* [',']
1.820 + # fpdef: NAME | '(' fplist ')'
1.821 + # fplist: fpdef (',' fpdef)* [',']
1.822 + names = []
1.823 + defaults = []
1.824 + flags = 0
1.825 +
1.826 + i = 0
1.827 + while i < len(nodelist):
1.828 + node = nodelist[i]
1.829 + if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
1.830 + if node[0] == token.STAR:
1.831 + node = nodelist[i+1]
1.832 + if node[0] == token.NAME:
1.833 + names.append(node[1])
1.834 + flags = flags | CO_VARARGS
1.835 + i = i + 3
1.836 +
1.837 + if i < len(nodelist):
1.838 + # should be DOUBLESTAR
1.839 + t = nodelist[i][0]
1.840 + if t == token.DOUBLESTAR:
1.841 + node = nodelist[i+1]
1.842 + else:
1.843 + raise ValueError, "unexpected token: %s" % t
1.844 + names.append(node[1])
1.845 + flags = flags | CO_VARKEYWORDS
1.846 +
1.847 + break
1.848 +
1.849 + # fpdef: NAME | '(' fplist ')'
1.850 + names.append(self.com_fpdef(node))
1.851 +
1.852 + i = i + 1
1.853 + if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
1.854 + defaults.append(self.com_node(nodelist[i + 1]))
1.855 + i = i + 2
1.856 + elif len(defaults):
1.857 + # we have already seen an argument with default, but here
1.858 + # came one without
1.859 + raise SyntaxError, "non-default argument follows default argument"
1.860 +
1.861 + # skip the comma
1.862 + i = i + 1
1.863 +
1.864 + return names, defaults, flags
1.865 +
1.866 + def com_fpdef(self, node):
1.867 + # fpdef: NAME | '(' fplist ')'
1.868 + if node[1][0] == token.LPAR:
1.869 + return self.com_fplist(node[2])
1.870 + return node[1][1]
1.871 +
1.872 + def com_fplist(self, node):
1.873 + # fplist: fpdef (',' fpdef)* [',']
1.874 + if len(node) == 2:
1.875 + return self.com_fpdef(node[1])
1.876 + list = []
1.877 + for i in range(1, len(node), 2):
1.878 + list.append(self.com_fpdef(node[i]))
1.879 + return tuple(list)
1.880 +
1.881 + def com_dotted_name(self, node):
1.882 + # String together the dotted names and return the string
1.883 + name = ""
1.884 + for n in node:
1.885 + if type(n) == type(()) and n[0] == 1:
1.886 + name = name + n[1] + '.'
1.887 + return name[:-1]
1.888 +
1.889 + def com_dotted_as_name(self, node):
1.890 + assert node[0] == symbol.dotted_as_name
1.891 + node = node[1:]
1.892 + dot = self.com_dotted_name(node[0][1:])
1.893 + if len(node) == 1:
1.894 + return dot, None
1.895 + assert node[1][1] == 'as'
1.896 + assert node[2][0] == token.NAME
1.897 + return dot, node[2][1]
1.898 +
1.899 + def com_dotted_as_names(self, node):
1.900 + assert node[0] == symbol.dotted_as_names
1.901 + node = node[1:]
1.902 + names = [self.com_dotted_as_name(node[0])]
1.903 + for i in range(2, len(node), 2):
1.904 + names.append(self.com_dotted_as_name(node[i]))
1.905 + return names
1.906 +
1.907 + def com_import_as_name(self, node):
1.908 + assert node[0] == symbol.import_as_name
1.909 + node = node[1:]
1.910 + assert node[0][0] == token.NAME
1.911 + if len(node) == 1:
1.912 + return node[0][1], None
1.913 + assert node[1][1] == 'as', node
1.914 + assert node[2][0] == token.NAME
1.915 + return node[0][1], node[2][1]
1.916 +
1.917 + def com_import_as_names(self, node):
1.918 + assert node[0] == symbol.import_as_names
1.919 + node = node[1:]
1.920 + names = [self.com_import_as_name(node[0])]
1.921 + for i in range(2, len(node), 2):
1.922 + names.append(self.com_import_as_name(node[i]))
1.923 + return names
1.924 +
1.925 + def com_bases(self, node):
1.926 + bases = []
1.927 + for i in range(1, len(node), 2):
1.928 + bases.append(self.com_node(node[i]))
1.929 + return bases
1.930 +
1.931 + def com_try_except_finally(self, nodelist):
1.932 + # ('try' ':' suite
1.933 + # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]
1.934 + # | 'finally' ':' suite))
1.935 +
1.936 + if nodelist[3][0] == token.NAME:
1.937 + # first clause is a finally clause: only try-finally
1.938 + return TryFinally(self.com_node(nodelist[2]),
1.939 + self.com_node(nodelist[5]),
1.940 + lineno=nodelist[0][2])
1.941 +
1.942 + #tryexcept: [TryNode, [except_clauses], elseNode)]
1.943 + clauses = []
1.944 + elseNode = None
1.945 + finallyNode = None
1.946 + for i in range(3, len(nodelist), 3):
1.947 + node = nodelist[i]
1.948 + if node[0] == symbol.except_clause:
1.949 + # except_clause: 'except' [expr [(',' | 'as') expr]] */
1.950 + if len(node) > 2:
1.951 + expr1 = self.com_node(node[2])
1.952 + if len(node) > 4:
1.953 + expr2 = self.com_assign(node[4], OP_ASSIGN)
1.954 + else:
1.955 + expr2 = None
1.956 + else:
1.957 + expr1 = expr2 = None
1.958 + clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
1.959 +
1.960 + if node[0] == token.NAME:
1.961 + if node[1] == 'else':
1.962 + elseNode = self.com_node(nodelist[i+2])
1.963 + elif node[1] == 'finally':
1.964 + finallyNode = self.com_node(nodelist[i+2])
1.965 + try_except = TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
1.966 + lineno=nodelist[0][2])
1.967 + if finallyNode:
1.968 + return TryFinally(try_except, finallyNode, lineno=nodelist[0][2])
1.969 + else:
1.970 + return try_except
1.971 +
1.972 + def com_with(self, nodelist):
1.973 + # with_stmt: 'with' with_item (',' with_item)* ':' suite
1.974 + body = self.com_node(nodelist[-1])
1.975 + for i in range(len(nodelist) - 3, 0, -2):
1.976 + ret = self.com_with_item(nodelist[i], body, nodelist[0][2])
1.977 + if i == 1:
1.978 + return ret
1.979 + body = ret
1.980 +
1.981 + def com_with_item(self, nodelist, body, lineno):
1.982 + # with_item: test ['as' expr]
1.983 + if len(nodelist) == 4:
1.984 + var = self.com_assign(nodelist[3], OP_ASSIGN)
1.985 + else:
1.986 + var = None
1.987 + expr = self.com_node(nodelist[1])
1.988 + return With(expr, var, body, lineno=lineno)
1.989 +
1.990 + def com_augassign_op(self, node):
1.991 + assert node[0] == symbol.augassign
1.992 + return node[1]
1.993 +
1.994 + def com_augassign(self, node):
1.995 + """Return node suitable for lvalue of augmented assignment
1.996 +
1.997 + Names, slices, and attributes are the only allowable nodes.
1.998 + """
1.999 + l = self.com_node(node)
1.1000 + if l.__class__ in (Name, Slice, Subscript, Getattr):
1.1001 + return l
1.1002 + raise SyntaxError, "can't assign to %s" % l.__class__.__name__
1.1003 +
1.1004 + def com_assign(self, node, assigning):
1.1005 + # return a node suitable for use as an "lvalue"
1.1006 + # loop to avoid trivial recursion
1.1007 + while 1:
1.1008 + t = node[0]
1.1009 + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp):
1.1010 + if len(node) > 2:
1.1011 + return self.com_assign_tuple(node, assigning)
1.1012 + node = node[1]
1.1013 + elif t in _assign_types:
1.1014 + if len(node) > 2:
1.1015 + raise SyntaxError, "can't assign to operator"
1.1016 + node = node[1]
1.1017 + elif t == symbol.power:
1.1018 + if node[1][0] != symbol.atom:
1.1019 + raise SyntaxError, "can't assign to operator"
1.1020 + if len(node) > 2:
1.1021 + primary = self.com_node(node[1])
1.1022 + for i in range(2, len(node)-1):
1.1023 + ch = node[i]
1.1024 + if ch[0] == token.DOUBLESTAR:
1.1025 + raise SyntaxError, "can't assign to operator"
1.1026 + primary = self.com_apply_trailer(primary, ch)
1.1027 + return self.com_assign_trailer(primary, node[-1],
1.1028 + assigning)
1.1029 + node = node[1]
1.1030 + elif t == symbol.atom:
1.1031 + t = node[1][0]
1.1032 + if t == token.LPAR:
1.1033 + node = node[2]
1.1034 + if node[0] == token.RPAR:
1.1035 + raise SyntaxError, "can't assign to ()"
1.1036 + elif t == token.LSQB:
1.1037 + node = node[2]
1.1038 + if node[0] == token.RSQB:
1.1039 + raise SyntaxError, "can't assign to []"
1.1040 + return self.com_assign_list(node, assigning)
1.1041 + elif t == token.NAME:
1.1042 + return self.com_assign_name(node[1], assigning)
1.1043 + else:
1.1044 + raise SyntaxError, "can't assign to literal"
1.1045 + else:
1.1046 + raise SyntaxError, "bad assignment (%s)" % t
1.1047 +
1.1048 + def com_assign_tuple(self, node, assigning):
1.1049 + assigns = []
1.1050 + for i in range(1, len(node), 2):
1.1051 + assigns.append(self.com_assign(node[i], assigning))
1.1052 + return AssTuple(assigns, lineno=extractLineNo(node))
1.1053 +
1.1054 + def com_assign_list(self, node, assigning):
1.1055 + assigns = []
1.1056 + for i in range(1, len(node), 2):
1.1057 + if i + 1 < len(node):
1.1058 + if node[i + 1][0] == symbol.list_for:
1.1059 + raise SyntaxError, "can't assign to list comprehension"
1.1060 + assert node[i + 1][0] == token.COMMA, node[i + 1]
1.1061 + assigns.append(self.com_assign(node[i], assigning))
1.1062 + return AssList(assigns, lineno=extractLineNo(node))
1.1063 +
1.1064 + def com_assign_name(self, node, assigning):
1.1065 + return AssName(node[1], assigning, lineno=node[2])
1.1066 +
1.1067 + def com_assign_trailer(self, primary, node, assigning):
1.1068 + t = node[1][0]
1.1069 + if t == token.DOT:
1.1070 + return self.com_assign_attr(primary, node[2], assigning)
1.1071 + if t == token.LSQB:
1.1072 + return self.com_subscriptlist(primary, node[2], assigning)
1.1073 + if t == token.LPAR:
1.1074 + raise SyntaxError, "can't assign to function call"
1.1075 + raise SyntaxError, "unknown trailer type: %s" % t
1.1076 +
1.1077 + def com_assign_attr(self, primary, node, assigning):
1.1078 + return AssAttr(primary, node[1], assigning, lineno=node[-1])
1.1079 +
1.1080 + def com_binary(self, constructor, nodelist):
1.1081 + "Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
1.1082 + l = len(nodelist)
1.1083 + if l == 1:
1.1084 + n = nodelist[0]
1.1085 + return self.lookup_node(n)(n[1:])
1.1086 + items = []
1.1087 + for i in range(0, l, 2):
1.1088 + n = nodelist[i]
1.1089 + items.append(self.lookup_node(n)(n[1:]))
1.1090 + return constructor(items, lineno=extractLineNo(nodelist))
1.1091 +
1.1092 + def com_stmt(self, node):
1.1093 + result = self.lookup_node(node)(node[1:])
1.1094 + assert result is not None
1.1095 + if isinstance(result, Stmt):
1.1096 + return result
1.1097 + return Stmt([result])
1.1098 +
1.1099 + def com_append_stmt(self, stmts, node):
1.1100 + result = self.lookup_node(node)(node[1:])
1.1101 + assert result is not None
1.1102 + if isinstance(result, Stmt):
1.1103 + stmts.extend(result.nodes)
1.1104 + else:
1.1105 + stmts.append(result)
1.1106 +
1.1107 + def com_list_constructor(self, nodelist):
1.1108 + # listmaker: test ( list_for | (',' test)* [','] )
1.1109 + values = []
1.1110 + for i in range(1, len(nodelist)):
1.1111 + if nodelist[i][0] == symbol.list_for:
1.1112 + assert len(nodelist[i:]) == 1
1.1113 + return self.com_list_comprehension(values[0],
1.1114 + nodelist[i])
1.1115 + elif nodelist[i][0] == token.COMMA:
1.1116 + continue
1.1117 + values.append(self.com_node(nodelist[i]))
1.1118 + return List(values, lineno=values[0].lineno)
1.1119 +
1.1120 + def com_list_comprehension(self, expr, node):
1.1121 + return self.com_comprehension(expr, None, node, 'list')
1.1122 +
1.1123 + def com_comprehension(self, expr1, expr2, node, type):
1.1124 + # list_iter: list_for | list_if
1.1125 + # list_for: 'for' exprlist 'in' testlist [list_iter]
1.1126 + # list_if: 'if' test [list_iter]
1.1127 +
1.1128 + # XXX should raise SyntaxError for assignment
1.1129 + # XXX(avassalotti) Set and dict comprehensions should have generator
1.1130 + # semantics. In other words, they shouldn't leak
1.1131 + # variables outside of the comprehension's scope.
1.1132 +
1.1133 + lineno = node[1][2]
1.1134 + fors = []
1.1135 + while node:
1.1136 + t = node[1][1]
1.1137 + if t == 'for':
1.1138 + assignNode = self.com_assign(node[2], OP_ASSIGN)
1.1139 + compNode = self.com_node(node[4])
1.1140 + newfor = ListCompFor(assignNode, compNode, [])
1.1141 + newfor.lineno = node[1][2]
1.1142 + fors.append(newfor)
1.1143 + if len(node) == 5:
1.1144 + node = None
1.1145 + elif type == 'list':
1.1146 + node = self.com_list_iter(node[5])
1.1147 + else:
1.1148 + node = self.com_comp_iter(node[5])
1.1149 + elif t == 'if':
1.1150 + test = self.com_node(node[2])
1.1151 + newif = ListCompIf(test, lineno=node[1][2])
1.1152 + newfor.ifs.append(newif)
1.1153 + if len(node) == 3:
1.1154 + node = None
1.1155 + elif type == 'list':
1.1156 + node = self.com_list_iter(node[3])
1.1157 + else:
1.1158 + node = self.com_comp_iter(node[3])
1.1159 + else:
1.1160 + raise SyntaxError, \
1.1161 + ("unexpected comprehension element: %s %d"
1.1162 + % (node, lineno))
1.1163 + if type == 'list':
1.1164 + return ListComp(expr1, fors, lineno=lineno)
1.1165 + elif type == 'set':
1.1166 + return SetComp(expr1, fors, lineno=lineno)
1.1167 + elif type == 'dict':
1.1168 + return DictComp(expr1, expr2, fors, lineno=lineno)
1.1169 + else:
1.1170 + raise ValueError("unexpected comprehension type: " + repr(type))
1.1171 +
1.1172 + def com_list_iter(self, node):
1.1173 + assert node[0] == symbol.list_iter
1.1174 + return node[1]
1.1175 +
1.1176 + def com_comp_iter(self, node):
1.1177 + assert node[0] == symbol.comp_iter
1.1178 + return node[1]
1.1179 +
1.1180 + def com_generator_expression(self, expr, node):
1.1181 + # comp_iter: comp_for | comp_if
1.1182 + # comp_for: 'for' exprlist 'in' test [comp_iter]
1.1183 + # comp_if: 'if' test [comp_iter]
1.1184 +
1.1185 + lineno = node[1][2]
1.1186 + fors = []
1.1187 + while node:
1.1188 + t = node[1][1]
1.1189 + if t == 'for':
1.1190 + assignNode = self.com_assign(node[2], OP_ASSIGN)
1.1191 + genNode = self.com_node(node[4])
1.1192 + newfor = GenExprFor(assignNode, genNode, [],
1.1193 + lineno=node[1][2])
1.1194 + fors.append(newfor)
1.1195 + if (len(node)) == 5:
1.1196 + node = None
1.1197 + else:
1.1198 + node = self.com_comp_iter(node[5])
1.1199 + elif t == 'if':
1.1200 + test = self.com_node(node[2])
1.1201 + newif = GenExprIf(test, lineno=node[1][2])
1.1202 + newfor.ifs.append(newif)
1.1203 + if len(node) == 3:
1.1204 + node = None
1.1205 + else:
1.1206 + node = self.com_comp_iter(node[3])
1.1207 + else:
1.1208 + raise SyntaxError, \
1.1209 + ("unexpected generator expression element: %s %d"
1.1210 + % (node, lineno))
1.1211 + fors[0].is_outmost = True
1.1212 + return GenExpr(GenExprInner(expr, fors), lineno=lineno)
1.1213 +
1.1214 + def com_dictorsetmaker(self, nodelist):
1.1215 + # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
1.1216 + # (test (comp_for | (',' test)* [','])) )
1.1217 + assert nodelist[0] == symbol.dictorsetmaker
1.1218 + nodelist = nodelist[1:]
1.1219 + if len(nodelist) == 1 or nodelist[1][0] == token.COMMA:
1.1220 + # set literal
1.1221 + items = []
1.1222 + for i in range(0, len(nodelist), 2):
1.1223 + items.append(self.com_node(nodelist[i]))
1.1224 + return Set(items, lineno=items[0].lineno)
1.1225 + elif nodelist[1][0] == symbol.comp_for:
1.1226 + # set comprehension
1.1227 + expr = self.com_node(nodelist[0])
1.1228 + return self.com_comprehension(expr, None, nodelist[1], 'set')
1.1229 + elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for:
1.1230 + # dict comprehension
1.1231 + assert nodelist[1][0] == token.COLON
1.1232 + key = self.com_node(nodelist[0])
1.1233 + value = self.com_node(nodelist[2])
1.1234 + return self.com_comprehension(key, value, nodelist[3], 'dict')
1.1235 + else:
1.1236 + # dict literal
1.1237 + items = []
1.1238 + for i in range(0, len(nodelist), 4):
1.1239 + items.append((self.com_node(nodelist[i]),
1.1240 + self.com_node(nodelist[i+2])))
1.1241 + return Dict(items, lineno=items[0][0].lineno)
1.1242 +
1.1243 + def com_apply_trailer(self, primaryNode, nodelist):
1.1244 + t = nodelist[1][0]
1.1245 + if t == token.LPAR:
1.1246 + return self.com_call_function(primaryNode, nodelist[2])
1.1247 + if t == token.DOT:
1.1248 + return self.com_select_member(primaryNode, nodelist[2])
1.1249 + if t == token.LSQB:
1.1250 + return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
1.1251 +
1.1252 + raise SyntaxError, 'unknown node type: %s' % t
1.1253 +
1.1254 + def com_select_member(self, primaryNode, nodelist):
1.1255 + if nodelist[0] != token.NAME:
1.1256 + raise SyntaxError, "member must be a name"
1.1257 + return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
1.1258 +
1.1259 + def com_call_function(self, primaryNode, nodelist):
1.1260 + if nodelist[0] == token.RPAR:
1.1261 + return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
1.1262 + args = []
1.1263 + kw = 0
1.1264 + star_node = dstar_node = None
1.1265 + len_nodelist = len(nodelist)
1.1266 + i = 1
1.1267 + while i < len_nodelist:
1.1268 + node = nodelist[i]
1.1269 +
1.1270 + if node[0]==token.STAR:
1.1271 + if star_node is not None:
1.1272 + raise SyntaxError, 'already have the varargs indentifier'
1.1273 + star_node = self.com_node(nodelist[i+1])
1.1274 + i = i + 3
1.1275 + continue
1.1276 + elif node[0]==token.DOUBLESTAR:
1.1277 + if dstar_node is not None:
1.1278 + raise SyntaxError, 'already have the kwargs indentifier'
1.1279 + dstar_node = self.com_node(nodelist[i+1])
1.1280 + i = i + 3
1.1281 + continue
1.1282 +
1.1283 + # positional or named parameters
1.1284 + kw, result = self.com_argument(node, kw, star_node)
1.1285 +
1.1286 + if len_nodelist != 2 and isinstance(result, GenExpr) \
1.1287 + and len(node) == 3 and node[2][0] == symbol.comp_for:
1.1288 + # allow f(x for x in y), but reject f(x for x in y, 1)
1.1289 + # should use f((x for x in y), 1) instead of f(x for x in y, 1)
1.1290 + raise SyntaxError, 'generator expression needs parenthesis'
1.1291 +
1.1292 + args.append(result)
1.1293 + i = i + 2
1.1294 +
1.1295 + return CallFunc(primaryNode, args, star_node, dstar_node,
1.1296 + lineno=extractLineNo(nodelist))
1.1297 +
1.1298 + def com_argument(self, nodelist, kw, star_node):
1.1299 + if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for:
1.1300 + test = self.com_node(nodelist[1])
1.1301 + return 0, self.com_generator_expression(test, nodelist[2])
1.1302 + if len(nodelist) == 2:
1.1303 + if kw:
1.1304 + raise SyntaxError, "non-keyword arg after keyword arg"
1.1305 + if star_node:
1.1306 + raise SyntaxError, "only named arguments may follow *expression"
1.1307 + return 0, self.com_node(nodelist[1])
1.1308 + result = self.com_node(nodelist[3])
1.1309 + n = nodelist[1]
1.1310 + while len(n) == 2 and n[0] != token.NAME:
1.1311 + n = n[1]
1.1312 + if n[0] != token.NAME:
1.1313 + raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
1.1314 + node = Keyword(n[1], result, lineno=n[2])
1.1315 + return 1, node
1.1316 +
1.1317 + def com_subscriptlist(self, primary, nodelist, assigning):
1.1318 + # slicing: simple_slicing | extended_slicing
1.1319 + # simple_slicing: primary "[" short_slice "]"
1.1320 + # extended_slicing: primary "[" slice_list "]"
1.1321 + # slice_list: slice_item ("," slice_item)* [","]
1.1322 +
1.1323 + # backwards compat slice for '[i:j]'
1.1324 + if len(nodelist) == 2:
1.1325 + sub = nodelist[1]
1.1326 + if (sub[1][0] == token.COLON or \
1.1327 + (len(sub) > 2 and sub[2][0] == token.COLON)) and \
1.1328 + sub[-1][0] != symbol.sliceop:
1.1329 + return self.com_slice(primary, sub, assigning)
1.1330 +
1.1331 + subscripts = []
1.1332 + for i in range(1, len(nodelist), 2):
1.1333 + subscripts.append(self.com_subscript(nodelist[i]))
1.1334 + return Subscript(primary, assigning, subscripts,
1.1335 + lineno=extractLineNo(nodelist))
1.1336 +
1.1337 + def com_subscript(self, node):
1.1338 + # slice_item: expression | proper_slice | ellipsis
1.1339 + ch = node[1]
1.1340 + t = ch[0]
1.1341 + if t == token.DOT and node[2][0] == token.DOT:
1.1342 + return Ellipsis()
1.1343 + if t == token.COLON or len(node) > 2:
1.1344 + return self.com_sliceobj(node)
1.1345 + return self.com_node(ch)
1.1346 +
1.1347 + def com_sliceobj(self, node):
1.1348 + # proper_slice: short_slice | long_slice
1.1349 + # short_slice: [lower_bound] ":" [upper_bound]
1.1350 + # long_slice: short_slice ":" [stride]
1.1351 + # lower_bound: expression
1.1352 + # upper_bound: expression
1.1353 + # stride: expression
1.1354 + #
1.1355 + # Note: a stride may be further slicing...
1.1356 +
1.1357 + items = []
1.1358 +
1.1359 + if node[1][0] == token.COLON:
1.1360 + items.append(Const(None))
1.1361 + i = 2
1.1362 + else:
1.1363 + items.append(self.com_node(node[1]))
1.1364 + # i == 2 is a COLON
1.1365 + i = 3
1.1366 +
1.1367 + if i < len(node) and node[i][0] == symbol.test:
1.1368 + items.append(self.com_node(node[i]))
1.1369 + i = i + 1
1.1370 + else:
1.1371 + items.append(Const(None))
1.1372 +
1.1373 + # a short_slice has been built. look for long_slice now by looking
1.1374 + # for strides...
1.1375 + for j in range(i, len(node)):
1.1376 + ch = node[j]
1.1377 + if len(ch) == 2:
1.1378 + items.append(Const(None))
1.1379 + else:
1.1380 + items.append(self.com_node(ch[2]))
1.1381 + return Sliceobj(items, lineno=extractLineNo(node))
1.1382 +
1.1383 + def com_slice(self, primary, node, assigning):
1.1384 + # short_slice: [lower_bound] ":" [upper_bound]
1.1385 + lower = upper = None
1.1386 + if len(node) == 3:
1.1387 + if node[1][0] == token.COLON:
1.1388 + upper = self.com_node(node[2])
1.1389 + else:
1.1390 + lower = self.com_node(node[1])
1.1391 + elif len(node) == 4:
1.1392 + lower = self.com_node(node[1])
1.1393 + upper = self.com_node(node[3])
1.1394 + return Slice(primary, assigning, lower, upper,
1.1395 + lineno=extractLineNo(node))
1.1396 +
1.1397 + def get_docstring(self, node, n=None):
1.1398 + if n is None:
1.1399 + n = node[0]
1.1400 + node = node[1:]
1.1401 + if n == symbol.suite:
1.1402 + if len(node) == 1:
1.1403 + return self.get_docstring(node[0])
1.1404 + for sub in node:
1.1405 + if sub[0] == symbol.stmt:
1.1406 + return self.get_docstring(sub)
1.1407 + return None
1.1408 + if n == symbol.file_input:
1.1409 + for sub in node:
1.1410 + if sub[0] == symbol.stmt:
1.1411 + return self.get_docstring(sub)
1.1412 + return None
1.1413 + if n == symbol.atom:
1.1414 + if node[0][0] == token.STRING:
1.1415 + s = ''
1.1416 + for t in node:
1.1417 + s = s + eval(t[1])
1.1418 + return s
1.1419 + return None
1.1420 + if n == symbol.stmt or n == symbol.simple_stmt \
1.1421 + or n == symbol.small_stmt:
1.1422 + return self.get_docstring(node[0])
1.1423 + if n in _doc_nodes and len(node) == 1:
1.1424 + return self.get_docstring(node[0])
1.1425 + return None
1.1426 +
1.1427 +
1.1428 +_doc_nodes = [
1.1429 + symbol.expr_stmt,
1.1430 + symbol.testlist,
1.1431 + symbol.testlist_safe,
1.1432 + symbol.test,
1.1433 + symbol.or_test,
1.1434 + symbol.and_test,
1.1435 + symbol.not_test,
1.1436 + symbol.comparison,
1.1437 + symbol.expr,
1.1438 + symbol.xor_expr,
1.1439 + symbol.and_expr,
1.1440 + symbol.shift_expr,
1.1441 + symbol.arith_expr,
1.1442 + symbol.term,
1.1443 + symbol.factor,
1.1444 + symbol.power,
1.1445 + ]
1.1446 +
1.1447 +# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.1448 +# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.1449 +_cmp_types = {
1.1450 + token.LESS : '<',
1.1451 + token.GREATER : '>',
1.1452 + token.EQEQUAL : '==',
1.1453 + token.EQUAL : '==',
1.1454 + token.LESSEQUAL : '<=',
1.1455 + token.GREATEREQUAL : '>=',
1.1456 + token.NOTEQUAL : '!=',
1.1457 + }
1.1458 +
1.1459 +_legal_node_types = [
1.1460 + symbol.funcdef,
1.1461 + symbol.classdef,
1.1462 + symbol.stmt,
1.1463 + symbol.small_stmt,
1.1464 + symbol.flow_stmt,
1.1465 + symbol.simple_stmt,
1.1466 + symbol.compound_stmt,
1.1467 + symbol.expr_stmt,
1.1468 + symbol.print_stmt,
1.1469 + symbol.del_stmt,
1.1470 + symbol.pass_stmt,
1.1471 + symbol.break_stmt,
1.1472 + symbol.continue_stmt,
1.1473 + symbol.return_stmt,
1.1474 + symbol.raise_stmt,
1.1475 + symbol.import_stmt,
1.1476 + symbol.global_stmt,
1.1477 + symbol.exec_stmt,
1.1478 + symbol.assert_stmt,
1.1479 + symbol.if_stmt,
1.1480 + symbol.while_stmt,
1.1481 + symbol.for_stmt,
1.1482 + symbol.try_stmt,
1.1483 + symbol.with_stmt,
1.1484 + symbol.suite,
1.1485 + symbol.testlist,
1.1486 + symbol.testlist_safe,
1.1487 + symbol.test,
1.1488 + symbol.and_test,
1.1489 + symbol.not_test,
1.1490 + symbol.comparison,
1.1491 + symbol.exprlist,
1.1492 + symbol.expr,
1.1493 + symbol.xor_expr,
1.1494 + symbol.and_expr,
1.1495 + symbol.shift_expr,
1.1496 + symbol.arith_expr,
1.1497 + symbol.term,
1.1498 + symbol.factor,
1.1499 + symbol.power,
1.1500 + symbol.atom,
1.1501 + ]
1.1502 +
1.1503 +if hasattr(symbol, 'yield_stmt'):
1.1504 + _legal_node_types.append(symbol.yield_stmt)
1.1505 +if hasattr(symbol, 'yield_expr'):
1.1506 + _legal_node_types.append(symbol.yield_expr)
1.1507 +
1.1508 +_assign_types = [
1.1509 + symbol.test,
1.1510 + symbol.or_test,
1.1511 + symbol.and_test,
1.1512 + symbol.not_test,
1.1513 + symbol.comparison,
1.1514 + symbol.expr,
1.1515 + symbol.xor_expr,
1.1516 + symbol.and_expr,
1.1517 + symbol.shift_expr,
1.1518 + symbol.arith_expr,
1.1519 + symbol.term,
1.1520 + symbol.factor,
1.1521 + ]
1.1522 +
1.1523 +_names = {}
1.1524 +for k, v in symbol.sym_name.items():
1.1525 + _names[k] = v
1.1526 +for k, v in token.tok_name.items():
1.1527 + _names[k] = v
1.1528 +
1.1529 +def debug_tree(tree):
1.1530 + l = []
1.1531 + for elt in tree:
1.1532 + if isinstance(elt, int):
1.1533 + l.append(_names.get(elt, elt))
1.1534 + elif isinstance(elt, str):
1.1535 + l.append(elt)
1.1536 + else:
1.1537 + l.append(debug_tree(elt))
1.1538 + return l