1.1 --- a/compiler/transformer.py Sun Jan 08 21:15:30 2017 +0100
1.2 +++ b/compiler/transformer.py Sun Jan 08 23:28:19 2017 +0100
1.3 @@ -26,9 +26,8 @@
1.4 # and replace OWNER, ORGANIZATION, and YEAR as appropriate.
1.5
1.6 from compiler.ast import *
1.7 +from pyparser.pygram import syms as symbol, sym_name, tokens as token, tok_name
1.8 import pyparser.pyparse as parser
1.9 -from pyparser.pygram import syms as symbol
1.10 -import token
1.11
1.12 class WalkerError(StandardError):
1.13 pass
1.14 @@ -89,17 +88,17 @@
1.15
1.16 def __init__(self):
1.17 self._dispatch = {}
1.18 - for value, name in symbol.sym_name.items():
1.19 + for value, name in sym_name.items():
1.20 if hasattr(self, name):
1.21 self._dispatch[value] = getattr(self, name)
1.22 - self._dispatch[token.NEWLINE] = self.com_NEWLINE
1.23 - self._atom_dispatch = {token.LPAR: self.atom_lpar,
1.24 - token.LSQB: self.atom_lsqb,
1.25 - token.LBRACE: self.atom_lbrace,
1.26 - token.BACKQUOTE: self.atom_backquote,
1.27 - token.NUMBER: self.atom_number,
1.28 - token.STRING: self.atom_string,
1.29 - token.NAME: self.atom_name,
1.30 + self._dispatch[token["NEWLINE"]] = self.com_NEWLINE
1.31 + self._atom_dispatch = {token["LPAR"]: self.atom_lpar,
1.32 + token["LSQB"]: self.atom_lsqb,
1.33 + token["LBRACE"]: self.atom_lbrace,
1.34 + token["BACKQUOTE"]: self.atom_backquote,
1.35 + token["NUMBER"]: self.atom_number,
1.36 + token["STRING"]: self.atom_string,
1.37 + token["NAME"]: self.atom_name,
1.38 }
1.39 self.encoding = None
1.40
1.41 @@ -132,22 +131,22 @@
1.42 ### emit a line-number node?
1.43 n = node[0]
1.44
1.45 - if n == symbol.encoding_decl:
1.46 + if n == symbol["encoding_decl"]:
1.47 self.encoding = node[2]
1.48 node = node[1]
1.49 n = node[0]
1.50
1.51 - if n == symbol.single_input:
1.52 + if n == symbol["single_input"]:
1.53 return self.single_input(node[1:])
1.54 - if n == symbol.file_input:
1.55 + if n == symbol["file_input"]:
1.56 return self.file_input(node[1:])
1.57 - if n == symbol.eval_input:
1.58 + if n == symbol["eval_input"]:
1.59 return self.eval_input(node[1:])
1.60 - if n == symbol.lambdef:
1.61 + if n == symbol["lambdef"]:
1.62 return self.lambdef(node[1:])
1.63 - if n == symbol.funcdef:
1.64 + if n == symbol["funcdef"]:
1.65 return self.funcdef(node[1:])
1.66 - if n == symbol.classdef:
1.67 + if n == symbol["classdef"]:
1.68 return self.classdef(node[1:])
1.69
1.70 raise WalkerError, ('unexpected node type', n)
1.71 @@ -157,20 +156,20 @@
1.72
1.73 # NEWLINE | simple_stmt | compound_stmt NEWLINE
1.74 n = node[0][0]
1.75 - if n != token.NEWLINE:
1.76 + if n != token["NEWLINE"]:
1.77 return self.com_stmt(node[0])
1.78
1.79 return Pass()
1.80
1.81 def file_input(self, nodelist):
1.82 - doc = self.get_docstring(nodelist, symbol.file_input)
1.83 + doc = self.get_docstring(nodelist, symbol["file_input"])
1.84 if doc is not None:
1.85 i = 1
1.86 else:
1.87 i = 0
1.88 stmts = []
1.89 for node in nodelist[i:]:
1.90 - if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
1.91 + if node[0] != token["ENDMARKER"] and node[0] != token["NEWLINE"]:
1.92 self.com_append_stmt(stmts, node)
1.93 return Module(doc, Stmt(stmts))
1.94
1.95 @@ -186,8 +185,8 @@
1.96 item = self.atom_name(nodelist)
1.97 i = 1
1.98 while i < listlen:
1.99 - assert nodelist[i][0] == token.DOT
1.100 - assert nodelist[i + 1][0] == token.NAME
1.101 + assert nodelist[i][0] == token["DOT"]
1.102 + assert nodelist[i + 1][0] == token["NAME"]
1.103 item = Getattr(item, nodelist[i + 1][1])
1.104 i += 2
1.105
1.106 @@ -196,14 +195,14 @@
1.107 def decorator(self, nodelist):
1.108 # '@' dotted_name [ '(' [arglist] ')' ]
1.109 assert len(nodelist) in (3, 5, 6)
1.110 - assert nodelist[0][0] == token.AT
1.111 - assert nodelist[-1][0] == token.NEWLINE
1.112 + assert nodelist[0][0] == token["AT"]
1.113 + assert nodelist[-1][0] == token["NEWLINE"]
1.114
1.115 - assert nodelist[1][0] == symbol.dotted_name
1.116 + assert nodelist[1][0] == symbol["dotted_name"]
1.117 funcname = self.decorator_name(nodelist[1][1:])
1.118
1.119 if len(nodelist) > 3:
1.120 - assert nodelist[2][0] == token.LPAR
1.121 + assert nodelist[2][0] == token["LPAR"]
1.122 expr = self.com_call_function(funcname, nodelist[3])
1.123 else:
1.124 expr = funcname
1.125 @@ -214,16 +213,16 @@
1.126 # decorators: decorator ([NEWLINE] decorator)* NEWLINE
1.127 items = []
1.128 for dec_nodelist in nodelist:
1.129 - assert dec_nodelist[0] == symbol.decorator
1.130 + assert dec_nodelist[0] == symbol["decorator"]
1.131 items.append(self.decorator(dec_nodelist[1:]))
1.132 return Decorators(items)
1.133
1.134 def decorated(self, nodelist):
1.135 - assert nodelist[0][0] == symbol.decorators
1.136 - if nodelist[1][0] == symbol.funcdef:
1.137 + assert nodelist[0][0] == symbol["decorators"]
1.138 + if nodelist[1][0] == symbol["funcdef"]:
1.139 n = [nodelist[0]] + list(nodelist[1][1:])
1.140 return self.funcdef(n)
1.141 - elif nodelist[1][0] == symbol.classdef:
1.142 + elif nodelist[1][0] == symbol["classdef"]:
1.143 decorators = self.decorators(nodelist[0][1:])
1.144 cls = self.classdef(nodelist[1][1:])
1.145 cls.decorators = decorators
1.146 @@ -236,7 +235,7 @@
1.147 # parameters: '(' [varargslist] ')'
1.148
1.149 if len(nodelist) == 6:
1.150 - assert nodelist[0][0] == symbol.decorators
1.151 + assert nodelist[0][0] == symbol["decorators"]
1.152 decorators = self.decorators(nodelist[0][1:])
1.153 else:
1.154 assert len(nodelist) == 5
1.155 @@ -246,7 +245,7 @@
1.156 name = nodelist[-4][1]
1.157 args = nodelist[-3][2]
1.158
1.159 - if args[0] == symbol.varargslist:
1.160 + if args[0] == symbol["varargslist"]:
1.161 names, defaults, flags = self.com_arglist(args[1:])
1.162 else:
1.163 names = defaults = ()
1.164 @@ -265,7 +264,7 @@
1.165
1.166 def lambdef(self, nodelist):
1.167 # lambdef: 'lambda' [varargslist] ':' test
1.168 - if nodelist[2][0] == symbol.varargslist:
1.169 + if nodelist[2][0] == symbol["varargslist"]:
1.170 names, defaults, flags = self.com_arglist(nodelist[2][1:])
1.171 else:
1.172 names = defaults = ()
1.173 @@ -282,9 +281,9 @@
1.174
1.175 name = nodelist[1][1]
1.176 doc = self.get_docstring(nodelist[-1])
1.177 - if nodelist[2][0] == token.COLON:
1.178 + if nodelist[2][0] == token["COLON"]:
1.179 bases = []
1.180 - elif nodelist[3][0] == token.RPAR:
1.181 + elif nodelist[3][0] == token["RPAR"]:
1.182 bases = []
1.183 else:
1.184 bases = self.com_bases(nodelist[3])
1.185 @@ -351,7 +350,7 @@
1.186 exprNode = self.lookup_node(en)(en[1:])
1.187 if len(nodelist) == 1:
1.188 return Discard(exprNode, lineno=exprNode.lineno)
1.189 - if nodelist[1][0] == token.EQUAL:
1.190 + if nodelist[1][0] == token["EQUAL"]:
1.191 nodesl = []
1.192 for i in range(0, len(nodelist) - 2, 2):
1.193 nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
1.194 @@ -368,9 +367,9 @@
1.195 if len(nodelist) == 1:
1.196 start = 1
1.197 dest = None
1.198 - elif nodelist[1][0] == token.RIGHTSHIFT:
1.199 + elif nodelist[1][0] == token["RIGHTSHIFT"]:
1.200 assert len(nodelist) == 3 \
1.201 - or nodelist[3][0] == token.COMMA
1.202 + or nodelist[3][0] == token["COMMA"]
1.203 dest = self.com_node(nodelist[2])
1.204 start = 4
1.205 else:
1.206 @@ -378,7 +377,7 @@
1.207 start = 1
1.208 for i in range(start, len(nodelist), 2):
1.209 items.append(self.com_node(nodelist[i]))
1.210 - if nodelist[-1][0] == token.COMMA:
1.211 + if nodelist[-1][0] == token["COMMA"]:
1.212 return Print(items, dest, lineno=nodelist[0][2])
1.213 return Printnl(items, dest, lineno=nodelist[0][2])
1.214
1.215 @@ -445,17 +444,17 @@
1.216 while nodelist[idx][1] == '.':
1.217 idx += 1
1.218 level = idx - 1
1.219 - if nodelist[idx][0] == symbol.dotted_name:
1.220 + if nodelist[idx][0] == symbol["dotted_name"]:
1.221 fromname = self.com_dotted_name(nodelist[idx])
1.222 idx += 1
1.223 else:
1.224 fromname = ""
1.225 assert nodelist[idx][1] == 'import'
1.226 - if nodelist[idx + 1][0] == token.STAR:
1.227 + if nodelist[idx + 1][0] == token["STAR"]:
1.228 return From(fromname, [('*', None)], level,
1.229 lineno=nodelist[0][2])
1.230 else:
1.231 - node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token.LPAR)]
1.232 + node = nodelist[idx + 1 + (nodelist[idx + 1][0] == token["LPAR"])]
1.233 return From(fromname, self.com_import_as_names(node), level,
1.234 lineno=nodelist[0][2])
1.235
1.236 @@ -545,7 +544,7 @@
1.237
1.238 stmts = []
1.239 for node in nodelist:
1.240 - if node[0] == symbol.stmt:
1.241 + if node[0] == symbol["stmt"]:
1.242 self.com_append_stmt(stmts, node)
1.243 return Stmt(stmts)
1.244
1.245 @@ -566,15 +565,15 @@
1.246
1.247 def testlist_comp(self, nodelist):
1.248 # test ( comp_for | (',' test)* [','] )
1.249 - assert nodelist[0][0] == symbol.test
1.250 - if len(nodelist) == 2 and nodelist[1][0] == symbol.comp_for:
1.251 + assert nodelist[0][0] == symbol["test"]
1.252 + if len(nodelist) == 2 and nodelist[1][0] == symbol["comp_for"]:
1.253 test = self.com_node(nodelist[0])
1.254 return self.com_generator_expression(test, nodelist[1])
1.255 return self.testlist(nodelist)
1.256
1.257 def test(self, nodelist):
1.258 # or_test ['if' or_test 'else' test] | lambdef
1.259 - if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.260 + if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]:
1.261 return self.lambdef(nodelist[0])
1.262 then = self.com_node(nodelist[0])
1.263 if len(nodelist) > 1:
1.264 @@ -588,7 +587,7 @@
1.265
1.266 def or_test(self, nodelist):
1.267 # and_test ('or' and_test)* | lambdef
1.268 - if len(nodelist) == 1 and nodelist[0][0] == symbol.lambdef:
1.269 + if len(nodelist) == 1 and nodelist[0][0] == symbol["lambdef"]:
1.270 return self.lambdef(nodelist[0])
1.271 return self.com_binary(Or, nodelist)
1.272 old_test = or_test
1.273 @@ -617,7 +616,7 @@
1.274 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.275 # | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.276 n = nl[1]
1.277 - if n[0] == token.NAME:
1.278 + if n[0] == token["NAME"]:
1.279 type = n[1]
1.280 if len(nl) == 3:
1.281 if type == 'not':
1.282 @@ -654,9 +653,9 @@
1.283 node = self.com_node(nodelist[0])
1.284 for i in range(2, len(nodelist), 2):
1.285 right = self.com_node(nodelist[i])
1.286 - if nodelist[i-1][0] == token.LEFTSHIFT:
1.287 + if nodelist[i-1][0] == token["LEFTSHIFT"]:
1.288 node = LeftShift([node, right], lineno=nodelist[1][2])
1.289 - elif nodelist[i-1][0] == token.RIGHTSHIFT:
1.290 + elif nodelist[i-1][0] == token["RIGHTSHIFT"]:
1.291 node = RightShift([node, right], lineno=nodelist[1][2])
1.292 else:
1.293 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.294 @@ -666,9 +665,9 @@
1.295 node = self.com_node(nodelist[0])
1.296 for i in range(2, len(nodelist), 2):
1.297 right = self.com_node(nodelist[i])
1.298 - if nodelist[i-1][0] == token.PLUS:
1.299 + if nodelist[i-1][0] == token["PLUS"]:
1.300 node = Add([node, right], lineno=nodelist[1][2])
1.301 - elif nodelist[i-1][0] == token.MINUS:
1.302 + elif nodelist[i-1][0] == token["MINUS"]:
1.303 node = Sub([node, right], lineno=nodelist[1][2])
1.304 else:
1.305 raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
1.306 @@ -679,13 +678,13 @@
1.307 for i in range(2, len(nodelist), 2):
1.308 right = self.com_node(nodelist[i])
1.309 t = nodelist[i-1][0]
1.310 - if t == token.STAR:
1.311 + if t == token["STAR"]:
1.312 node = Mul([node, right])
1.313 - elif t == token.SLASH:
1.314 + elif t == token["SLASH"]:
1.315 node = Div([node, right])
1.316 - elif t == token.PERCENT:
1.317 + elif t == token["PERCENT"]:
1.318 node = Mod([node, right])
1.319 - elif t == token.DOUBLESLASH:
1.320 + elif t == token["DOUBLESLASH"]:
1.321 node = FloorDiv([node, right])
1.322 else:
1.323 raise ValueError, "unexpected token: %s" % t
1.324 @@ -697,11 +696,11 @@
1.325 t = elt[0]
1.326 node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
1.327 # need to handle (unary op)constant here...
1.328 - if t == token.PLUS:
1.329 + if t == token["PLUS"]:
1.330 return UnaryAdd(node, lineno=elt[2])
1.331 - elif t == token.MINUS:
1.332 + elif t == token["MINUS"]:
1.333 return UnarySub(node, lineno=elt[2])
1.334 - elif t == token.TILDE:
1.335 + elif t == token["TILDE"]:
1.336 node = Invert(node, lineno=elt[2])
1.337 return node
1.338
1.339 @@ -710,7 +709,7 @@
1.340 node = self.com_node(nodelist[0])
1.341 for i in range(1, len(nodelist)):
1.342 elt = nodelist[i]
1.343 - if elt[0] == token.DOUBLESTAR:
1.344 + if elt[0] == token["DOUBLESTAR"]:
1.345 return Power([node, self.com_node(nodelist[i+1])],
1.346 lineno=elt[2])
1.347
1.348 @@ -722,17 +721,17 @@
1.349 return self._atom_dispatch[nodelist[0][0]](nodelist)
1.350
1.351 def atom_lpar(self, nodelist):
1.352 - if nodelist[1][0] == token.RPAR:
1.353 + if nodelist[1][0] == token["RPAR"]:
1.354 return Tuple((), lineno=nodelist[0][2])
1.355 return self.com_node(nodelist[1])
1.356
1.357 def atom_lsqb(self, nodelist):
1.358 - if nodelist[1][0] == token.RSQB:
1.359 + if nodelist[1][0] == token["RSQB"]:
1.360 return List((), lineno=nodelist[0][2])
1.361 return self.com_list_constructor(nodelist[1])
1.362
1.363 def atom_lbrace(self, nodelist):
1.364 - if nodelist[1][0] == token.RBRACE:
1.365 + if nodelist[1][0] == token["RBRACE"]:
1.366 return Dict((), lineno=nodelist[0][2])
1.367 return self.com_dictorsetmaker(nodelist[1])
1.368
1.369 @@ -806,10 +805,10 @@
1.370 i = 0
1.371 while i < len(nodelist):
1.372 node = nodelist[i]
1.373 - if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
1.374 - if node[0] == token.STAR:
1.375 + if node[0] == token["STAR"] or node[0] == token["DOUBLESTAR"]:
1.376 + if node[0] == token["STAR"]:
1.377 node = nodelist[i+1]
1.378 - if node[0] == token.NAME:
1.379 + if node[0] == token["NAME"]:
1.380 names.append(node[1])
1.381 flags = flags | CO_VARARGS
1.382 i = i + 3
1.383 @@ -817,7 +816,7 @@
1.384 if i < len(nodelist):
1.385 # should be DOUBLESTAR
1.386 t = nodelist[i][0]
1.387 - if t == token.DOUBLESTAR:
1.388 + if t == token["DOUBLESTAR"]:
1.389 node = nodelist[i+1]
1.390 else:
1.391 raise ValueError, "unexpected token: %s" % t
1.392 @@ -830,7 +829,7 @@
1.393 names.append(self.com_fpdef(node))
1.394
1.395 i = i + 1
1.396 - if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
1.397 + if i < len(nodelist) and nodelist[i][0] == token["EQUAL"]:
1.398 defaults.append(self.com_node(nodelist[i + 1]))
1.399 i = i + 2
1.400 elif len(defaults):
1.401 @@ -845,7 +844,7 @@
1.402
1.403 def com_fpdef(self, node):
1.404 # fpdef: NAME | '(' fplist ')'
1.405 - if node[1][0] == token.LPAR:
1.406 + if node[1][0] == token["LPAR"]:
1.407 return self.com_fplist(node[2])
1.408 return node[1][1]
1.409
1.410 @@ -867,17 +866,17 @@
1.411 return name[:-1]
1.412
1.413 def com_dotted_as_name(self, node):
1.414 - assert node[0] == symbol.dotted_as_name
1.415 + assert node[0] == symbol["dotted_as_name"]
1.416 node = node[1:]
1.417 dot = self.com_dotted_name(node[0][1:])
1.418 if len(node) == 1:
1.419 return dot, None
1.420 assert node[1][1] == 'as'
1.421 - assert node[2][0] == token.NAME
1.422 + assert node[2][0] == token["NAME"]
1.423 return dot, node[2][1]
1.424
1.425 def com_dotted_as_names(self, node):
1.426 - assert node[0] == symbol.dotted_as_names
1.427 + assert node[0] == symbol["dotted_as_names"]
1.428 node = node[1:]
1.429 names = [self.com_dotted_as_name(node[0])]
1.430 for i in range(2, len(node), 2):
1.431 @@ -885,17 +884,17 @@
1.432 return names
1.433
1.434 def com_import_as_name(self, node):
1.435 - assert node[0] == symbol.import_as_name
1.436 + assert node[0] == symbol["import_as_name"]
1.437 node = node[1:]
1.438 - assert node[0][0] == token.NAME
1.439 + assert node[0][0] == token["NAME"]
1.440 if len(node) == 1:
1.441 return node[0][1], None
1.442 assert node[1][1] == 'as', node
1.443 - assert node[2][0] == token.NAME
1.444 + assert node[2][0] == token["NAME"]
1.445 return node[0][1], node[2][1]
1.446
1.447 def com_import_as_names(self, node):
1.448 - assert node[0] == symbol.import_as_names
1.449 + assert node[0] == symbol["import_as_names"]
1.450 node = node[1:]
1.451 names = [self.com_import_as_name(node[0])]
1.452 for i in range(2, len(node), 2):
1.453 @@ -913,7 +912,7 @@
1.454 # ((except_clause ':' suite)+ ['else' ':' suite] ['finally' ':' suite]
1.455 # | 'finally' ':' suite))
1.456
1.457 - if nodelist[3][0] == token.NAME:
1.458 + if nodelist[3][0] == token["NAME"]:
1.459 # first clause is a finally clause: only try-finally
1.460 return TryFinally(self.com_node(nodelist[2]),
1.461 self.com_node(nodelist[5]),
1.462 @@ -925,7 +924,7 @@
1.463 finallyNode = None
1.464 for i in range(3, len(nodelist), 3):
1.465 node = nodelist[i]
1.466 - if node[0] == symbol.except_clause:
1.467 + if node[0] == symbol["except_clause"]:
1.468 # except_clause: 'except' [expr [(',' | 'as') expr]] */
1.469 if len(node) > 2:
1.470 expr1 = self.com_node(node[2])
1.471 @@ -937,7 +936,7 @@
1.472 expr1 = expr2 = None
1.473 clauses.append((expr1, expr2, self.com_node(nodelist[i+2])))
1.474
1.475 - if node[0] == token.NAME:
1.476 + if node[0] == token["NAME"]:
1.477 if node[1] == 'else':
1.478 elseNode = self.com_node(nodelist[i+2])
1.479 elif node[1] == 'finally':
1.480 @@ -968,7 +967,7 @@
1.481 return With(expr, var, body, lineno=lineno)
1.482
1.483 def com_augassign_op(self, node):
1.484 - assert node[0] == symbol.augassign
1.485 + assert node[0] == symbol["augassign"]
1.486 return node[1]
1.487
1.488 def com_augassign(self, node):
1.489 @@ -986,7 +985,7 @@
1.490 # loop to avoid trivial recursion
1.491 while 1:
1.492 t = node[0]
1.493 - if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_comp):
1.494 + if t in (symbol["exprlist"], symbol["testlist"], symbol["testlist_safe"], symbol["testlist_comp"]):
1.495 if len(node) > 2:
1.496 return self.com_assign_tuple(node, assigning)
1.497 node = node[1]
1.498 @@ -994,31 +993,31 @@
1.499 if len(node) > 2:
1.500 raise SyntaxError, "can't assign to operator"
1.501 node = node[1]
1.502 - elif t == symbol.power:
1.503 - if node[1][0] != symbol.atom:
1.504 + elif t == symbol["power"]:
1.505 + if node[1][0] != symbol["atom"]:
1.506 raise SyntaxError, "can't assign to operator"
1.507 if len(node) > 2:
1.508 primary = self.com_node(node[1])
1.509 for i in range(2, len(node)-1):
1.510 ch = node[i]
1.511 - if ch[0] == token.DOUBLESTAR:
1.512 + if ch[0] == token["DOUBLESTAR"]:
1.513 raise SyntaxError, "can't assign to operator"
1.514 primary = self.com_apply_trailer(primary, ch)
1.515 return self.com_assign_trailer(primary, node[-1],
1.516 assigning)
1.517 node = node[1]
1.518 - elif t == symbol.atom:
1.519 + elif t == symbol["atom"]:
1.520 t = node[1][0]
1.521 - if t == token.LPAR:
1.522 + if t == token["LPAR"]:
1.523 node = node[2]
1.524 - if node[0] == token.RPAR:
1.525 + if node[0] == token["RPAR"]:
1.526 raise SyntaxError, "can't assign to ()"
1.527 - elif t == token.LSQB:
1.528 + elif t == token["LSQB"]:
1.529 node = node[2]
1.530 - if node[0] == token.RSQB:
1.531 + if node[0] == token["RSQB"]:
1.532 raise SyntaxError, "can't assign to []"
1.533 return self.com_assign_list(node, assigning)
1.534 - elif t == token.NAME:
1.535 + elif t == token["NAME"]:
1.536 return self.com_assign_name(node[1], assigning)
1.537 else:
1.538 raise SyntaxError, "can't assign to literal"
1.539 @@ -1035,9 +1034,9 @@
1.540 assigns = []
1.541 for i in range(1, len(node), 2):
1.542 if i + 1 < len(node):
1.543 - if node[i + 1][0] == symbol.list_for:
1.544 + if node[i + 1][0] == symbol["list_for"]:
1.545 raise SyntaxError, "can't assign to list comprehension"
1.546 - assert node[i + 1][0] == token.COMMA, node[i + 1]
1.547 + assert node[i + 1][0] == token["COMMA"], node[i + 1]
1.548 assigns.append(self.com_assign(node[i], assigning))
1.549 return AssList(assigns, lineno=extractLineNo(node))
1.550
1.551 @@ -1046,11 +1045,11 @@
1.552
1.553 def com_assign_trailer(self, primary, node, assigning):
1.554 t = node[1][0]
1.555 - if t == token.DOT:
1.556 + if t == token["DOT"]:
1.557 return self.com_assign_attr(primary, node[2], assigning)
1.558 - if t == token.LSQB:
1.559 + if t == token["LSQB"]:
1.560 return self.com_subscriptlist(primary, node[2], assigning)
1.561 - if t == token.LPAR:
1.562 + if t == token["LPAR"]:
1.563 raise SyntaxError, "can't assign to function call"
1.564 raise SyntaxError, "unknown trailer type: %s" % t
1.565
1.566 @@ -1088,11 +1087,11 @@
1.567 # listmaker: test ( list_for | (',' test)* [','] )
1.568 values = []
1.569 for i in range(1, len(nodelist)):
1.570 - if nodelist[i][0] == symbol.list_for:
1.571 + if nodelist[i][0] == symbol["list_for"]:
1.572 assert len(nodelist[i:]) == 1
1.573 return self.com_list_comprehension(values[0],
1.574 nodelist[i])
1.575 - elif nodelist[i][0] == token.COMMA:
1.576 + elif nodelist[i][0] == token["COMMA"]:
1.577 continue
1.578 values.append(self.com_node(nodelist[i]))
1.579 return List(values, lineno=values[0].lineno)
1.580 @@ -1150,11 +1149,11 @@
1.581 raise ValueError("unexpected comprehension type: " + repr(type))
1.582
1.583 def com_list_iter(self, node):
1.584 - assert node[0] == symbol.list_iter
1.585 + assert node[0] == symbol["list_iter"]
1.586 return node[1]
1.587
1.588 def com_comp_iter(self, node):
1.589 - assert node[0] == symbol.comp_iter
1.590 + assert node[0] == symbol["comp_iter"]
1.591 return node[1]
1.592
1.593 def com_generator_expression(self, expr, node):
1.594 @@ -1194,21 +1193,21 @@
1.595 def com_dictorsetmaker(self, nodelist):
1.596 # dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
1.597 # (test (comp_for | (',' test)* [','])) )
1.598 - assert nodelist[0] == symbol.dictorsetmaker
1.599 + assert nodelist[0] == symbol["dictorsetmaker"]
1.600 nodelist = nodelist[1:]
1.601 - if len(nodelist) == 1 or nodelist[1][0] == token.COMMA:
1.602 + if len(nodelist) == 1 or nodelist[1][0] == token["COMMA"]:
1.603 # set literal
1.604 items = []
1.605 for i in range(0, len(nodelist), 2):
1.606 items.append(self.com_node(nodelist[i]))
1.607 return Set(items, lineno=items[0].lineno)
1.608 - elif nodelist[1][0] == symbol.comp_for:
1.609 + elif nodelist[1][0] == symbol["comp_for"]:
1.610 # set comprehension
1.611 expr = self.com_node(nodelist[0])
1.612 return self.com_comprehension(expr, None, nodelist[1], 'set')
1.613 - elif len(nodelist) > 3 and nodelist[3][0] == symbol.comp_for:
1.614 + elif len(nodelist) > 3 and nodelist[3][0] == symbol["comp_for"]:
1.615 # dict comprehension
1.616 - assert nodelist[1][0] == token.COLON
1.617 + assert nodelist[1][0] == token["COLON"]
1.618 key = self.com_node(nodelist[0])
1.619 value = self.com_node(nodelist[2])
1.620 return self.com_comprehension(key, value, nodelist[3], 'dict')
1.621 @@ -1222,22 +1221,22 @@
1.622
1.623 def com_apply_trailer(self, primaryNode, nodelist):
1.624 t = nodelist[1][0]
1.625 - if t == token.LPAR:
1.626 + if t == token["LPAR"]:
1.627 return self.com_call_function(primaryNode, nodelist[2])
1.628 - if t == token.DOT:
1.629 + if t == token["DOT"]:
1.630 return self.com_select_member(primaryNode, nodelist[2])
1.631 - if t == token.LSQB:
1.632 + if t == token["LSQB"]:
1.633 return self.com_subscriptlist(primaryNode, nodelist[2], OP_APPLY)
1.634
1.635 raise SyntaxError, 'unknown node type: %s' % t
1.636
1.637 def com_select_member(self, primaryNode, nodelist):
1.638 - if nodelist[0] != token.NAME:
1.639 + if nodelist[0] != token["NAME"]:
1.640 raise SyntaxError, "member must be a name"
1.641 return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
1.642
1.643 def com_call_function(self, primaryNode, nodelist):
1.644 - if nodelist[0] == token.RPAR:
1.645 + if nodelist[0] == token["RPAR"]:
1.646 return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
1.647 args = []
1.648 kw = 0
1.649 @@ -1247,13 +1246,13 @@
1.650 while i < len_nodelist:
1.651 node = nodelist[i]
1.652
1.653 - if node[0]==token.STAR:
1.654 + if node[0]==token["STAR"]:
1.655 if star_node is not None:
1.656 raise SyntaxError, 'already have the varargs indentifier'
1.657 star_node = self.com_node(nodelist[i+1])
1.658 i = i + 3
1.659 continue
1.660 - elif node[0]==token.DOUBLESTAR:
1.661 + elif node[0]==token["DOUBLESTAR"]:
1.662 if dstar_node is not None:
1.663 raise SyntaxError, 'already have the kwargs indentifier'
1.664 dstar_node = self.com_node(nodelist[i+1])
1.665 @@ -1264,7 +1263,7 @@
1.666 kw, result = self.com_argument(node, kw, star_node)
1.667
1.668 if len_nodelist != 2 and isinstance(result, GenExpr) \
1.669 - and len(node) == 3 and node[2][0] == symbol.comp_for:
1.670 + and len(node) == 3 and node[2][0] == symbol["comp_for"]:
1.671 # allow f(x for x in y), but reject f(x for x in y, 1)
1.672 # should use f((x for x in y), 1) instead of f(x for x in y, 1)
1.673 raise SyntaxError, 'generator expression needs parenthesis'
1.674 @@ -1276,7 +1275,7 @@
1.675 lineno=extractLineNo(nodelist))
1.676
1.677 def com_argument(self, nodelist, kw, star_node):
1.678 - if len(nodelist) == 3 and nodelist[2][0] == symbol.comp_for:
1.679 + if len(nodelist) == 3 and nodelist[2][0] == symbol["comp_for"]:
1.680 test = self.com_node(nodelist[1])
1.681 return 0, self.com_generator_expression(test, nodelist[2])
1.682 if len(nodelist) == 2:
1.683 @@ -1287,9 +1286,9 @@
1.684 return 0, self.com_node(nodelist[1])
1.685 result = self.com_node(nodelist[3])
1.686 n = nodelist[1]
1.687 - while len(n) == 2 and n[0] != token.NAME:
1.688 + while len(n) == 2 and n[0] != token["NAME"]:
1.689 n = n[1]
1.690 - if n[0] != token.NAME:
1.691 + if n[0] != token["NAME"]:
1.692 raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
1.693 node = Keyword(n[1], result, lineno=n[2])
1.694 return 1, node
1.695 @@ -1303,9 +1302,9 @@
1.696 # backwards compat slice for '[i:j]'
1.697 if len(nodelist) == 2:
1.698 sub = nodelist[1]
1.699 - if (sub[1][0] == token.COLON or \
1.700 - (len(sub) > 2 and sub[2][0] == token.COLON)) and \
1.701 - sub[-1][0] != symbol.sliceop:
1.702 + if (sub[1][0] == token["COLON"] or \
1.703 + (len(sub) > 2 and sub[2][0] == token["COLON"])) and \
1.704 + sub[-1][0] != symbol["sliceop"]:
1.705 return self.com_slice(primary, sub, assigning)
1.706
1.707 subscripts = []
1.708 @@ -1318,9 +1317,9 @@
1.709 # slice_item: expression | proper_slice | ellipsis
1.710 ch = node[1]
1.711 t = ch[0]
1.712 - if t == token.DOT and node[2][0] == token.DOT:
1.713 + if t == token["DOT"] and node[2][0] == token["DOT"]:
1.714 return Ellipsis()
1.715 - if t == token.COLON or len(node) > 2:
1.716 + if t == token["COLON"] or len(node) > 2:
1.717 return self.com_sliceobj(node)
1.718 return self.com_node(ch)
1.719
1.720 @@ -1336,7 +1335,7 @@
1.721
1.722 items = []
1.723
1.724 - if node[1][0] == token.COLON:
1.725 + if node[1][0] == token["COLON"]:
1.726 items.append(Const(None))
1.727 i = 2
1.728 else:
1.729 @@ -1344,7 +1343,7 @@
1.730 # i == 2 is a COLON
1.731 i = 3
1.732
1.733 - if i < len(node) and node[i][0] == symbol.test:
1.734 + if i < len(node) and node[i][0] == symbol["test"]:
1.735 items.append(self.com_node(node[i]))
1.736 i = i + 1
1.737 else:
1.738 @@ -1364,7 +1363,7 @@
1.739 # short_slice: [lower_bound] ":" [upper_bound]
1.740 lower = upper = None
1.741 if len(node) == 3:
1.742 - if node[1][0] == token.COLON:
1.743 + if node[1][0] == token["COLON"]:
1.744 upper = self.com_node(node[2])
1.745 else:
1.746 lower = self.com_node(node[1])
1.747 @@ -1378,27 +1377,27 @@
1.748 if n is None:
1.749 n = node[0]
1.750 node = node[1:]
1.751 - if n == symbol.suite:
1.752 + if n == symbol["suite"]:
1.753 if len(node) == 1:
1.754 return self.get_docstring(node[0])
1.755 for sub in node:
1.756 - if sub[0] == symbol.stmt:
1.757 + if sub[0] == symbol["stmt"]:
1.758 return self.get_docstring(sub)
1.759 return None
1.760 - if n == symbol.file_input:
1.761 + if n == symbol["file_input"]:
1.762 for sub in node:
1.763 - if sub[0] == symbol.stmt:
1.764 + if sub[0] == symbol["stmt"]:
1.765 return self.get_docstring(sub)
1.766 return None
1.767 - if n == symbol.atom:
1.768 - if node[0][0] == token.STRING:
1.769 + if n == symbol["atom"]:
1.770 + if node[0][0] == token["STRING"]:
1.771 s = ''
1.772 for t in node:
1.773 s = s + eval(t[1])
1.774 return s
1.775 return None
1.776 - if n == symbol.stmt or n == symbol.simple_stmt \
1.777 - or n == symbol.small_stmt:
1.778 + if n == symbol["stmt"] or n == symbol["simple_stmt"] \
1.779 + or n == symbol["small_stmt"]:
1.780 return self.get_docstring(node[0])
1.781 if n in _doc_nodes and len(node) == 1:
1.782 return self.get_docstring(node[0])
1.783 @@ -1406,101 +1405,101 @@
1.784
1.785
1.786 _doc_nodes = [
1.787 - symbol.expr_stmt,
1.788 - symbol.testlist,
1.789 - symbol.testlist_safe,
1.790 - symbol.test,
1.791 - symbol.or_test,
1.792 - symbol.and_test,
1.793 - symbol.not_test,
1.794 - symbol.comparison,
1.795 - symbol.expr,
1.796 - symbol.xor_expr,
1.797 - symbol.and_expr,
1.798 - symbol.shift_expr,
1.799 - symbol.arith_expr,
1.800 - symbol.term,
1.801 - symbol.factor,
1.802 - symbol.power,
1.803 + symbol["expr_stmt"],
1.804 + symbol["testlist"],
1.805 + symbol["testlist_safe"],
1.806 + symbol["test"],
1.807 + symbol["or_test"],
1.808 + symbol["and_test"],
1.809 + symbol["not_test"],
1.810 + symbol["comparison"],
1.811 + symbol["expr"],
1.812 + symbol["xor_expr"],
1.813 + symbol["and_expr"],
1.814 + symbol["shift_expr"],
1.815 + symbol["arith_expr"],
1.816 + symbol["term"],
1.817 + symbol["factor"],
1.818 + symbol["power"],
1.819 ]
1.820
1.821 # comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
1.822 # | 'in' | 'not' 'in' | 'is' | 'is' 'not'
1.823 _cmp_types = {
1.824 - token.LESS : '<',
1.825 - token.GREATER : '>',
1.826 - token.EQEQUAL : '==',
1.827 - token.EQUAL : '==',
1.828 - token.LESSEQUAL : '<=',
1.829 - token.GREATEREQUAL : '>=',
1.830 - token.NOTEQUAL : '!=',
1.831 + token["LESS"] : '<',
1.832 + token["GREATER"] : '>',
1.833 + token["EQEQUAL"] : '==',
1.834 + token["EQUAL"] : '==',
1.835 + token["LESSEQUAL"] : '<=',
1.836 + token["GREATEREQUAL"] : '>=',
1.837 + token["NOTEQUAL"] : '!=',
1.838 }
1.839
1.840 _legal_node_types = [
1.841 - symbol.funcdef,
1.842 - symbol.classdef,
1.843 - symbol.stmt,
1.844 - symbol.small_stmt,
1.845 - symbol.flow_stmt,
1.846 - symbol.simple_stmt,
1.847 - symbol.compound_stmt,
1.848 - symbol.expr_stmt,
1.849 - symbol.print_stmt,
1.850 - symbol.del_stmt,
1.851 - symbol.pass_stmt,
1.852 - symbol.break_stmt,
1.853 - symbol.continue_stmt,
1.854 - symbol.return_stmt,
1.855 - symbol.raise_stmt,
1.856 - symbol.import_stmt,
1.857 - symbol.global_stmt,
1.858 - symbol.exec_stmt,
1.859 - symbol.assert_stmt,
1.860 - symbol.if_stmt,
1.861 - symbol.while_stmt,
1.862 - symbol.for_stmt,
1.863 - symbol.try_stmt,
1.864 - symbol.with_stmt,
1.865 - symbol.suite,
1.866 - symbol.testlist,
1.867 - symbol.testlist_safe,
1.868 - symbol.test,
1.869 - symbol.and_test,
1.870 - symbol.not_test,
1.871 - symbol.comparison,
1.872 - symbol.exprlist,
1.873 - symbol.expr,
1.874 - symbol.xor_expr,
1.875 - symbol.and_expr,
1.876 - symbol.shift_expr,
1.877 - symbol.arith_expr,
1.878 - symbol.term,
1.879 - symbol.factor,
1.880 - symbol.power,
1.881 - symbol.atom,
1.882 - symbol.yield_stmt,
1.883 - symbol.yield_expr,
1.884 + symbol["funcdef"],
1.885 + symbol["classdef"],
1.886 + symbol["stmt"],
1.887 + symbol["small_stmt"],
1.888 + symbol["flow_stmt"],
1.889 + symbol["simple_stmt"],
1.890 + symbol["compound_stmt"],
1.891 + symbol["expr_stmt"],
1.892 + symbol["print_stmt"],
1.893 + symbol["del_stmt"],
1.894 + symbol["pass_stmt"],
1.895 + symbol["break_stmt"],
1.896 + symbol["continue_stmt"],
1.897 + symbol["return_stmt"],
1.898 + symbol["raise_stmt"],
1.899 + symbol["import_stmt"],
1.900 + symbol["global_stmt"],
1.901 + symbol["exec_stmt"],
1.902 + symbol["assert_stmt"],
1.903 + symbol["if_stmt"],
1.904 + symbol["while_stmt"],
1.905 + symbol["for_stmt"],
1.906 + symbol["try_stmt"],
1.907 + symbol["with_stmt"],
1.908 + symbol["suite"],
1.909 + symbol["testlist"],
1.910 + symbol["testlist_safe"],
1.911 + symbol["test"],
1.912 + symbol["and_test"],
1.913 + symbol["not_test"],
1.914 + symbol["comparison"],
1.915 + symbol["exprlist"],
1.916 + symbol["expr"],
1.917 + symbol["xor_expr"],
1.918 + symbol["and_expr"],
1.919 + symbol["shift_expr"],
1.920 + symbol["arith_expr"],
1.921 + symbol["term"],
1.922 + symbol["factor"],
1.923 + symbol["power"],
1.924 + symbol["atom"],
1.925 + symbol["yield_stmt"],
1.926 + symbol["yield_expr"],
1.927 ]
1.928
1.929 _assign_types = [
1.930 - symbol.test,
1.931 - symbol.or_test,
1.932 - symbol.and_test,
1.933 - symbol.not_test,
1.934 - symbol.comparison,
1.935 - symbol.expr,
1.936 - symbol.xor_expr,
1.937 - symbol.and_expr,
1.938 - symbol.shift_expr,
1.939 - symbol.arith_expr,
1.940 - symbol.term,
1.941 - symbol.factor,
1.942 + symbol["test"],
1.943 + symbol["or_test"],
1.944 + symbol["and_test"],
1.945 + symbol["not_test"],
1.946 + symbol["comparison"],
1.947 + symbol["expr"],
1.948 + symbol["xor_expr"],
1.949 + symbol["and_expr"],
1.950 + symbol["shift_expr"],
1.951 + symbol["arith_expr"],
1.952 + symbol["term"],
1.953 + symbol["factor"],
1.954 ]
1.955
1.956 _names = {}
1.957 -for k, v in symbol.sym_name.items():
1.958 +for k, v in sym_name.items():
1.959 _names[k] = v
1.960 -for k, v in token.tok_name.items():
1.961 +for k, v in tok_name.items():
1.962 _names[k] = v
1.963
1.964 def debug_tree(tree):
4.1 --- a/pyparser/pytokenizer.py Sun Jan 08 21:15:30 2017 +0100
4.2 +++ b/pyparser/pytokenizer.py Sun Jan 08 23:28:19 2017 +0100
4.3 @@ -103,7 +103,7 @@
4.4 endmatch = endDFA.recognize(line)
4.5 if endmatch >= 0:
4.6 pos = end = endmatch
4.7 - tok = (tokens.STRING, contstr + line[:end], strstart[0],
4.8 + tok = (tokens["STRING"], contstr + line[:end], strstart[0],
4.9 strstart[1], line)
4.10 token_list.append(tok)
4.11 last_comment = ''
4.12 @@ -111,7 +111,7 @@
4.13 contline = None
4.14 elif (needcont and not line.endswith('\\\n') and
4.15 not line.endswith('\\\r\n')):
4.16 - tok = (tokens.ERRORTOKEN, contstr + line, strstart[0],
4.17 + tok = (tokens["ERRORTOKEN"], contstr + line, strstart[0],
4.18 strstart[1], line)
4.19 token_list.append(tok)
4.20 last_comment = ''
4.21 @@ -140,11 +140,11 @@
4.22
4.23 if column > indents[-1]: # count indents or dedents
4.24 indents.append(column)
4.25 - token_list.append((tokens.INDENT, line[:pos], lnum, 0, line))
4.26 + token_list.append((tokens["INDENT"], line[:pos], lnum, 0, line))
4.27 last_comment = ''
4.28 while column < indents[-1]:
4.29 indents = indents[:-1]
4.30 - token_list.append((tokens.DEDENT, '', lnum, pos, line))
4.31 + token_list.append((tokens["DEDENT"], '', lnum, pos, line))
4.32 last_comment = ''
4.33 if column != indents[-1]:
4.34 err = "unindent does not match any outer indentation level"
4.35 @@ -177,11 +177,11 @@
4.36 token, initial = line[start:end], line[start]
4.37 if initial in numchars or \
4.38 (initial == '.' and token != '.'): # ordinary number
4.39 - token_list.append((tokens.NUMBER, token, lnum, start, line))
4.40 + token_list.append((tokens["NUMBER"], token, lnum, start, line))
4.41 last_comment = ''
4.42 elif initial in '\r\n':
4.43 if parenlev <= 0:
4.44 - tok = (tokens.NEWLINE, last_comment, lnum, start, line)
4.45 + tok = (tokens["NEWLINE"], last_comment, lnum, start, line)
4.46 token_list.append(tok)
4.47 last_comment = ''
4.48 elif initial == '#':
4.49 @@ -193,7 +193,7 @@
4.50 if endmatch >= 0: # all on one line
4.51 pos = endmatch
4.52 token = line[start:pos]
4.53 - tok = (tokens.STRING, token, lnum, start, line)
4.54 + tok = (tokens["STRING"], token, lnum, start, line)
4.55 token_list.append(tok)
4.56 last_comment = ''
4.57 else:
4.58 @@ -212,11 +212,11 @@
4.59 contline = line
4.60 break
4.61 else: # ordinary string
4.62 - tok = (tokens.STRING, token, lnum, start, line)
4.63 + tok = (tokens["STRING"], token, lnum, start, line)
4.64 token_list.append(tok)
4.65 last_comment = ''
4.66 elif initial in namechars: # ordinary name
4.67 - token_list.append((tokens.NAME, token, lnum, start, line))
4.68 + token_list.append((tokens["NAME"], token, lnum, start, line))
4.69 last_comment = ''
4.70 elif initial == '\\': # continued stmt
4.71 continued = 1
4.72 @@ -233,7 +233,7 @@
4.73 if token in python_opmap:
4.74 punct = python_opmap[token]
4.75 else:
4.76 - punct = tokens.OP
4.77 + punct = tokens["OP"]
4.78 token_list.append((punct, token, lnum, start, line))
4.79 last_comment = ''
4.80 else:
4.81 @@ -243,22 +243,22 @@
4.82 if start<max and line[start] in single_quoted:
4.83 raise TokenError("EOL while scanning string literal",
4.84 line, lnum, start+1, token_list)
4.85 - tok = (tokens.ERRORTOKEN, line[pos], lnum, pos, line)
4.86 + tok = (tokens["ERRORTOKEN"], line[pos], lnum, pos, line)
4.87 token_list.append(tok)
4.88 last_comment = ''
4.89 pos = pos + 1
4.90
4.91 lnum -= 1
4.92 if not (flags & consts.PyCF_DONT_IMPLY_DEDENT):
4.93 - if token_list and token_list[-1][0] != tokens.NEWLINE:
4.94 - tok = (tokens.NEWLINE, '', lnum, 0, '\n')
4.95 + if token_list and token_list[-1][0] != tokens["NEWLINE"]:
4.96 + tok = (tokens["NEWLINE"], '', lnum, 0, '\n')
4.97 token_list.append(tok)
4.98 for indent in indents[1:]: # pop remaining indent levels
4.99 - token_list.append((tokens.DEDENT, '', lnum, pos, line))
4.100 - tok = (tokens.NEWLINE, '', lnum, 0, '\n')
4.101 + token_list.append((tokens["DEDENT"], '', lnum, pos, line))
4.102 + tok = (tokens["NEWLINE"], '', lnum, 0, '\n')
4.103 token_list.append(tok)
4.104
4.105 - token_list.append((tokens.ENDMARKER, '', lnum, pos, line))
4.106 + token_list.append((tokens["ENDMARKER"], '', lnum, pos, line))
4.107 return token_list
4.108
4.109