src/third_party/cython/src/Cython/Compiler/ParseTreeTransforms.py

   1 import cython
   2 cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
   3                Options=object, UtilNodes=object, LetNode=object,
   4                LetRefNode=object, TreeFragment=object, EncodedString=object,
   5                error=object, warning=object, copy=object)
   6
   7 import PyrexTypes
   8 import Naming
   9 import ExprNodes
  10 import Nodes
  11 import Options
  12 import Builtin
  13
  14 from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
  15 from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
  16 from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
  17 from Cython.Compiler.TreeFragment import TreeFragment
  18 from Cython.Compiler.StringEncoding import EncodedString
  19 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
  20 from Cython.Compiler.Code import UtilityCode
  21
  22 import copy
  23
  24
  25 class NameNodeCollector(TreeVisitor):
  26     """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
  27     attribute.
  28     """
  29     def __init__(self):
  30         super(NameNodeCollector, self).__init__()
  31         self.name_nodes = []
  32
  33     def visit_NameNode(self, node):
  34         self.name_nodes.append(node)
  35
  36     def visit_Node(self, node):
  37         self._visitchildren(node, None)
  38
  39
  40 class SkipDeclarations(object):
  41     """
  42     Variable and function declarations can often have a deep tree structure,
  43     and yet most transformations don't need to descend to this depth.
  44
  45     Declaration nodes are removed after AnalyseDeclarationsTransform, so there
  46     is no need to use this for transformations after that point.
  47     """
  48     def visit_CTypeDefNode(self, node):
  49         return node
  50
  51     def visit_CVarDefNode(self, node):
  52         return node
  53
  54     def visit_CDeclaratorNode(self, node):
  55         return node
  56
  57     def visit_CBaseTypeNode(self, node):
  58         return node
  59
  60     def visit_CEnumDefNode(self, node):
  61         return node
  62
  63     def visit_CStructOrUnionDefNode(self, node):
  64         return node
  65
  66 class NormalizeTree(CythonTransform):
  67     """
  68     This transform fixes up a few things after parsing
  69     in order to make the parse tree more suitable for
  70     transforms.
  71
  72     a) After parsing, blocks with only one statement will
  73     be represented by that statement, not by a StatListNode.
  74     When doing transforms this is annoying and inconsistent,
  75     as one cannot in general remove a statement in a consistent
  76     way and so on. This transform wraps any single statements
  77     in a StatListNode containing a single statement.
  78
  79     b) The PassStatNode is a noop and serves no purpose beyond
  80     plugging such one-statement blocks; i.e., once parsed a
  81 `    "pass" can just as well be represented using an empty
  82     StatListNode. This means less special cases to worry about
  83     in subsequent transforms (one always checks to see if a
  84     StatListNode has no children to see if the block is empty).
  85     """
  86
  87     def __init__(self, context):
  88         super(NormalizeTree, self).__init__(context)
  89         self.is_in_statlist = False
  90         self.is_in_expr = False
  91
  92     def visit_ExprNode(self, node):
  93         stacktmp = self.is_in_expr
  94         self.is_in_expr = True
  95         self.visitchildren(node)
  96         self.is_in_expr = stacktmp
  97         return node
  98
  99     def visit_StatNode(self, node, is_listcontainer=False):
 100         stacktmp = self.is_in_statlist
 101         self.is_in_statlist = is_listcontainer
 102         self.visitchildren(node)
 103         self.is_in_statlist = stacktmp
 104         if not self.is_in_statlist and not self.is_in_expr:
 105             return Nodes.StatListNode(pos=node.pos, stats=[node])
 106         else:
 107             return node
 108
 109     def visit_StatListNode(self, node):
 110         self.is_in_statlist = True
 111         self.visitchildren(node)
 112         self.is_in_statlist = False
 113         return node
 114
 115     def visit_ParallelAssignmentNode(self, node):
 116         return self.visit_StatNode(node, True)
 117
 118     def visit_CEnumDefNode(self, node):
 119         return self.visit_StatNode(node, True)
 120
 121     def visit_CStructOrUnionDefNode(self, node):
 122         return self.visit_StatNode(node, True)
 123
 124     def visit_PassStatNode(self, node):
 125         """Eliminate PassStatNode"""
 126         if not self.is_in_statlist:
 127             return Nodes.StatListNode(pos=node.pos, stats=[])
 128         else:
 129             return []
 130
 131     def visit_ExprStatNode(self, node):
 132         """Eliminate useless string literals"""
 133         if node.expr.is_string_literal:
 134             return self.visit_PassStatNode(node)
 135         else:
 136             return self.visit_StatNode(node)
 137
 138     def visit_CDeclaratorNode(self, node):
 139         return node
 140
 141
 142 class PostParseError(CompileError): pass
 143
 144 # error strings checked by unit tests, so define them
 145 ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
 146 ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
 147 ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
 148 class PostParse(ScopeTrackingTransform):
 149     """
 150     Basic interpretation of the parse tree, as well as validity
 151     checking that can be done on a very basic level on the parse
 152     tree (while still not being a problem with the basic syntax,
 153     as such).
 154
 155     Specifically:
 156     - Default values to cdef assignments are turned into single
 157     assignments following the declaration (everywhere but in class
 158     bodies, where they raise a compile error)
 159
 160     - Interpret some node structures into Python runtime values.
 161     Some nodes take compile-time arguments (currently:
 162     TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
 163     which should be interpreted. This happens in a general way
 164     and other steps should be taken to ensure validity.
 165
 166     Type arguments cannot be interpreted in this way.
 167
 168     - For __cythonbufferdefaults__ the arguments are checked for
 169     validity.
 170
 171     TemplatedTypeNode has its directives interpreted:
 172     Any first positional argument goes into the "dtype" attribute,
 173     any "ndim" keyword argument goes into the "ndim" attribute and
 174     so on. Also it is checked that the directive combination is valid.
 175     - __cythonbufferdefaults__ attributes are parsed and put into the
 176     type information.
 177
 178     Note: Currently Parsing.py does a lot of interpretation and
 179     reorganization that can be refactored into this transform
 180     if a more pure Abstract Syntax Tree is wanted.
 181     """
 182
 183     def __init__(self, context):
 184         super(PostParse, self).__init__(context)
 185         self.specialattribute_handlers = {
 186             '__cythonbufferdefaults__' : self.handle_bufferdefaults
 187         }
 188
 189     def visit_ModuleNode(self, node):
 190         self.lambda_counter = 1
 191         self.genexpr_counter = 1
 192         return super(PostParse, self).visit_ModuleNode(node)
 193
 194     def visit_LambdaNode(self, node):
 195         # unpack a lambda expression into the corresponding DefNode
 196         lambda_id = self.lambda_counter
 197         self.lambda_counter += 1
 198         node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
 199         collector = YieldNodeCollector()
 200         collector.visitchildren(node.result_expr)
 201         if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
 202             body = Nodes.ExprStatNode(
 203                 node.result_expr.pos, expr=node.result_expr)
 204         else:
 205             body = Nodes.ReturnStatNode(
 206                 node.result_expr.pos, value=node.result_expr)
 207         node.def_node = Nodes.DefNode(
 208             node.pos, name=node.name, lambda_name=node.lambda_name,
 209             args=node.args, star_arg=node.star_arg,
 210             starstar_arg=node.starstar_arg,
 211             body=body, doc=None)
 212         self.visitchildren(node)
 213         return node
 214
 215     def visit_GeneratorExpressionNode(self, node):
 216         # unpack a generator expression into the corresponding DefNode
 217         genexpr_id = self.genexpr_counter
 218         self.genexpr_counter += 1
 219         node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
 220
 221         node.def_node = Nodes.DefNode(node.pos, name=node.name,
 222                                       doc=None,
 223                                       args=[], star_arg=None,
 224                                       starstar_arg=None,
 225                                       body=node.loop)
 226         self.visitchildren(node)
 227         return node
 228
 229     # cdef variables
 230     def handle_bufferdefaults(self, decl):
 231         if not isinstance(decl.default, ExprNodes.DictNode):
 232             raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
 233         self.scope_node.buffer_defaults_node = decl.default
 234         self.scope_node.buffer_defaults_pos = decl.pos
 235
 236     def visit_CVarDefNode(self, node):
 237         # This assumes only plain names and pointers are assignable on
 238         # declaration. Also, it makes use of the fact that a cdef decl
 239         # must appear before the first use, so we don't have to deal with
 240         # "i = 3; cdef int i = i" and can simply move the nodes around.
 241         try:
 242             self.visitchildren(node)
 243             stats = [node]
 244             newdecls = []
 245             for decl in node.declarators:
 246                 declbase = decl
 247                 while isinstance(declbase, Nodes.CPtrDeclaratorNode):
 248                     declbase = declbase.base
 249                 if isinstance(declbase, Nodes.CNameDeclaratorNode):
 250                     if declbase.default is not None:
 251                         if self.scope_type in ('cclass', 'pyclass', 'struct'):
 252                             if isinstance(self.scope_node, Nodes.CClassDefNode):
 253                                 handler = self.specialattribute_handlers.get(decl.name)
 254                                 if handler:
 255                                     if decl is not declbase:
 256                                         raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
 257                                     handler(decl)
 258                                     continue # Remove declaration
 259                             raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
 260                         first_assignment = self.scope_type != 'module'
 261                         stats.append(Nodes.SingleAssignmentNode(node.pos,
 262                             lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
 263                             rhs=declbase.default, first=first_assignment))
 264                         declbase.default = None
 265                 newdecls.append(decl)
 266             node.declarators = newdecls
 267             return stats
 268         except PostParseError, e:
 269             # An error in a cdef clause is ok, simply remove the declaration
 270             # and try to move on to report more errors
 271             self.context.nonfatal_error(e)
 272             return None
 273
 274     # Split parallel assignments (a,b = b,a) into separate partial
 275     # assignments that are executed rhs-first using temps.  This
 276     # restructuring must be applied before type analysis so that known
 277     # types on rhs and lhs can be matched directly.  It is required in
 278     # the case that the types cannot be coerced to a Python type in
 279     # order to assign from a tuple.
 280
 281     def visit_SingleAssignmentNode(self, node):
 282         self.visitchildren(node)
 283         return self._visit_assignment_node(node, [node.lhs, node.rhs])
 284
 285     def visit_CascadedAssignmentNode(self, node):
 286         self.visitchildren(node)
 287         return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
 288
 289     def _visit_assignment_node(self, node, expr_list):
 290         """Flatten parallel assignments into separate single
 291         assignments or cascaded assignments.
 292         """
 293         if sum([ 1 for expr in expr_list
 294                  if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
 295             # no parallel assignments => nothing to do
 296             return node
 297
 298         expr_list_list = []
 299         flatten_parallel_assignments(expr_list, expr_list_list)
 300         temp_refs = []
 301         eliminate_rhs_duplicates(expr_list_list, temp_refs)
 302
 303         nodes = []
 304         for expr_list in expr_list_list:
 305             lhs_list = expr_list[:-1]
 306             rhs = expr_list[-1]
 307             if len(lhs_list) == 1:
 308                 node = Nodes.SingleAssignmentNode(rhs.pos,
 309                     lhs = lhs_list[0], rhs = rhs)
 310             else:
 311                 node = Nodes.CascadedAssignmentNode(rhs.pos,
 312                     lhs_list = lhs_list, rhs = rhs)
 313             nodes.append(node)
 314
 315         if len(nodes) == 1:
 316             assign_node = nodes[0]
 317         else:
 318             assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
 319
 320         if temp_refs:
 321             duplicates_and_temps = [ (temp.expression, temp)
 322                                      for temp in temp_refs ]
 323             sort_common_subsequences(duplicates_and_temps)
 324             for _, temp_ref in duplicates_and_temps[::-1]:
 325                 assign_node = LetNode(temp_ref, assign_node)
 326
 327         return assign_node
 328
 329     def _flatten_sequence(self, seq, result):
 330         for arg in seq.args:
 331             if arg.is_sequence_constructor:
 332                 self._flatten_sequence(arg, result)
 333             else:
 334                 result.append(arg)
 335         return result
 336
 337     def visit_DelStatNode(self, node):
 338         self.visitchildren(node)
 339         node.args = self._flatten_sequence(node, [])
 340         return node
 341
 342     def visit_ExceptClauseNode(self, node):
 343         if node.is_except_as:
 344             # except-as must delete NameNode target at the end
 345             del_target = Nodes.DelStatNode(
 346                 node.pos,
 347                 args=[ExprNodes.NameNode(
 348                     node.target.pos, name=node.target.name)],
 349                 ignore_nonexisting=True)
 350             node.body = Nodes.StatListNode(
 351                 node.pos,
 352                 stats=[Nodes.TryFinallyStatNode(
 353                     node.pos,
 354                     body=node.body,
 355                     finally_clause=Nodes.StatListNode(
 356                         node.pos,
 357                         stats=[del_target]))])
 358         self.visitchildren(node)
 359         return node
 360
 361
 362 def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
 363     """Replace rhs items by LetRefNodes if they appear more than once.
 364     Creates a sequence of LetRefNodes that set up the required temps
 365     and appends them to ref_node_sequence.  The input list is modified
 366     in-place.
 367     """
 368     seen_nodes = set()
 369     ref_nodes = {}
 370     def find_duplicates(node):
 371         if node.is_literal or node.is_name:
 372             # no need to replace those; can't include attributes here
 373             # as their access is not necessarily side-effect free
 374             return
 375         if node in seen_nodes:
 376             if node not in ref_nodes:
 377                 ref_node = LetRefNode(node)
 378                 ref_nodes[node] = ref_node
 379                 ref_node_sequence.append(ref_node)
 380         else:
 381             seen_nodes.add(node)
 382             if node.is_sequence_constructor:
 383                 for item in node.args:
 384                     find_duplicates(item)
 385
 386     for expr_list in expr_list_list:
 387         rhs = expr_list[-1]
 388         find_duplicates(rhs)
 389     if not ref_nodes:
 390         return
 391
 392     def substitute_nodes(node):
 393         if node in ref_nodes:
 394             return ref_nodes[node]
 395         elif node.is_sequence_constructor:
 396             node.args = list(map(substitute_nodes, node.args))
 397         return node
 398
 399     # replace nodes inside of the common subexpressions
 400     for node in ref_nodes:
 401         if node.is_sequence_constructor:
 402             node.args = list(map(substitute_nodes, node.args))
 403
 404     # replace common subexpressions on all rhs items
 405     for expr_list in expr_list_list:
 406         expr_list[-1] = substitute_nodes(expr_list[-1])
 407
 408 def sort_common_subsequences(items):
 409     """Sort items/subsequences so that all items and subsequences that
 410     an item contains appear before the item itself.  This is needed
 411     because each rhs item must only be evaluated once, so its value
 412     must be evaluated first and then reused when packing sequences
 413     that contain it.
 414
 415     This implies a partial order, and the sort must be stable to
 416     preserve the original order as much as possible, so we use a
 417     simple insertion sort (which is very fast for short sequences, the
 418     normal case in practice).
 419     """
 420     def contains(seq, x):
 421         for item in seq:
 422             if item is x:
 423                 return True
 424             elif item.is_sequence_constructor and contains(item.args, x):
 425                 return True
 426         return False
 427     def lower_than(a,b):
 428         return b.is_sequence_constructor and contains(b.args, a)
 429
 430     for pos, item in enumerate(items):
 431         key = item[1] # the ResultRefNode which has already been injected into the sequences
 432         new_pos = pos
 433         for i in xrange(pos-1, -1, -1):
 434             if lower_than(key, items[i][0]):
 435                 new_pos = i
 436         if new_pos != pos:
 437             for i in xrange(pos, new_pos, -1):
 438                 items[i] = items[i-1]
 439             items[new_pos] = item
 440
 441 def unpack_string_to_character_literals(literal):
 442     chars = []
 443     pos = literal.pos
 444     stype = literal.__class__
 445     sval = literal.value
 446     sval_type = sval.__class__
 447     for char in sval:
 448         cval = sval_type(char)
 449         chars.append(stype(pos, value=cval, constant_result=cval))
 450     return chars
 451
 452 def flatten_parallel_assignments(input, output):
 453     #  The input is a list of expression nodes, representing the LHSs
 454     #  and RHS of one (possibly cascaded) assignment statement.  For
 455     #  sequence constructors, rearranges the matching parts of both
 456     #  sides into a list of equivalent assignments between the
 457     #  individual elements.  This transformation is applied
 458     #  recursively, so that nested structures get matched as well.
 459     rhs = input[-1]
 460     if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
 461         or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
 462         output.append(input)
 463         return
 464
 465     complete_assignments = []
 466
 467     if rhs.is_sequence_constructor:
 468         rhs_args = rhs.args
 469     elif rhs.is_string_literal:
 470         rhs_args = unpack_string_to_character_literals(rhs)
 471
 472     rhs_size = len(rhs_args)
 473     lhs_targets = [ [] for _ in xrange(rhs_size) ]
 474     starred_assignments = []
 475     for lhs in input[:-1]:
 476         if not lhs.is_sequence_constructor:
 477             if lhs.is_starred:
 478                 error(lhs.pos, "starred assignment target must be in a list or tuple")
 479             complete_assignments.append(lhs)
 480             continue
 481         lhs_size = len(lhs.args)
 482         starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
 483         if starred_targets > 1:
 484             error(lhs.pos, "more than 1 starred expression in assignment")
 485             output.append([lhs,rhs])
 486             continue
 487         elif lhs_size - starred_targets > rhs_size:
 488             error(lhs.pos, "need more than %d value%s to unpack"
 489                   % (rhs_size, (rhs_size != 1) and 's' or ''))
 490             output.append([lhs,rhs])
 491             continue
 492         elif starred_targets:
 493             map_starred_assignment(lhs_targets, starred_assignments,
 494                                    lhs.args, rhs_args)
 495         elif lhs_size < rhs_size:
 496             error(lhs.pos, "too many values to unpack (expected %d, got %d)"
 497                   % (lhs_size, rhs_size))
 498             output.append([lhs,rhs])
 499             continue
 500         else:
 501             for targets, expr in zip(lhs_targets, lhs.args):
 502                 targets.append(expr)
 503
 504     if complete_assignments:
 505         complete_assignments.append(rhs)
 506         output.append(complete_assignments)
 507
 508     # recursively flatten partial assignments
 509     for cascade, rhs in zip(lhs_targets, rhs_args):
 510         if cascade:
 511             cascade.append(rhs)
 512             flatten_parallel_assignments(cascade, output)
 513
 514     # recursively flatten starred assignments
 515     for cascade in starred_assignments:
 516         if cascade[0].is_sequence_constructor:
 517             flatten_parallel_assignments(cascade, output)
 518         else:
 519             output.append(cascade)
 520
 521 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
 522     # Appends the fixed-position LHS targets to the target list that
 523     # appear left and right of the starred argument.
 524     #
 525     # The starred_assignments list receives a new tuple
 526     # (lhs_target, rhs_values_list) that maps the remaining arguments
 527     # (those that match the starred target) to a list.
 528
 529     # left side of the starred target
 530     for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
 531         if expr.is_starred:
 532             starred = i
 533             lhs_remaining = len(lhs_args) - i - 1
 534             break
 535         targets.append(expr)
 536     else:
 537         raise InternalError("no starred arg found when splitting starred assignment")
 538
 539     # right side of the starred target
 540     for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
 541                                             lhs_args[starred + 1:])):
 542         targets.append(expr)
 543
 544     # the starred target itself, must be assigned a (potentially empty) list
 545     target = lhs_args[starred].target # unpack starred node
 546     starred_rhs = rhs_args[starred:]
 547     if lhs_remaining:
 548         starred_rhs = starred_rhs[:-lhs_remaining]
 549     if starred_rhs:
 550         pos = starred_rhs[0].pos
 551     else:
 552         pos = target.pos
 553     starred_assignments.append([
 554         target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
 555
 556
 557 class PxdPostParse(CythonTransform, SkipDeclarations):
 558     """
 559     Basic interpretation/validity checking that should only be
 560     done on pxd trees.
 561
 562     A lot of this checking currently happens in the parser; but
 563     what is listed below happens here.
 564
 565     - "def" functions are let through only if they fill the
 566     getbuffer/releasebuffer slots
 567
 568     - cdef functions are let through only if they are on the
 569     top level and are declared "inline"
 570     """
 571     ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
 572     ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
 573
 574     def __call__(self, node):
 575         self.scope_type = 'pxd'
 576         return super(PxdPostParse, self).__call__(node)
 577
 578     def visit_CClassDefNode(self, node):
 579         old = self.scope_type
 580         self.scope_type = 'cclass'
 581         self.visitchildren(node)
 582         self.scope_type = old
 583         return node
 584
 585     def visit_FuncDefNode(self, node):
 586         # FuncDefNode always come with an implementation (without
 587         # an imp they are CVarDefNodes..)
 588         err = self.ERR_INLINE_ONLY
 589
 590         if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
 591             and node.name in ('__getbuffer__', '__releasebuffer__')):
 592             err = None # allow these slots
 593
 594         if isinstance(node, Nodes.CFuncDefNode):
 595             if (u'inline' in node.modifiers and
 596                 self.scope_type in ('pxd', 'cclass')):
 597                 node.inline_in_pxd = True
 598                 if node.visibility != 'private':
 599                     err = self.ERR_NOGO_WITH_INLINE % node.visibility
 600                 elif node.api:
 601                     err = self.ERR_NOGO_WITH_INLINE % 'api'
 602                 else:
 603                     err = None # allow inline function
 604             else:
 605                 err = self.ERR_INLINE_ONLY
 606
 607         if err:
 608             self.context.nonfatal_error(PostParseError(node.pos, err))
 609             return None
 610         else:
 611             return node
 612
 613 class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
 614     """
 615     After parsing, directives can be stored in a number of places:
 616     - #cython-comments at the top of the file (stored in ModuleNode)
 617     - Command-line arguments overriding these
 618     - @cython.directivename decorators
 619     - with cython.directivename: statements
 620
 621     This transform is responsible for interpreting these various sources
 622     and store the directive in two ways:
 623     - Set the directives attribute of the ModuleNode for global directives.
 624     - Use a CompilerDirectivesNode to override directives for a subtree.
 625
 626     (The first one is primarily to not have to modify with the tree
 627     structure, so that ModuleNode stay on top.)
 628
 629     The directives are stored in dictionaries from name to value in effect.
 630     Each such dictionary is always filled in for all possible directives,
 631     using default values where no value is given by the user.
 632
 633     The available directives are controlled in Options.py.
 634
 635     Note that we have to run this prior to analysis, and so some minor
 636     duplication of functionality has to occur: We manually track cimports
 637     and which names the "cython" module may have been imported to.
 638     """
 639     unop_method_nodes = {
 640         'typeof': ExprNodes.TypeofNode,
 641
 642         'operator.address': ExprNodes.AmpersandNode,
 643         'operator.dereference': ExprNodes.DereferenceNode,
 644         'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
 645         'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
 646         'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
 647         'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
 648
 649         # For backwards compatability.
 650         'address': ExprNodes.AmpersandNode,
 651     }
 652
 653     binop_method_nodes = {
 654         'operator.comma'        : ExprNodes.c_binop_constructor(','),
 655     }
 656
 657     special_methods = set(['declare', 'union', 'struct', 'typedef',
 658                            'sizeof', 'cast', 'pointer', 'compiled',
 659                            'NULL', 'fused_type', 'parallel'])
 660     special_methods.update(unop_method_nodes.keys())
 661
 662     valid_parallel_directives = set([
 663         "parallel",
 664         "prange",
 665         "threadid",
 666 #        "threadsavailable",
 667     ])
 668
 669     def __init__(self, context, compilation_directive_defaults):
 670         super(InterpretCompilerDirectives, self).__init__(context)
 671         self.compilation_directive_defaults = {}
 672         for key, value in compilation_directive_defaults.items():
 673             self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
 674         self.cython_module_names = set()
 675         self.directive_names = {}
 676         self.parallel_directives = {}
 677
 678     def check_directive_scope(self, pos, directive, scope):
 679         legal_scopes = Options.directive_scopes.get(directive, None)
 680         if legal_scopes and scope not in legal_scopes:
 681             self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
 682                                         'is not allowed in %s scope' % (directive, scope)))
 683             return False
 684         else:
 685             if (directive not in Options.directive_defaults
 686                     and directive not in Options.directive_types):
 687                 error(pos, "Invalid directive: '%s'." % (directive,))
 688             return True
 689
 690     # Set up processing and handle the cython: comments.
 691     def visit_ModuleNode(self, node):
 692         for key, value in node.directive_comments.items():
 693             if not self.check_directive_scope(node.pos, key, 'module'):
 694                 self.wrong_scope_error(node.pos, key, 'module')
 695                 del node.directive_comments[key]
 696
 697         self.module_scope = node.scope
 698
 699         directives = copy.deepcopy(Options.directive_defaults)
 700         directives.update(copy.deepcopy(self.compilation_directive_defaults))
 701         directives.update(node.directive_comments)
 702         self.directives = directives
 703         node.directives = directives
 704         node.parallel_directives = self.parallel_directives
 705         self.visitchildren(node)
 706         node.cython_module_names = self.cython_module_names
 707         return node
 708
 709     # The following four functions track imports and cimports that
 710     # begin with "cython"
 711     def is_cython_directive(self, name):
 712         return (name in Options.directive_types or
 713                 name in self.special_methods or
 714                 PyrexTypes.parse_basic_type(name))
 715
 716     def is_parallel_directive(self, full_name, pos):
 717         """
 718         Checks to see if fullname (e.g. cython.parallel.prange) is a valid
 719         parallel directive. If it is a star import it also updates the
 720         parallel_directives.
 721         """
 722         result = (full_name + ".").startswith("cython.parallel.")
 723
 724         if result:
 725             directive = full_name.split('.')
 726             if full_name == u"cython.parallel":
 727                 self.parallel_directives[u"parallel"] = u"cython.parallel"
 728             elif full_name == u"cython.parallel.*":
 729                 for name in self.valid_parallel_directives:
 730                     self.parallel_directives[name] = u"cython.parallel.%s" % name
 731             elif (len(directive) != 3 or
 732                   directive[-1] not in self.valid_parallel_directives):
 733                 error(pos, "No such directive: %s" % full_name)
 734
 735             self.module_scope.use_utility_code(
 736                 UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
 737
 738         return result
 739
 740     def visit_CImportStatNode(self, node):
 741         if node.module_name == u"cython":
 742             self.cython_module_names.add(node.as_name or u"cython")
 743         elif node.module_name.startswith(u"cython."):
 744             if node.module_name.startswith(u"cython.parallel."):
 745                 error(node.pos, node.module_name + " is not a module")
 746             if node.module_name == u"cython.parallel":
 747                 if node.as_name and node.as_name != u"cython":
 748                     self.parallel_directives[node.as_name] = node.module_name
 749                 else:
 750                     self.cython_module_names.add(u"cython")
 751                     self.parallel_directives[
 752                                     u"cython.parallel"] = node.module_name
 753                 self.module_scope.use_utility_code(
 754                     UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
 755             elif node.as_name:
 756                 self.directive_names[node.as_name] = node.module_name[7:]
 757             else:
 758                 self.cython_module_names.add(u"cython")
 759             # if this cimport was a compiler directive, we don't
 760             # want to leave the cimport node sitting in the tree
 761             return None
 762         return node
 763
 764     def visit_FromCImportStatNode(self, node):
 765         if (node.module_name == u"cython") or \
 766                node.module_name.startswith(u"cython."):
 767             submodule = (node.module_name + u".")[7:]
 768             newimp = []
 769
 770             for pos, name, as_name, kind in node.imported_names:
 771                 full_name = submodule + name
 772                 qualified_name = u"cython." + full_name
 773
 774                 if self.is_parallel_directive(qualified_name, node.pos):
 775                     # from cython cimport parallel, or
 776                     # from cython.parallel cimport parallel, prange, ...
 777                     self.parallel_directives[as_name or name] = qualified_name
 778                 elif self.is_cython_directive(full_name):
 779                     if as_name is None:
 780                         as_name = full_name
 781
 782                     self.directive_names[as_name] = full_name
 783                     if kind is not None:
 784                         self.context.nonfatal_error(PostParseError(pos,
 785                             "Compiler directive imports must be plain imports"))
 786                 else:
 787                     newimp.append((pos, name, as_name, kind))
 788
 789             if not newimp:
 790                 return None
 791
 792             node.imported_names = newimp
 793         return node
 794
 795     def visit_FromImportStatNode(self, node):
 796         if (node.module.module_name.value == u"cython") or \
 797                node.module.module_name.value.startswith(u"cython."):
 798             submodule = (node.module.module_name.value + u".")[7:]
 799             newimp = []
 800             for name, name_node in node.items:
 801                 full_name = submodule + name
 802                 qualified_name = u"cython." + full_name
 803                 if self.is_parallel_directive(qualified_name, node.pos):
 804                     self.parallel_directives[name_node.name] = qualified_name
 805                 elif self.is_cython_directive(full_name):
 806                     self.directive_names[name_node.name] = full_name
 807                 else:
 808                     newimp.append((name, name_node))
 809             if not newimp:
 810                 return None
 811             node.items = newimp
 812         return node
 813
 814     def visit_SingleAssignmentNode(self, node):
 815         if isinstance(node.rhs, ExprNodes.ImportNode):
 816             module_name = node.rhs.module_name.value
 817             is_parallel = (module_name + u".").startswith(u"cython.parallel.")
 818
 819             if module_name != u"cython" and not is_parallel:
 820                 return node
 821
 822             module_name = node.rhs.module_name.value
 823             as_name = node.lhs.name
 824
 825             node = Nodes.CImportStatNode(node.pos,
 826                                          module_name = module_name,
 827                                          as_name = as_name)
 828             node = self.visit_CImportStatNode(node)
 829         else:
 830             self.visitchildren(node)
 831
 832         return node
 833
 834     def visit_NameNode(self, node):
 835         if node.name in self.cython_module_names:
 836             node.is_cython_module = True
 837         else:
 838             node.cython_attribute = self.directive_names.get(node.name)
 839         return node
 840
 841     def try_to_parse_directives(self, node):
 842         # If node is the contents of an directive (in a with statement or
 843         # decorator), returns a list of (directivename, value) pairs.
 844         # Otherwise, returns None
 845         if isinstance(node, ExprNodes.CallNode):
 846             self.visit(node.function)
 847             optname = node.function.as_cython_attribute()
 848             if optname:
 849                 directivetype = Options.directive_types.get(optname)
 850                 if directivetype:
 851                     args, kwds = node.explicit_args_kwds()
 852                     directives = []
 853                     key_value_pairs = []
 854                     if kwds is not None and directivetype is not dict:
 855                         for keyvalue in kwds.key_value_pairs:
 856                             key, value = keyvalue
 857                             sub_optname = "%s.%s" % (optname, key.value)
 858                             if Options.directive_types.get(sub_optname):
 859                                 directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
 860                             else:
 861                                 key_value_pairs.append(keyvalue)
 862                         if not key_value_pairs:
 863                             kwds = None
 864                         else:
 865                             kwds.key_value_pairs = key_value_pairs
 866                         if directives and not kwds and not args:
 867                             return directives
 868                     directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
 869                     return directives
 870         elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
 871             self.visit(node)
 872             optname = node.as_cython_attribute()
 873             if optname:
 874                 directivetype = Options.directive_types.get(optname)
 875                 if directivetype is bool:
 876                     return [(optname, True)]
 877                 elif directivetype is None:
 878                     return [(optname, None)]
 879                 else:
 880                     raise PostParseError(
 881                         node.pos, "The '%s' directive should be used as a function call." % optname)
 882         return None
 883
 884     def try_to_parse_directive(self, optname, args, kwds, pos):
 885         directivetype = Options.directive_types.get(optname)
 886         if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
 887             return optname, Options.directive_defaults[optname]
 888         elif directivetype is bool:
 889             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
 890                 raise PostParseError(pos,
 891                     'The %s directive takes one compile-time boolean argument' % optname)
 892             return (optname, args[0].value)
 893         elif directivetype is int:
 894             if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
 895                 raise PostParseError(pos,
 896                     'The %s directive takes one compile-time integer argument' % optname)
 897             return (optname, int(args[0].value))
 898         elif directivetype is str:
 899             if kwds is not None or len(args) != 1 or not isinstance(
 900                     args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
 901                 raise PostParseError(pos,
 902                     'The %s directive takes one compile-time string argument' % optname)
 903             return (optname, str(args[0].value))
 904         elif directivetype is type:
 905             if kwds is not None or len(args) != 1:
 906                 raise PostParseError(pos,
 907                     'The %s directive takes one type argument' % optname)
 908             return (optname, args[0])
 909         elif directivetype is dict:
 910             if len(args) != 0:
 911                 raise PostParseError(pos,
 912                     'The %s directive takes no prepositional arguments' % optname)
 913             return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
 914         elif directivetype is list:
 915             if kwds and len(kwds) != 0:
 916                 raise PostParseError(pos,
 917                     'The %s directive takes no keyword arguments' % optname)
 918             return optname, [ str(arg.value) for arg in args ]
 919         elif callable(directivetype):
 920             if kwds is not None or len(args) != 1 or not isinstance(
 921                     args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
 922                 raise PostParseError(pos,
 923                     'The %s directive takes one compile-time string argument' % optname)
 924             return (optname, directivetype(optname, str(args[0].value)))
 925         else:
 926             assert False
 927
 928     def visit_with_directives(self, body, directives):
 929         olddirectives = self.directives
 930         newdirectives = copy.copy(olddirectives)
 931         newdirectives.update(directives)
 932         self.directives = newdirectives
 933         assert isinstance(body, Nodes.StatListNode), body
 934         retbody = self.visit_Node(body)
 935         directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
 936                                                  directives=newdirectives)
 937         self.directives = olddirectives
 938         return directive
 939
 940     # Handle decorators
 941     def visit_FuncDefNode(self, node):
 942         directives = self._extract_directives(node, 'function')
 943         if not directives:
 944             return self.visit_Node(node)
 945         body = Nodes.StatListNode(node.pos, stats=[node])
 946         return self.visit_with_directives(body, directives)
 947
 948     def visit_CVarDefNode(self, node):
 949         directives = self._extract_directives(node, 'function')
 950         if not directives:
 951             return node
 952         for name, value in directives.iteritems():
 953             if name == 'locals':
 954                 node.directive_locals = value
 955             elif name != 'final':
 956                 self.context.nonfatal_error(PostParseError(
 957                     node.pos,
 958                     "Cdef functions can only take cython.locals() "
 959                     "or final decorators, got %s." % name))
 960         body = Nodes.StatListNode(node.pos, stats=[node])
 961         return self.visit_with_directives(body, directives)
 962
 963     def visit_CClassDefNode(self, node):
 964         directives = self._extract_directives(node, 'cclass')
 965         if not directives:
 966             return self.visit_Node(node)
 967         body = Nodes.StatListNode(node.pos, stats=[node])
 968         return self.visit_with_directives(body, directives)
 969
 970     def visit_PyClassDefNode(self, node):
 971         directives = self._extract_directives(node, 'class')
 972         if not directives:
 973             return self.visit_Node(node)
 974         body = Nodes.StatListNode(node.pos, stats=[node])
 975         return self.visit_with_directives(body, directives)
 976
 977     def _extract_directives(self, node, scope_name):
 978         if not node.decorators:
 979             return {}
 980         # Split the decorators into two lists -- real decorators and directives
 981         directives = []
 982         realdecs = []
 983         for dec in node.decorators:
 984             new_directives = self.try_to_parse_directives(dec.decorator)
 985             if new_directives is not None:
 986                 for directive in new_directives:
 987                     if self.check_directive_scope(node.pos, directive[0], scope_name):
 988                         directives.append(directive)
 989             else:
 990                 realdecs.append(dec)
 991         if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
 992             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
 993         else:
 994             node.decorators = realdecs
 995         # merge or override repeated directives
 996         optdict = {}
 997         directives.reverse() # Decorators coming first take precedence
 998         for directive in directives:
 999             name, value = directive
1000             if name in optdict:
1001                 old_value = optdict[name]
1002                 # keywords and arg lists can be merged, everything
1003                 # else overrides completely
1004                 if isinstance(old_value, dict):
1005                     old_value.update(value)
1006                 elif isinstance(old_value, list):
1007                     old_value.extend(value)
1008                 else:
1009                     optdict[name] = value
1010             else:
1011                 optdict[name] = value
1012         return optdict
1013
1014     # Handle with statements
1015     def visit_WithStatNode(self, node):
1016         directive_dict = {}
1017         for directive in self.try_to_parse_directives(node.manager) or []:
1018             if directive is not None:
1019                 if node.target is not None:
1020                     self.context.nonfatal_error(
1021                         PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
1022                 else:
1023                     name, value = directive
1024                     if name in ('nogil', 'gil'):
1025                         # special case: in pure mode, "with nogil" spells "with cython.nogil"
1026                         node = Nodes.GILStatNode(node.pos, state = name, body = node.body)
1027                         return self.visit_Node(node)
1028                     if self.check_directive_scope(node.pos, name, 'with statement'):
1029                         directive_dict[name] = value
1030         if directive_dict:
1031             return self.visit_with_directives(node.body, directive_dict)
1032         return self.visit_Node(node)
1033
1034
1035 class ParallelRangeTransform(CythonTransform, SkipDeclarations):
1036     """
1037     Transform cython.parallel stuff. The parallel_directives come from the
1038     module node, set there by InterpretCompilerDirectives.
1039
1040         x = cython.parallel.threadavailable()   -> ParallelThreadAvailableNode
1041         with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
1042             print cython.parallel.threadid()    -> ParallelThreadIdNode
1043             for i in cython.parallel.prange(...):  -> ParallelRangeNode
1044                 ...
1045     """
1046
1047     # a list of names, maps 'cython.parallel.prange' in the code to
1048     # ['cython', 'parallel', 'prange']
1049     parallel_directive = None
1050
1051     # Indicates whether a namenode in an expression is the cython module
1052     namenode_is_cython_module = False
1053
1054     # Keep track of whether we are the context manager of a 'with' statement
1055     in_context_manager_section = False
1056
1057     # One of 'prange' or 'with parallel'. This is used to disallow closely
1058     # nested 'with parallel:' blocks
1059     state = None
1060
1061     directive_to_node = {
1062         u"cython.parallel.parallel": Nodes.ParallelWithBlockNode,
1063         # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
1064         u"cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
1065         u"cython.parallel.prange": Nodes.ParallelRangeNode,
1066     }
1067
1068     def node_is_parallel_directive(self, node):
1069         return node.name in self.parallel_directives or node.is_cython_module
1070
1071     def get_directive_class_node(self, node):
1072         """
1073         Figure out which parallel directive was used and return the associated
1074         Node class.
1075
1076         E.g. for a cython.parallel.prange() call we return ParallelRangeNode
1077         """
1078         if self.namenode_is_cython_module:
1079             directive = '.'.join(self.parallel_directive)
1080         else:
1081             directive = self.parallel_directives[self.parallel_directive[0]]
1082             directive = '%s.%s' % (directive,
1083                                    '.'.join(self.parallel_directive[1:]))
1084             directive = directive.rstrip('.')
1085
1086         cls = self.directive_to_node.get(directive)
1087         if cls is None and not (self.namenode_is_cython_module and
1088                                 self.parallel_directive[0] != 'parallel'):
1089             error(node.pos, "Invalid directive: %s" % directive)
1090
1091         self.namenode_is_cython_module = False
1092         self.parallel_directive = None
1093
1094         return cls
1095
1096     def visit_ModuleNode(self, node):
1097         """
1098         If any parallel directives were imported, copy them over and visit
1099         the AST
1100         """
1101         if node.parallel_directives:
1102             self.parallel_directives = node.parallel_directives
1103             return self.visit_Node(node)
1104
1105         # No parallel directives were imported, so they can't be used :)
1106         return node
1107
1108     def visit_NameNode(self, node):
1109         if self.node_is_parallel_directive(node):
1110             self.parallel_directive = [node.name]
1111             self.namenode_is_cython_module = node.is_cython_module
1112         return node
1113
1114     def visit_AttributeNode(self, node):
1115         self.visitchildren(node)
1116         if self.parallel_directive:
1117             self.parallel_directive.append(node.attribute)
1118         return node
1119
1120     def visit_CallNode(self, node):
1121         self.visit(node.function)
1122         if not self.parallel_directive:
1123             return node
1124
1125         # We are a parallel directive, replace this node with the
1126         # corresponding ParallelSomethingSomething node
1127
1128         if isinstance(node, ExprNodes.GeneralCallNode):
1129             args = node.positional_args.args
1130             kwargs = node.keyword_args
1131         else:
1132             args = node.args
1133             kwargs = {}
1134
1135         parallel_directive_class = self.get_directive_class_node(node)
1136         if parallel_directive_class:
1137             # Note: in case of a parallel() the body is set by
1138             # visit_WithStatNode
1139             node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)
1140
1141         return node
1142
1143     def visit_WithStatNode(self, node):
1144         "Rewrite with cython.parallel.parallel() blocks"
1145         newnode = self.visit(node.manager)
1146
1147         if isinstance(newnode, Nodes.ParallelWithBlockNode):
1148             if self.state == 'parallel with':
1149                 error(node.manager.pos,
1150                       "Nested parallel with blocks are disallowed")
1151
1152             self.state = 'parallel with'
1153             body = self.visit(node.body)
1154             self.state = None
1155
1156             newnode.body = body
1157             return newnode
1158         elif self.parallel_directive:
1159             parallel_directive_class = self.get_directive_class_node(node)
1160
1161             if not parallel_directive_class:
1162                 # There was an error, stop here and now
1163                 return None
1164
1165             if parallel_directive_class is Nodes.ParallelWithBlockNode:
1166                 error(node.pos, "The parallel directive must be called")
1167                 return None
1168
1169         node.body = self.visit(node.body)
1170         return node
1171
1172     def visit_ForInStatNode(self, node):
1173         "Rewrite 'for i in cython.parallel.prange(...):'"
1174         self.visit(node.iterator)
1175         self.visit(node.target)
1176
1177         in_prange = isinstance(node.iterator.sequence,
1178                                Nodes.ParallelRangeNode)
1179         previous_state = self.state
1180
1181         if in_prange:
1182             # This will replace the entire ForInStatNode, so copy the
1183             # attributes
1184             parallel_range_node = node.iterator.sequence
1185
1186             parallel_range_node.target = node.target
1187             parallel_range_node.body = node.body
1188             parallel_range_node.else_clause = node.else_clause
1189
1190             node = parallel_range_node
1191
1192             if not isinstance(node.target, ExprNodes.NameNode):
1193                 error(node.target.pos,
1194                       "Can only iterate over an iteration variable")
1195
1196             self.state = 'prange'
1197
1198         self.visit(node.body)
1199         self.state = previous_state
1200         self.visit(node.else_clause)
1201         return node
1202
1203     def visit(self, node):
1204         "Visit a node that may be None"
1205         if node is not None:
1206             return super(ParallelRangeTransform, self).visit(node)
1207
1208
1209 class WithTransform(CythonTransform, SkipDeclarations):
1210     def visit_WithStatNode(self, node):
1211         self.visitchildren(node, 'body')
1212         pos = node.pos
1213         body, target, manager = node.body, node.target, node.manager
1214         node.enter_call = ExprNodes.SimpleCallNode(
1215             pos, function=ExprNodes.AttributeNode(
1216                 pos, obj=ExprNodes.CloneNode(manager),
1217                 attribute=EncodedString('__enter__'),
1218                 is_special_lookup=True),
1219             args=[],
1220             is_temp=True)
1221         if target is not None:
1222             body = Nodes.StatListNode(
1223                 pos, stats = [
1224                     Nodes.WithTargetAssignmentStatNode(
1225                         pos, lhs = target,
1226                         rhs = ResultRefNode(node.enter_call),
1227                         orig_rhs = node.enter_call),
1228                     body])
1229
1230         excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
1231             ExprNodes.ExcValueNode(pos) for _ in range(3)])
1232         except_clause = Nodes.ExceptClauseNode(
1233             pos, body=Nodes.IfStatNode(
1234                 pos, if_clauses=[
1235                     Nodes.IfClauseNode(
1236                         pos, condition=ExprNodes.NotNode(
1237                             pos, operand=ExprNodes.WithExitCallNode(
1238                                 pos, with_stat=node,
1239                                 test_if_run=False,
1240                                 args=excinfo_target)),
1241                         body=Nodes.ReraiseStatNode(pos),
1242                         ),
1243                     ],
1244                 else_clause=None),
1245             pattern=None,
1246             target=None,
1247             excinfo_target=excinfo_target,
1248             )
1249
1250         node.body = Nodes.TryFinallyStatNode(
1251             pos, body=Nodes.TryExceptStatNode(
1252                 pos, body=body,
1253                 except_clauses=[except_clause],
1254                 else_clause=None,
1255                 ),
1256             finally_clause=Nodes.ExprStatNode(
1257                 pos, expr=ExprNodes.WithExitCallNode(
1258                     pos, with_stat=node,
1259                     test_if_run=True,
1260                     args=ExprNodes.TupleNode(
1261                         pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]
1262                         ))),
1263             handle_error_case=False,
1264             )
1265         return node
1266
1267     def visit_ExprNode(self, node):
1268         # With statements are never inside expressions.
1269         return node
1270
1271
1272 class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
1273     """Originally, this was the only place where decorators were
1274     transformed into the corresponding calling code.  Now, this is
1275     done directly in DefNode and PyClassDefNode to avoid reassignments
1276     to the function/class name - except for cdef class methods.  For
1277     those, the reassignment is required as methods are originally
1278     defined in the PyMethodDef struct.
1279
1280     The IndirectionNode allows DefNode to override the decorator
1281     """
1282
1283     def visit_DefNode(self, func_node):
1284         scope_type = self.scope_type
1285         func_node = self.visit_FuncDefNode(func_node)
1286         if scope_type != 'cclass' or not func_node.decorators:
1287             return func_node
1288         return self.handle_decorators(func_node, func_node.decorators,
1289                                       func_node.name)
1290
1291     def handle_decorators(self, node, decorators, name):
1292         decorator_result = ExprNodes.NameNode(node.pos, name = name)
1293         for decorator in decorators[::-1]:
1294             decorator_result = ExprNodes.SimpleCallNode(
1295                 decorator.pos,
1296                 function = decorator.decorator,
1297                 args = [decorator_result])
1298
1299         name_node = ExprNodes.NameNode(node.pos, name = name)
1300         reassignment = Nodes.SingleAssignmentNode(
1301             node.pos,
1302             lhs = name_node,
1303             rhs = decorator_result)
1304
1305         reassignment = Nodes.IndirectionNode([reassignment])
1306         node.decorator_indirection = reassignment
1307         return [node, reassignment]
1308
1309 class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
1310     """
1311     Only part of the CythonUtilityCode pipeline. Must be run before
1312     DecoratorTransform in case this is a decorator for a cdef class.
1313     It filters out @cname('my_cname') decorators and rewrites them to
1314     CnameDecoratorNodes.
1315     """
1316
1317     def handle_function(self, node):
1318         if not getattr(node, 'decorators', None):
1319             return self.visit_Node(node)
1320
1321         for i, decorator in enumerate(node.decorators):
1322             decorator = decorator.decorator
1323
1324             if (isinstance(decorator, ExprNodes.CallNode) and
1325                     decorator.function.is_name and
1326                     decorator.function.name == 'cname'):
1327                 args, kwargs = decorator.explicit_args_kwds()
1328
1329                 if kwargs:
1330                     raise AssertionError(
1331                             "cname decorator does not take keyword arguments")
1332
1333                 if len(args) != 1:
1334                     raise AssertionError(
1335                             "cname decorator takes exactly one argument")
1336
1337                 if not (args[0].is_literal and
1338                         args[0].type == Builtin.str_type):
1339                     raise AssertionError(
1340                             "argument to cname decorator must be a string literal")
1341
1342                 cname = args[0].compile_time_value(None).decode('UTF-8')
1343                 del node.decorators[i]
1344                 node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
1345                                                 cname=cname)
1346                 break
1347
1348         return self.visit_Node(node)
1349
1350     visit_FuncDefNode = handle_function
1351     visit_CClassDefNode = handle_function
1352     visit_CEnumDefNode = handle_function
1353     visit_CStructOrUnionDefNode = handle_function
1354
1355
1356 class ForwardDeclareTypes(CythonTransform):
1357
1358     def visit_CompilerDirectivesNode(self, node):
1359         env = self.module_scope
1360         old = env.directives
1361         env.directives = node.directives
1362         self.visitchildren(node)
1363         env.directives = old
1364         return node
1365
1366     def visit_ModuleNode(self, node):
1367         self.module_scope = node.scope
1368         self.module_scope.directives = node.directives
1369         self.visitchildren(node)
1370         return node
1371
1372     def visit_CDefExternNode(self, node):
1373         old_cinclude_flag = self.module_scope.in_cinclude
1374         self.module_scope.in_cinclude = 1
1375         self.visitchildren(node)
1376         self.module_scope.in_cinclude = old_cinclude_flag
1377         return node
1378
1379     def visit_CEnumDefNode(self, node):
1380         node.declare(self.module_scope)
1381         return node
1382
1383     def visit_CStructOrUnionDefNode(self, node):
1384         if node.name not in self.module_scope.entries:
1385             node.declare(self.module_scope)
1386         return node
1387
1388     def visit_CClassDefNode(self, node):
1389         if node.class_name not in self.module_scope.entries:
1390             node.declare(self.module_scope)
1391         return node
1392
1393
1394 class AnalyseDeclarationsTransform(EnvTransform):
1395
1396     basic_property = TreeFragment(u"""
1397 property NAME:
1398     def __get__(self):
1399         return ATTR
1400     def __set__(self, value):
1401         ATTR = value
1402     """, level='c_class', pipeline=[NormalizeTree(None)])
1403     basic_pyobject_property = TreeFragment(u"""
1404 property NAME:
1405     def __get__(self):
1406         return ATTR
1407     def __set__(self, value):
1408         ATTR = value
1409     def __del__(self):
1410         ATTR = None
1411     """, level='c_class', pipeline=[NormalizeTree(None)])
1412     basic_property_ro = TreeFragment(u"""
1413 property NAME:
1414     def __get__(self):
1415         return ATTR
1416     """, level='c_class', pipeline=[NormalizeTree(None)])
1417
1418     struct_or_union_wrapper = TreeFragment(u"""
1419 cdef class NAME:
1420     cdef TYPE value
1421     def __init__(self, MEMBER=None):
1422         cdef int count
1423         count = 0
1424         INIT_ASSIGNMENTS
1425         if IS_UNION and count > 1:
1426             raise ValueError, "At most one union member should be specified."
1427     def __str__(self):
1428         return STR_FORMAT % MEMBER_TUPLE
1429     def __repr__(self):
1430         return REPR_FORMAT % MEMBER_TUPLE
1431     """, pipeline=[NormalizeTree(None)])
1432
1433     init_assignment = TreeFragment(u"""
1434 if VALUE is not None:
1435     ATTR = VALUE
1436     count += 1
1437     """, pipeline=[NormalizeTree(None)])
1438
1439     fused_function = None
1440     in_lambda = 0
1441
1442     def __call__(self, root):
1443         # needed to determine if a cdef var is declared after it's used.
1444         self.seen_vars_stack = []
1445         self.fused_error_funcs = set()
1446         super_class = super(AnalyseDeclarationsTransform, self)
1447         self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
1448         return super_class.__call__(root)
1449
1450     def visit_NameNode(self, node):
1451         self.seen_vars_stack[-1].add(node.name)
1452         return node
1453
1454     def visit_ModuleNode(self, node):
1455         self.seen_vars_stack.append(set())
1456         node.analyse_declarations(self.current_env())
1457         self.visitchildren(node)
1458         self.seen_vars_stack.pop()
1459         return node
1460
1461     def visit_LambdaNode(self, node):
1462         self.in_lambda += 1
1463         node.analyse_declarations(self.current_env())
1464         self.visitchildren(node)
1465         self.in_lambda -= 1
1466         return node
1467
1468     def visit_CClassDefNode(self, node):
1469         node = self.visit_ClassDefNode(node)
1470         if node.scope and node.scope.implemented:
1471             stats = []
1472             for entry in node.scope.var_entries:
1473                 if entry.needs_property:
1474                     property = self.create_Property(entry)
1475                     property.analyse_declarations(node.scope)
1476                     self.visit(property)
1477                     stats.append(property)
1478             if stats:
1479                 node.body.stats += stats
1480         return node
1481
1482     def _handle_fused_def_decorators(self, old_decorators, env, node):
1483         """
1484         Create function calls to the decorators and reassignments to
1485         the function.
1486         """
1487         # Delete staticmethod and classmethod decorators, this is
1488         # handled directly by the fused function object.
1489         decorators = []
1490         for decorator in old_decorators:
1491             func = decorator.decorator
1492             if (not func.is_name or
1493                 func.name not in ('staticmethod', 'classmethod') or
1494                 env.lookup_here(func.name)):
1495                 # not a static or classmethod
1496                 decorators.append(decorator)
1497
1498         if decorators:
1499             transform = DecoratorTransform(self.context)
1500             def_node = node.node
1501             _, reassignments = transform.handle_decorators(
1502                 def_node, decorators, def_node.name)
1503             reassignments.analyse_declarations(env)
1504             node = [node, reassignments]
1505
1506         return node
1507
1508     def _handle_def(self, decorators, env, node):
1509         "Handle def or cpdef fused functions"
1510         # Create PyCFunction nodes for each specialization
1511         node.stats.insert(0, node.py_func)
1512         node.py_func = self.visit(node.py_func)
1513         node.update_fused_defnode_entry(env)
1514         pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func,
1515                                                          True)
1516         pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
1517         node.resulting_fused_function = pycfunc
1518         # Create assignment node for our def function
1519         node.fused_func_assignment = self._create_assignment(
1520             node.py_func, ExprNodes.CloneNode(pycfunc), env)
1521
1522         if decorators:
1523             node = self._handle_fused_def_decorators(decorators, env, node)
1524
1525         return node
1526
1527     def _create_fused_function(self, env, node):
1528         "Create a fused function for a DefNode with fused arguments"
1529         from Cython.Compiler import FusedNode
1530
1531         if self.fused_function or self.in_lambda:
1532             if self.fused_function not in self.fused_error_funcs:
1533                 if self.in_lambda:
1534                     error(node.pos, "Fused lambdas not allowed")
1535                 else:
1536                     error(node.pos, "Cannot nest fused functions")
1537
1538             self.fused_error_funcs.add(self.fused_function)
1539
1540             node.body = Nodes.PassStatNode(node.pos)
1541             for arg in node.args:
1542                 if arg.type.is_fused:
1543                     arg.type = arg.type.get_fused_types()[0]
1544
1545             return node
1546
1547         decorators = getattr(node, 'decorators', None)
1548         node = FusedNode.FusedCFuncDefNode(node, env)
1549         self.fused_function = node
1550         self.visitchildren(node)
1551         self.fused_function = None
1552         if node.py_func:
1553             node = self._handle_def(decorators, env, node)
1554
1555         return node
1556
1557     def _handle_nogil_cleanup(self, lenv, node):
1558         "Handle cleanup for 'with gil' blocks in nogil functions."
1559         if lenv.nogil and lenv.has_with_gil_block:
1560             # Acquire the GIL for cleanup in 'nogil' functions, by wrapping
1561             # the entire function body in try/finally.
1562             # The corresponding release will be taken care of by
1563             # Nodes.FuncDefNode.generate_function_definitions()
1564             node.body = Nodes.NogilTryFinallyStatNode(
1565                 node.body.pos,
1566                 body=node.body,
1567                 finally_clause=Nodes.EnsureGILNode(node.body.pos))
1568
1569     def _handle_fused(self, node):
1570         if node.is_generator and node.has_fused_arguments:
1571             node.has_fused_arguments = False
1572             error(node.pos, "Fused generators not supported")
1573             node.gbody = Nodes.StatListNode(node.pos,
1574                                             stats=[],
1575                                             body=Nodes.PassStatNode(node.pos))
1576
1577         return node.has_fused_arguments
1578
1579     def visit_FuncDefNode(self, node):
1580         """
1581         Analyse a function and its body, as that hasn't happend yet. Also
1582         analyse the directive_locals set by @cython.locals(). Then, if we are
1583         a function with fused arguments, replace the function (after it has
1584         declared itself in the symbol table!) with a FusedCFuncDefNode, and
1585         analyse its children (which are in turn normal functions). If we're a
1586         normal function, just analyse the body of the function.
1587         """
1588         env = self.current_env()
1589
1590         self.seen_vars_stack.append(set())
1591         lenv = node.local_scope
1592         node.declare_arguments(lenv)
1593
1594         for var, type_node in node.directive_locals.items():
1595             if not lenv.lookup_here(var):   # don't redeclare args
1596                 type = type_node.analyse_as_type(lenv)
1597                 if type:
1598                     lenv.declare_var(var, type, type_node.pos)
1599                 else:
1600                     error(type_node.pos, "Not a type")
1601
1602         if self._handle_fused(node):
1603             node = self._create_fused_function(env, node)
1604         else:
1605             node.body.analyse_declarations(lenv)
1606             self._handle_nogil_cleanup(lenv, node)
1607             self._super_visit_FuncDefNode(node)
1608
1609         self.seen_vars_stack.pop()
1610         return node
1611
1612     def visit_DefNode(self, node):
1613         node = self.visit_FuncDefNode(node)
1614         env = self.current_env()
1615         if (not isinstance(node, Nodes.DefNode) or
1616             node.fused_py_func or node.is_generator_body or
1617             not node.needs_assignment_synthesis(env)):
1618             return node
1619         return [node, self._synthesize_assignment(node, env)]
1620
1621     def visit_GeneratorBodyDefNode(self, node):
1622         return self.visit_FuncDefNode(node)
1623
1624     def _synthesize_assignment(self, node, env):
1625         # Synthesize assignment node and put it right after defnode
1626         genv = env
1627         while genv.is_py_class_scope or genv.is_c_class_scope:
1628             genv = genv.outer_scope
1629
1630         if genv.is_closure_scope:
1631             rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode(
1632                 node.pos, def_node=node,
1633                 pymethdef_cname=node.entry.pymethdef_cname,
1634                 code_object=ExprNodes.CodeObjectNode(node))
1635         else:
1636             binding = self.current_directives.get('binding')
1637             rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
1638
1639         if env.is_py_class_scope:
1640             rhs.binding = True
1641
1642         node.is_cyfunction = rhs.binding
1643         return self._create_assignment(node, rhs, env)
1644
1645     def _create_assignment(self, def_node, rhs, env):
1646         if def_node.decorators:
1647             for decorator in def_node.decorators[::-1]:
1648                 rhs = ExprNodes.SimpleCallNode(
1649                     decorator.pos,
1650                     function = decorator.decorator,
1651                     args = [rhs])
1652             def_node.decorators = None
1653
1654         assmt = Nodes.SingleAssignmentNode(
1655             def_node.pos,
1656             lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
1657             rhs=rhs)
1658         assmt.analyse_declarations(env)
1659         return assmt
1660
1661     def visit_ScopedExprNode(self, node):
1662         env = self.current_env()
1663         node.analyse_declarations(env)
1664         # the node may or may not have a local scope
1665         if node.has_local_scope:
1666             self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
1667             self.enter_scope(node, node.expr_scope)
1668             node.analyse_scoped_declarations(node.expr_scope)
1669             self.visitchildren(node)
1670             self.exit_scope()
1671             self.seen_vars_stack.pop()
1672         else:
1673             node.analyse_scoped_declarations(env)
1674             self.visitchildren(node)
1675         return node
1676
1677     def visit_TempResultFromStatNode(self, node):
1678         self.visitchildren(node)
1679         node.analyse_declarations(self.current_env())
1680         return node
1681
1682     def visit_CppClassNode(self, node):
1683         if node.visibility == 'extern':
1684             return None
1685         else:
1686             return self.visit_ClassDefNode(node)
1687
1688     def visit_CStructOrUnionDefNode(self, node):
1689         # Create a wrapper node if needed.
1690         # We want to use the struct type information (so it can't happen
1691         # before this phase) but also create new objects to be declared
1692         # (so it can't happen later).
1693         # Note that we don't return the original node, as it is
1694         # never used after this phase.
1695         if True: # private (default)
1696             return None
1697
1698         self_value = ExprNodes.AttributeNode(
1699             pos = node.pos,
1700             obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
1701             attribute = EncodedString(u"value"))
1702         var_entries = node.entry.type.scope.var_entries
1703         attributes = []
1704         for entry in var_entries:
1705             attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
1706                                                       obj = self_value,
1707                                                       attribute = entry.name))
1708         # __init__ assignments
1709         init_assignments = []
1710         for entry, attr in zip(var_entries, attributes):
1711             # TODO: branch on visibility
1712             init_assignments.append(self.init_assignment.substitute({
1713                     u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
1714                     u"ATTR": attr,
1715                 }, pos = entry.pos))
1716
1717         # create the class
1718         str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
1719         wrapper_class = self.struct_or_union_wrapper.substitute({
1720             u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
1721             u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
1722             u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
1723             u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
1724             u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
1725         }, pos = node.pos).stats[0]
1726         wrapper_class.class_name = node.name
1727         wrapper_class.shadow = True
1728         class_body = wrapper_class.body.stats
1729
1730         # fix value type
1731         assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
1732         class_body[0].base_type.name = node.name
1733
1734         # fix __init__ arguments
1735         init_method = class_body[1]
1736         assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
1737         arg_template = init_method.args[1]
1738         if not node.entry.type.is_struct:
1739             arg_template.kw_only = True
1740         del init_method.args[1]
1741         for entry, attr in zip(var_entries, attributes):
1742             arg = copy.deepcopy(arg_template)
1743             arg.declarator.name = entry.name
1744             init_method.args.append(arg)
1745
1746         # setters/getters
1747         for entry, attr in zip(var_entries, attributes):
1748             # TODO: branch on visibility
1749             if entry.type.is_pyobject:
1750                 template = self.basic_pyobject_property
1751             else:
1752                 template = self.basic_property
1753             property = template.substitute({
1754                     u"ATTR": attr,
1755                 }, pos = entry.pos).stats[0]
1756             property.name = entry.name
1757             wrapper_class.body.stats.append(property)
1758
1759         wrapper_class.analyse_declarations(self.current_env())
1760         return self.visit_CClassDefNode(wrapper_class)
1761
1762     # Some nodes are no longer needed after declaration
1763     # analysis and can be dropped. The analysis was performed
1764     # on these nodes in a seperate recursive process from the
1765     # enclosing function or module, so we can simply drop them.
1766     def visit_CDeclaratorNode(self, node):
1767         # necessary to ensure that all CNameDeclaratorNodes are visited.
1768         self.visitchildren(node)
1769         return node
1770
1771     def visit_CTypeDefNode(self, node):
1772         return node
1773
1774     def visit_CBaseTypeNode(self, node):
1775         return None
1776
1777     def visit_CEnumDefNode(self, node):
1778         if node.visibility == 'public':
1779             return node
1780         else:
1781             return None
1782
1783     def visit_CNameDeclaratorNode(self, node):
1784         if node.name in self.seen_vars_stack[-1]:
1785             entry = self.current_env().lookup(node.name)
1786             if (entry is None or entry.visibility != 'extern'
1787                 and not entry.scope.is_c_class_scope):
1788                 warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1789         self.visitchildren(node)
1790         return node
1791
1792     def visit_CVarDefNode(self, node):
1793         # to ensure all CNameDeclaratorNodes are visited.
1794         self.visitchildren(node)
1795         return None
1796
1797     def visit_CnameDecoratorNode(self, node):
1798         child_node = self.visit(node.node)
1799         if not child_node:
1800             return None
1801         if type(child_node) is list: # Assignment synthesized
1802             node.child_node = child_node[0]
1803             return [node] + child_node[1:]
1804         node.node = child_node
1805         return node
1806
1807     def create_Property(self, entry):
1808         if entry.visibility == 'public':
1809             if entry.type.is_pyobject:
1810                 template = self.basic_pyobject_property
1811             else:
1812                 template = self.basic_property
1813         elif entry.visibility == 'readonly':
1814             template = self.basic_property_ro
1815         property = template.substitute({
1816                 u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1817                                                  obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1818                                                  attribute=entry.name),
1819             }, pos=entry.pos).stats[0]
1820         property.name = entry.name
1821         property.doc = entry.doc
1822         return property
1823
1824
1825 class CalculateQualifiedNamesTransform(EnvTransform):
1826     """
1827     Calculate and store the '__qualname__' and the global
1828     module name on some nodes.
1829     """
1830     def visit_ModuleNode(self, node):
1831         self.module_name = self.global_scope().qualified_name
1832         self.qualified_name = []
1833         _super = super(CalculateQualifiedNamesTransform, self)
1834         self._super_visit_FuncDefNode = _super.visit_FuncDefNode
1835         self._super_visit_ClassDefNode = _super.visit_ClassDefNode
1836         self.visitchildren(node)
1837         return node
1838
1839     def _set_qualname(self, node, name=None):
1840         if name:
1841             qualname = self.qualified_name[:]
1842             qualname.append(name)
1843         else:
1844             qualname = self.qualified_name
1845         node.qualname = EncodedString('.'.join(qualname))
1846         node.module_name = self.module_name
1847         self.visitchildren(node)
1848         return node
1849
1850     def _append_entry(self, entry):
1851         if entry.is_pyglobal and not entry.is_pyclass_attr:
1852             self.qualified_name = [entry.name]
1853         else:
1854             self.qualified_name.append(entry.name)
1855
1856     def visit_ClassNode(self, node):
1857         return self._set_qualname(node, node.name)
1858
1859     def visit_PyClassNamespaceNode(self, node):
1860         # class name was already added by parent node
1861         return self._set_qualname(node)
1862
1863     def visit_PyCFunctionNode(self, node):
1864         return self._set_qualname(node, node.def_node.name)
1865
1866     def visit_FuncDefNode(self, node):
1867         orig_qualified_name = self.qualified_name[:]
1868         if getattr(node, 'name', None) == '<lambda>':
1869             self.qualified_name.append('<lambda>')
1870         else:
1871             self._append_entry(node.entry)
1872         self.qualified_name.append('<locals>')
1873         self._super_visit_FuncDefNode(node)
1874         self.qualified_name = orig_qualified_name
1875         return node
1876
1877     def visit_ClassDefNode(self, node):
1878         orig_qualified_name = self.qualified_name[:]
1879         entry = (getattr(node, 'entry', None) or             # PyClass
1880                  self.current_env().lookup_here(node.name))  # CClass
1881         self._append_entry(entry)
1882         self._super_visit_ClassDefNode(node)
1883         self.qualified_name = orig_qualified_name
1884         return node
1885
1886
1887 class AnalyseExpressionsTransform(CythonTransform):
1888
1889     def visit_ModuleNode(self, node):
1890         node.scope.infer_types()
1891         node.body = node.body.analyse_expressions(node.scope)
1892         self.visitchildren(node)
1893         return node
1894
1895     def visit_FuncDefNode(self, node):
1896         node.local_scope.infer_types()
1897         node.body = node.body.analyse_expressions(node.local_scope)
1898         self.visitchildren(node)
1899         return node
1900
1901     def visit_ScopedExprNode(self, node):
1902         if node.has_local_scope:
1903             node.expr_scope.infer_types()
1904             node = node.analyse_scoped_expressions(node.expr_scope)
1905         self.visitchildren(node)
1906         return node
1907
1908     def visit_IndexNode(self, node):
1909         """
1910         Replace index nodes used to specialize cdef functions with fused
1911         argument types with the Attribute- or NameNode referring to the
1912         function. We then need to copy over the specialization properties to
1913         the attribute or name node.
1914
1915         Because the indexing might be a Python indexing operation on a fused
1916         function, or (usually) a Cython indexing operation, we need to
1917         re-analyse the types.
1918         """
1919         self.visit_Node(node)
1920
1921         if node.is_fused_index and not node.type.is_error:
1922             node = node.base
1923         elif node.memslice_ellipsis_noop:
1924             # memoryviewslice[...] expression, drop the IndexNode
1925             node = node.base
1926
1927         return node
1928
1929
1930 class FindInvalidUseOfFusedTypes(CythonTransform):
1931
1932     def visit_FuncDefNode(self, node):
1933         # Errors related to use in functions with fused args will already
1934         # have been detected
1935         if not node.has_fused_arguments:
1936             if not node.is_generator_body and node.return_type.is_fused:
1937                 error(node.pos, "Return type is not specified as argument type")
1938             else:
1939                 self.visitchildren(node)
1940
1941         return node
1942
1943     def visit_ExprNode(self, node):
1944         if node.type and node.type.is_fused:
1945             error(node.pos, "Invalid use of fused types, type cannot be specialized")
1946         else:
1947             self.visitchildren(node)
1948
1949         return node
1950
1951
1952 class ExpandInplaceOperators(EnvTransform):
1953
1954     def visit_InPlaceAssignmentNode(self, node):
1955         lhs = node.lhs
1956         rhs = node.rhs
1957         if lhs.type.is_cpp_class:
1958             # No getting around this exact operator here.
1959             return node
1960         if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1961             # There is code to handle this case.
1962             return node
1963
1964         env = self.current_env()
1965         def side_effect_free_reference(node, setting=False):
1966             if isinstance(node, ExprNodes.NameNode):
1967                 return node, []
1968             elif node.type.is_pyobject and not setting:
1969                 node = LetRefNode(node)
1970                 return node, [node]
1971             elif isinstance(node, ExprNodes.IndexNode):
1972                 if node.is_buffer_access:
1973                     raise ValueError("Buffer access")
1974                 base, temps = side_effect_free_reference(node.base)
1975                 index = LetRefNode(node.index)
1976                 return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
1977             elif isinstance(node, ExprNodes.AttributeNode):
1978                 obj, temps = side_effect_free_reference(node.obj)
1979                 return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
1980             else:
1981                 node = LetRefNode(node)
1982                 return node, [node]
1983         try:
1984             lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
1985         except ValueError:
1986             return node
1987         dup = lhs.__class__(**lhs.__dict__)
1988         binop = ExprNodes.binop_node(node.pos,
1989                                      operator = node.operator,
1990                                      operand1 = dup,
1991                                      operand2 = rhs,
1992                                      inplace=True)
1993         # Manually analyse types for new node.
1994         lhs.analyse_target_types(env)
1995         dup.analyse_types(env)
1996         binop.analyse_operation(env)
1997         node = Nodes.SingleAssignmentNode(
1998             node.pos,
1999             lhs = lhs,
2000             rhs=binop.coerce_to(lhs.type, env))
2001         # Use LetRefNode to avoid side effects.
2002         let_ref_nodes.reverse()
2003         for t in let_ref_nodes:
2004             node = LetNode(t, node)
2005         return node
2006
2007     def visit_ExprNode(self, node):
2008         # In-place assignments can't happen within an expression.
2009         return node
2010
2011 class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
2012     """
2013     Adjust function and class definitions by the decorator directives:
2014
2015     @cython.cfunc
2016     @cython.cclass
2017     @cython.ccall
2018     """
2019
2020     def visit_ModuleNode(self, node):
2021         self.directives = node.directives
2022         self.in_py_class = False
2023         self.visitchildren(node)
2024         return node
2025
2026     def visit_CompilerDirectivesNode(self, node):
2027         old_directives = self.directives
2028         self.directives = node.directives
2029         self.visitchildren(node)
2030         self.directives = old_directives
2031         return node
2032
2033     def visit_DefNode(self, node):
2034         if 'ccall' in self.directives:
2035             node = node.as_cfunction(overridable=True, returns=self.directives.get('returns'))
2036             return self.visit(node)
2037         if 'cfunc' in self.directives:
2038             if self.in_py_class:
2039                 error(node.pos, "cfunc directive is not allowed here")
2040             else:
2041                 node = node.as_cfunction(overridable=False, returns=self.directives.get('returns'))
2042                 return self.visit(node)
2043         self.visitchildren(node)
2044         return node
2045
2046     def visit_PyClassDefNode(self, node):
2047         if 'cclass' in self.directives:
2048             node = node.as_cclass()
2049             return self.visit(node)
2050         else:
2051             old_in_pyclass = self.in_py_class
2052             self.in_py_class = True
2053             self.visitchildren(node)
2054             self.in_py_class = old_in_pyclass
2055             return node
2056
2057     def visit_CClassDefNode(self, node):
2058         old_in_pyclass = self.in_py_class
2059         self.in_py_class = False
2060         self.visitchildren(node)
2061         self.in_py_class = old_in_pyclass
2062         return node
2063
2064
2065 class AlignFunctionDefinitions(CythonTransform):
2066     """
2067     This class takes the signatures from a .pxd file and applies them to
2068     the def methods in a .py file.
2069     """
2070
2071     def visit_ModuleNode(self, node):
2072         self.scope = node.scope
2073         self.directives = node.directives
2074         self.imported_names = set()  # hack, see visit_FromImportStatNode()
2075         self.visitchildren(node)
2076         return node
2077
2078     def visit_PyClassDefNode(self, node):
2079         pxd_def = self.scope.lookup(node.name)
2080         if pxd_def:
2081             if pxd_def.is_cclass:
2082                 return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
2083             elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
2084                 error(node.pos, "'%s' redeclared" % node.name)
2085                 if pxd_def.pos:
2086                     error(pxd_def.pos, "previous declaration here")
2087                 return None
2088         return node
2089
2090     def visit_CClassDefNode(self, node, pxd_def=None):
2091         if pxd_def is None:
2092             pxd_def = self.scope.lookup(node.class_name)
2093         if pxd_def:
2094             outer_scope = self.scope
2095             self.scope = pxd_def.type.scope
2096         self.visitchildren(node)
2097         if pxd_def:
2098             self.scope = outer_scope
2099         return node
2100
2101     def visit_DefNode(self, node):
2102         pxd_def = self.scope.lookup(node.name)
2103         if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
2104             if not pxd_def.is_cfunction:
2105                 error(node.pos, "'%s' redeclared" % node.name)
2106                 if pxd_def.pos:
2107                     error(pxd_def.pos, "previous declaration here")
2108                 return None
2109             node = node.as_cfunction(pxd_def)
2110         elif (self.scope.is_module_scope and self.directives['auto_cpdef']
2111               and not node.name in self.imported_names
2112               and node.is_cdef_func_compatible()):
2113             # FIXME: cpdef-ing should be done in analyse_declarations()
2114             node = node.as_cfunction(scope=self.scope)
2115         # Enable this when nested cdef functions are allowed.
2116         # self.visitchildren(node)
2117         return node
2118
2119     def visit_FromImportStatNode(self, node):
2120         # hack to prevent conditional import fallback functions from
2121         # being cdpef-ed (global Python variables currently conflict
2122         # with imports)
2123         if self.scope.is_module_scope:
2124             for name, _ in node.items:
2125                 self.imported_names.add(name)
2126         return node
2127
2128     def visit_ExprNode(self, node):
2129         # ignore lambdas and everything else that appears in expressions
2130         return node
2131
2132
2133 class RemoveUnreachableCode(CythonTransform):
2134     def visit_StatListNode(self, node):
2135         if not self.current_directives['remove_unreachable']:
2136             return node
2137         self.visitchildren(node)
2138         for idx, stat in enumerate(node.stats):
2139             idx += 1
2140             if stat.is_terminator:
2141                 if idx < len(node.stats):
2142                     if self.current_directives['warn.unreachable']:
2143                         warning(node.stats[idx].pos, "Unreachable code", 2)
2144                     node.stats = node.stats[:idx]
2145                 node.is_terminator = True
2146                 break
2147         return node
2148
2149     def visit_IfClauseNode(self, node):
2150         self.visitchildren(node)
2151         if node.body.is_terminator:
2152             node.is_terminator = True
2153         return node
2154
2155     def visit_IfStatNode(self, node):
2156         self.visitchildren(node)
2157         if node.else_clause and node.else_clause.is_terminator:
2158             for clause in node.if_clauses:
2159                 if not clause.is_terminator:
2160                     break
2161             else:
2162                 node.is_terminator = True
2163         return node
2164
2165     def visit_TryExceptStatNode(self, node):
2166         self.visitchildren(node)
2167         if node.body.is_terminator and node.else_clause:
2168             if self.current_directives['warn.unreachable']:
2169                 warning(node.else_clause.pos, "Unreachable code", 2)
2170             node.else_clause = None
2171         return node
2172
2173
2174 class YieldNodeCollector(TreeVisitor):
2175
2176     def __init__(self):
2177         super(YieldNodeCollector, self).__init__()
2178         self.yields = []
2179         self.returns = []
2180         self.has_return_value = False
2181
2182     def visit_Node(self, node):
2183         self.visitchildren(node)
2184
2185     def visit_YieldExprNode(self, node):
2186         self.yields.append(node)
2187         self.visitchildren(node)
2188
2189     def visit_ReturnStatNode(self, node):
2190         self.visitchildren(node)
2191         if node.value:
2192             self.has_return_value = True
2193         self.returns.append(node)
2194
2195     def visit_ClassDefNode(self, node):
2196         pass
2197
2198     def visit_FuncDefNode(self, node):
2199         pass
2200
2201     def visit_LambdaNode(self, node):
2202         pass
2203
2204     def visit_GeneratorExpressionNode(self, node):
2205         pass
2206
2207
2208 class MarkClosureVisitor(CythonTransform):
2209
2210     def visit_ModuleNode(self, node):
2211         self.needs_closure = False
2212         self.visitchildren(node)
2213         return node
2214
2215     def visit_FuncDefNode(self, node):
2216         self.needs_closure = False
2217         self.visitchildren(node)
2218         node.needs_closure = self.needs_closure
2219         self.needs_closure = True
2220
2221         collector = YieldNodeCollector()
2222         collector.visitchildren(node)
2223
2224         if collector.yields:
2225             if isinstance(node, Nodes.CFuncDefNode):
2226                 # Will report error later
2227                 return node
2228             for i, yield_expr in enumerate(collector.yields):
2229                 yield_expr.label_num = i + 1  # no enumerate start arg in Py2.4
2230             for retnode in collector.returns:
2231                 retnode.in_generator = True
2232
2233             gbody = Nodes.GeneratorBodyDefNode(
2234                 pos=node.pos, name=node.name, body=node.body)
2235             generator = Nodes.GeneratorDefNode(
2236                 pos=node.pos, name=node.name, args=node.args,
2237                 star_arg=node.star_arg, starstar_arg=node.starstar_arg,
2238                 doc=node.doc, decorators=node.decorators,
2239                 gbody=gbody, lambda_name=node.lambda_name)
2240             return generator
2241         return node
2242
2243     def visit_CFuncDefNode(self, node):
2244         self.visit_FuncDefNode(node)
2245         if node.needs_closure:
2246             error(node.pos, "closures inside cdef functions not yet supported")
2247         return node
2248
2249     def visit_LambdaNode(self, node):
2250         self.needs_closure = False
2251         self.visitchildren(node)
2252         node.needs_closure = self.needs_closure
2253         self.needs_closure = True
2254         return node
2255
2256     def visit_ClassDefNode(self, node):
2257         self.visitchildren(node)
2258         self.needs_closure = True
2259         return node
2260
2261 class CreateClosureClasses(CythonTransform):
2262     # Output closure classes in module scope for all functions
2263     # that really need it.
2264
2265     def __init__(self, context):
2266         super(CreateClosureClasses, self).__init__(context)
2267         self.path = []
2268         self.in_lambda = False
2269
2270     def visit_ModuleNode(self, node):
2271         self.module_scope = node.scope
2272         self.visitchildren(node)
2273         return node
2274
2275     def find_entries_used_in_closures(self, node):
2276         from_closure = []
2277         in_closure = []
2278         for name, entry in node.local_scope.entries.items():
2279             if entry.from_closure:
2280                 from_closure.append((name, entry))
2281             elif entry.in_closure:
2282                 in_closure.append((name, entry))
2283         return from_closure, in_closure
2284
2285     def create_class_from_scope(self, node, target_module_scope, inner_node=None):
2286         # move local variables into closure
2287         if node.is_generator:
2288             for entry in node.local_scope.entries.values():
2289                 if not entry.from_closure:
2290                     entry.in_closure = True
2291
2292         from_closure, in_closure = self.find_entries_used_in_closures(node)
2293         in_closure.sort()
2294
2295         # Now from the begining
2296         node.needs_closure = False
2297         node.needs_outer_scope = False
2298
2299         func_scope = node.local_scope
2300         cscope = node.entry.scope
2301         while cscope.is_py_class_scope or cscope.is_c_class_scope:
2302             cscope = cscope.outer_scope
2303
2304         if not from_closure and (self.path or inner_node):
2305             if not inner_node:
2306                 if not node.py_cfunc_node:
2307                     raise InternalError("DefNode does not have assignment node")
2308                 inner_node = node.py_cfunc_node
2309             inner_node.needs_self_code = False
2310             node.needs_outer_scope = False
2311
2312         if node.is_generator:
2313             pass
2314         elif not in_closure and not from_closure:
2315             return
2316         elif not in_closure:
2317             func_scope.is_passthrough = True
2318             func_scope.scope_class = cscope.scope_class
2319             node.needs_outer_scope = True
2320             return
2321
2322         as_name = '%s_%s' % (
2323             target_module_scope.next_id(Naming.closure_class_prefix),
2324             node.entry.cname)
2325
2326         entry = target_module_scope.declare_c_class(
2327             name=as_name, pos=node.pos, defining=True,
2328             implementing=True)
2329         entry.type.is_final_type = True
2330
2331         func_scope.scope_class = entry
2332         class_scope = entry.type.scope
2333         class_scope.is_internal = True
2334         if Options.closure_freelist_size:
2335             class_scope.directives['freelist'] = Options.closure_freelist_size
2336
2337         if from_closure:
2338             assert cscope.is_closure_scope
2339             class_scope.declare_var(pos=node.pos,
2340                                     name=Naming.outer_scope_cname,
2341                                     cname=Naming.outer_scope_cname,
2342                                     type=cscope.scope_class.type,
2343                                     is_cdef=True)
2344             node.needs_outer_scope = True
2345         for name, entry in in_closure:
2346             closure_entry = class_scope.declare_var(pos=entry.pos,
2347                                     name=entry.name,
2348                                     cname=entry.cname,
2349                                     type=entry.type,
2350                                     is_cdef=True)
2351             if entry.is_declared_generic:
2352                 closure_entry.is_declared_generic = 1
2353         node.needs_closure = True
2354         # Do it here because other classes are already checked
2355         target_module_scope.check_c_class(func_scope.scope_class)
2356
2357     def visit_LambdaNode(self, node):
2358         if not isinstance(node.def_node, Nodes.DefNode):
2359             # fused function, an error has been previously issued
2360             return node
2361
2362         was_in_lambda = self.in_lambda
2363         self.in_lambda = True
2364         self.create_class_from_scope(node.def_node, self.module_scope, node)
2365         self.visitchildren(node)
2366         self.in_lambda = was_in_lambda
2367         return node
2368
2369     def visit_FuncDefNode(self, node):
2370         if self.in_lambda:
2371             self.visitchildren(node)
2372             return node
2373         if node.needs_closure or self.path:
2374             self.create_class_from_scope(node, self.module_scope)
2375             self.path.append(node)
2376             self.visitchildren(node)
2377             self.path.pop()
2378         return node
2379
2380     def visit_GeneratorBodyDefNode(self, node):
2381         self.visitchildren(node)
2382         return node
2383
2384     def visit_CFuncDefNode(self, node):
2385         self.visitchildren(node)
2386         return node
2387
2388
2389 class GilCheck(VisitorTransform):
2390     """
2391     Call `node.gil_check(env)` on each node to make sure we hold the
2392     GIL when we need it.  Raise an error when on Python operations
2393     inside a `nogil` environment.
2394
2395     Additionally, raise exceptions for closely nested with gil or with nogil
2396     statements. The latter would abort Python.
2397     """
2398
2399     def __call__(self, root):
2400         self.env_stack = [root.scope]
2401         self.nogil = False
2402
2403         # True for 'cdef func() nogil:' functions, as the GIL may be held while
2404         # calling this function (thus contained 'nogil' blocks may be valid).
2405         self.nogil_declarator_only = False
2406         return super(GilCheck, self).__call__(root)
2407
2408     def visit_FuncDefNode(self, node):
2409         self.env_stack.append(node.local_scope)
2410         was_nogil = self.nogil
2411         self.nogil = node.local_scope.nogil
2412
2413         if self.nogil:
2414             self.nogil_declarator_only = True
2415
2416         if self.nogil and node.nogil_check:
2417             node.nogil_check(node.local_scope)
2418
2419         self.visitchildren(node)
2420
2421         # This cannot be nested, so it doesn't need backup/restore
2422         self.nogil_declarator_only = False
2423
2424         self.env_stack.pop()
2425         self.nogil = was_nogil
2426         return node
2427
2428     def visit_GILStatNode(self, node):
2429         if self.nogil and node.nogil_check:
2430             node.nogil_check()
2431
2432         was_nogil = self.nogil
2433         self.nogil = (node.state == 'nogil')
2434
2435         if was_nogil == self.nogil and not self.nogil_declarator_only:
2436             if not was_nogil:
2437                 error(node.pos, "Trying to acquire the GIL while it is "
2438                                 "already held.")
2439             else:
2440                 error(node.pos, "Trying to release the GIL while it was "
2441                                 "previously released.")
2442
2443         if isinstance(node.finally_clause, Nodes.StatListNode):
2444             # The finally clause of the GILStatNode is a GILExitNode,
2445             # which is wrapped in a StatListNode. Just unpack that.
2446             node.finally_clause, = node.finally_clause.stats
2447
2448         self.visitchildren(node)
2449         self.nogil = was_nogil
2450         return node
2451
2452     def visit_ParallelRangeNode(self, node):
2453         if node.nogil:
2454             node.nogil = False
2455             node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
2456             return self.visit_GILStatNode(node)
2457
2458         if not self.nogil:
2459             error(node.pos, "prange() can only be used without the GIL")
2460             # Forget about any GIL-related errors that may occur in the body
2461             return None
2462
2463         node.nogil_check(self.env_stack[-1])
2464         self.visitchildren(node)
2465         return node
2466
2467     def visit_ParallelWithBlockNode(self, node):
2468         if not self.nogil:
2469             error(node.pos, "The parallel section may only be used without "
2470                             "the GIL")
2471             return None
2472
2473         if node.nogil_check:
2474             # It does not currently implement this, but test for it anyway to
2475             # avoid potential future surprises
2476             node.nogil_check(self.env_stack[-1])
2477
2478         self.visitchildren(node)
2479         return node
2480
2481     def visit_TryFinallyStatNode(self, node):
2482         """
2483         Take care of try/finally statements in nogil code sections.
2484         """
2485         if not self.nogil or isinstance(node, Nodes.GILStatNode):
2486             return self.visit_Node(node)
2487
2488         node.nogil_check = None
2489         node.is_try_finally_in_nogil = True
2490         self.visitchildren(node)
2491         return node
2492
2493     def visit_Node(self, node):
2494         if self.env_stack and self.nogil and node.nogil_check:
2495             node.nogil_check(self.env_stack[-1])
2496         self.visitchildren(node)
2497         node.in_nogil_context = self.nogil
2498         return node
2499
2500
2501 class TransformBuiltinMethods(EnvTransform):
2502
2503     def visit_SingleAssignmentNode(self, node):
2504         if node.declaration_only:
2505             return None
2506         else:
2507             self.visitchildren(node)
2508             return node
2509
2510     def visit_AttributeNode(self, node):
2511         self.visitchildren(node)
2512         return self.visit_cython_attribute(node)
2513
2514     def visit_NameNode(self, node):
2515         return self.visit_cython_attribute(node)
2516
2517     def visit_cython_attribute(self, node):
2518         attribute = node.as_cython_attribute()
2519         if attribute:
2520             if attribute == u'compiled':
2521                 node = ExprNodes.BoolNode(node.pos, value=True)
2522             elif attribute == u'__version__':
2523                 import Cython
2524                 node = ExprNodes.StringNode(node.pos, value=EncodedString(Cython.__version__))
2525             elif attribute == u'NULL':
2526                 node = ExprNodes.NullNode(node.pos)
2527             elif attribute in (u'set', u'frozenset'):
2528                 node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
2529                                           entry=self.current_env().builtin_scope().lookup_here(attribute))
2530             elif PyrexTypes.parse_basic_type(attribute):
2531                 pass
2532             elif self.context.cython_scope.lookup_qualified_name(attribute):
2533                 pass
2534             else:
2535                 error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
2536         return node
2537
2538     def visit_ExecStatNode(self, node):
2539         lenv = self.current_env()
2540         self.visitchildren(node)
2541         if len(node.args) == 1:
2542             node.args.append(ExprNodes.GlobalsExprNode(node.pos))
2543             if not lenv.is_module_scope:
2544                 node.args.append(
2545                     ExprNodes.LocalsExprNode(
2546                         node.pos, self.current_scope_node(), lenv))
2547         return node
2548
2549     def _inject_locals(self, node, func_name):
2550         # locals()/dir()/vars() builtins
2551         lenv = self.current_env()
2552         entry = lenv.lookup_here(func_name)
2553         if entry:
2554             # not the builtin
2555             return node
2556         pos = node.pos
2557         if func_name in ('locals', 'vars'):
2558             if func_name == 'locals' and len(node.args) > 0:
2559                 error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
2560                       % len(node.args))
2561                 return node
2562             elif func_name == 'vars':
2563                 if len(node.args) > 1:
2564                     error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
2565                           % len(node.args))
2566                 if len(node.args) > 0:
2567                     return node # nothing to do
2568             return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
2569         else: # dir()
2570             if len(node.args) > 1:
2571                 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
2572                       % len(node.args))
2573             if len(node.args) > 0:
2574                 # optimised in Builtin.py
2575                 return node
2576             if lenv.is_py_class_scope or lenv.is_module_scope:
2577                 if lenv.is_py_class_scope:
2578                     pyclass = self.current_scope_node()
2579                     locals_dict = ExprNodes.CloneNode(pyclass.dict)
2580                 else:
2581                     locals_dict = ExprNodes.GlobalsExprNode(pos)
2582                 return ExprNodes.SortedDictKeysNode(locals_dict)
2583             local_names = [ var.name for var in lenv.entries.values() if var.name ]
2584             items = [ ExprNodes.IdentifierStringNode(pos, value=var)
2585                       for var in local_names ]
2586             return ExprNodes.ListNode(pos, args=items)
2587
2588     def visit_PrimaryCmpNode(self, node):
2589         # special case: for in/not-in test, we do not need to sort locals()
2590         self.visitchildren(node)
2591         if node.operator in 'not_in':  # in/not_in
2592             if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
2593                 arg = node.operand2.arg
2594                 if isinstance(arg, ExprNodes.NoneCheckNode):
2595                     arg = arg.arg
2596                 node.operand2 = arg
2597         return node
2598
2599     def visit_CascadedCmpNode(self, node):
2600         return self.visit_PrimaryCmpNode(node)
2601
2602     def _inject_eval(self, node, func_name):
2603         lenv = self.current_env()
2604         entry = lenv.lookup_here(func_name)
2605         if entry or len(node.args) != 1:
2606             return node
2607         # Inject globals and locals
2608         node.args.append(ExprNodes.GlobalsExprNode(node.pos))
2609         if not lenv.is_module_scope:
2610             node.args.append(
2611                 ExprNodes.LocalsExprNode(
2612                     node.pos, self.current_scope_node(), lenv))
2613         return node
2614
2615     def _inject_super(self, node, func_name):
2616         lenv = self.current_env()
2617         entry = lenv.lookup_here(func_name)
2618         if entry or node.args:
2619             return node
2620         # Inject no-args super
2621         def_node = self.current_scope_node()
2622         if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or
2623             len(self.env_stack) < 2):
2624             return node
2625         class_node, class_scope = self.env_stack[-2]
2626         if class_scope.is_py_class_scope:
2627             def_node.requires_classobj = True
2628             class_node.class_cell.is_active = True
2629             node.args = [
2630                 ExprNodes.ClassCellNode(
2631                     node.pos, is_generator=def_node.is_generator),
2632                 ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
2633                 ]
2634         elif class_scope.is_c_class_scope:
2635             node.args = [
2636                 ExprNodes.NameNode(
2637                     node.pos, name=class_node.scope.name,
2638                     entry=class_node.entry),
2639                 ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
2640                 ]
2641         return node
2642
2643     def visit_SimpleCallNode(self, node):
2644         # cython.foo
2645         function = node.function.as_cython_attribute()
2646         if function:
2647             if function in InterpretCompilerDirectives.unop_method_nodes:
2648                 if len(node.args) != 1:
2649                     error(node.function.pos, u"%s() takes exactly one argument" % function)
2650                 else:
2651                     node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
2652             elif function in InterpretCompilerDirectives.binop_method_nodes:
2653                 if len(node.args) != 2:
2654                     error(node.function.pos, u"%s() takes exactly two arguments" % function)
2655                 else:
2656                     node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
2657             elif function == u'cast':
2658                 if len(node.args) != 2:
2659                     error(node.function.pos, u"cast() takes exactly two arguments")
2660                 else:
2661                     type = node.args[0].analyse_as_type(self.current_env())
2662                     if type:
2663                         node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
2664                     else:
2665                         error(node.args[0].pos, "Not a type")
2666             elif function == u'sizeof':
2667                 if len(node.args) != 1:
2668                     error(node.function.pos, u"sizeof() takes exactly one argument")
2669                 else:
2670                     type = node.args[0].analyse_as_type(self.current_env())
2671                     if type:
2672                         node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
2673                     else:
2674                         node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
2675             elif function == 'cmod':
2676                 if len(node.args) != 2:
2677                     error(node.function.pos, u"cmod() takes exactly two arguments")
2678                 else:
2679                     node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
2680                     node.cdivision = True
2681             elif function == 'cdiv':
2682                 if len(node.args) != 2:
2683                     error(node.function.pos, u"cdiv() takes exactly two arguments")
2684                 else:
2685                     node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
2686                     node.cdivision = True
2687             elif function == u'set':
2688                 node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
2689             elif self.context.cython_scope.lookup_qualified_name(function):
2690                 pass
2691             else:
2692                 error(node.function.pos,
2693                       u"'%s' not a valid cython language construct" % function)
2694
2695         self.visitchildren(node)
2696
2697         if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
2698             func_name = node.function.name
2699             if func_name in ('dir', 'locals', 'vars'):
2700                 return self._inject_locals(node, func_name)
2701             if func_name == 'eval':
2702                 return self._inject_eval(node, func_name)
2703             if func_name == 'super':
2704                 return self._inject_super(node, func_name)
2705         return node
2706
2707
2708 class ReplaceFusedTypeChecks(VisitorTransform):
2709     """
2710     This is not a transform in the pipeline. It is invoked on the specific
2711     versions of a cdef function with fused argument types. It filters out any
2712     type branches that don't match. e.g.
2713
2714         if fused_t is mytype:
2715             ...
2716         elif fused_t in other_fused_type:
2717             ...
2718     """
2719     def __init__(self, local_scope):
2720         super(ReplaceFusedTypeChecks, self).__init__()
2721         self.local_scope = local_scope
2722         # defer the import until now to avoid circular import time dependencies
2723         from Cython.Compiler import Optimize
2724         self.transform = Optimize.ConstantFolding(reevaluate=True)
2725
2726     def visit_IfStatNode(self, node):
2727         """
2728         Filters out any if clauses with false compile time type check
2729         expression.
2730         """
2731         self.visitchildren(node)
2732         return self.transform(node)
2733
2734     def visit_PrimaryCmpNode(self, node):
2735         type1 = node.operand1.analyse_as_type(self.local_scope)
2736         type2 = node.operand2.analyse_as_type(self.local_scope)
2737
2738         if type1 and type2:
2739             false_node = ExprNodes.BoolNode(node.pos, value=False)
2740             true_node = ExprNodes.BoolNode(node.pos, value=True)
2741
2742             type1 = self.specialize_type(type1, node.operand1.pos)
2743             op = node.operator
2744
2745             if op in ('is', 'is_not', '==', '!='):
2746                 type2 = self.specialize_type(type2, node.operand2.pos)
2747
2748                 is_same = type1.same_as(type2)
2749                 eq = op in ('is', '==')
2750
2751                 if (is_same and eq) or (not is_same and not eq):
2752                     return true_node
2753
2754             elif op in ('in', 'not_in'):
2755                 # We have to do an instance check directly, as operand2
2756                 # needs to be a fused type and not a type with a subtype
2757                 # that is fused. First unpack the typedef
2758                 if isinstance(type2, PyrexTypes.CTypedefType):
2759                     type2 = type2.typedef_base_type
2760
2761                 if type1.is_fused:
2762                     error(node.operand1.pos, "Type is fused")
2763                 elif not type2.is_fused:
2764                     error(node.operand2.pos,
2765                           "Can only use 'in' or 'not in' on a fused type")
2766                 else:
2767                     types = PyrexTypes.get_specialized_types(type2)
2768
2769                     for specialized_type in types:
2770                         if type1.same_as(specialized_type):
2771                             if op == 'in':
2772                                 return true_node
2773                             else:
2774                                 return false_node
2775
2776                     if op == 'not_in':
2777                         return true_node
2778
2779             return false_node
2780
2781         return node
2782
2783     def specialize_type(self, type, pos):
2784         try:
2785             return type.specialize(self.local_scope.fused_to_specific)
2786         except KeyError:
2787             error(pos, "Type is not specific")
2788             return type
2789
2790     def visit_Node(self, node):
2791         self.visitchildren(node)
2792         return node
2793
2794
2795 class DebugTransform(CythonTransform):
2796     """
2797     Write debug information for this Cython module.
2798     """
2799
2800     def __init__(self, context, options, result):
2801         super(DebugTransform, self).__init__(context)
2802         self.visited = set()
2803         # our treebuilder and debug output writer
2804         # (see Cython.Debugger.debug_output.CythonDebugWriter)
2805         self.tb = self.context.gdb_debug_outputwriter
2806         #self.c_output_file = options.output_file
2807         self.c_output_file = result.c_file
2808
2809         # Closure support, basically treat nested functions as if the AST were
2810         # never nested
2811         self.nested_funcdefs = []
2812
2813         # tells visit_NameNode whether it should register step-into functions
2814         self.register_stepinto = False
2815
2816     def visit_ModuleNode(self, node):
2817         self.tb.module_name = node.full_module_name
2818         attrs = dict(
2819             module_name=node.full_module_name,
2820             filename=node.pos[0].filename,
2821             c_filename=self.c_output_file)
2822
2823         self.tb.start('Module', attrs)
2824
2825         # serialize functions
2826         self.tb.start('Functions')
2827         # First, serialize functions normally...
2828         self.visitchildren(node)
2829
2830         # ... then, serialize nested functions
2831         for nested_funcdef in self.nested_funcdefs:
2832             self.visit_FuncDefNode(nested_funcdef)
2833
2834         self.register_stepinto = True
2835         self.serialize_modulenode_as_function(node)
2836         self.register_stepinto = False
2837         self.tb.end('Functions')
2838
2839         # 2.3 compatibility. Serialize global variables
2840         self.tb.start('Globals')
2841         entries = {}
2842
2843         for k, v in node.scope.entries.iteritems():
2844             if (v.qualified_name not in self.visited and not
2845                 v.name.startswith('__pyx_') and not
2846                 v.type.is_cfunction and not
2847                 v.type.is_extension_type):
2848                 entries[k]= v
2849
2850         self.serialize_local_variables(entries)
2851         self.tb.end('Globals')
2852         # self.tb.end('Module') # end Module after the line number mapping in
2853         # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
2854         return node
2855
2856     def visit_FuncDefNode(self, node):
2857         self.visited.add(node.local_scope.qualified_name)
2858
2859         if getattr(node, 'is_wrapper', False):
2860             return node
2861
2862         if self.register_stepinto:
2863             self.nested_funcdefs.append(node)
2864             return node
2865
2866         # node.entry.visibility = 'extern'
2867         if node.py_func is None:
2868             pf_cname = ''
2869         else:
2870             pf_cname = node.py_func.entry.func_cname
2871
2872         attrs = dict(
2873             name=node.entry.name or getattr(node, 'name', '<unknown>'),
2874             cname=node.entry.func_cname,
2875             pf_cname=pf_cname,
2876             qualified_name=node.local_scope.qualified_name,
2877             lineno=str(node.pos[1]))
2878
2879         self.tb.start('Function', attrs=attrs)
2880
2881         self.tb.start('Locals')
2882         self.serialize_local_variables(node.local_scope.entries)
2883         self.tb.end('Locals')
2884
2885         self.tb.start('Arguments')
2886         for arg in node.local_scope.arg_entries:
2887             self.tb.start(arg.name)
2888             self.tb.end(arg.name)
2889         self.tb.end('Arguments')
2890
2891         self.tb.start('StepIntoFunctions')
2892         self.register_stepinto = True
2893         self.visitchildren(node)
2894         self.register_stepinto = False
2895         self.tb.end('StepIntoFunctions')
2896         self.tb.end('Function')
2897
2898         return node
2899
2900     def visit_NameNode(self, node):
2901         if (self.register_stepinto and
2902             node.type.is_cfunction and
2903             getattr(node, 'is_called', False) and
2904             node.entry.func_cname is not None):
2905             # don't check node.entry.in_cinclude, as 'cdef extern: ...'
2906             # declared functions are not 'in_cinclude'.
2907             # This means we will list called 'cdef' functions as
2908             # "step into functions", but this is not an issue as they will be
2909             # recognized as Cython functions anyway.
2910             attrs = dict(name=node.entry.func_cname)
2911             self.tb.start('StepIntoFunction', attrs=attrs)
2912             self.tb.end('StepIntoFunction')
2913
2914         self.visitchildren(node)
2915         return node
2916
2917     def serialize_modulenode_as_function(self, node):
2918         """
2919         Serialize the module-level code as a function so the debugger will know
2920         it's a "relevant frame" and it will know where to set the breakpoint
2921         for 'break modulename'.
2922         """
2923         name = node.full_module_name.rpartition('.')[-1]
2924
2925         cname_py2 = 'init' + name
2926         cname_py3 = 'PyInit_' + name
2927
2928         py2_attrs = dict(
2929             name=name,
2930             cname=cname_py2,
2931             pf_cname='',
2932             # Ignore the qualified_name, breakpoints should be set using
2933             # `cy break modulename:lineno` for module-level breakpoints.
2934             qualified_name='',
2935             lineno='1',
2936             is_initmodule_function="True",
2937         )
2938
2939         py3_attrs = dict(py2_attrs, cname=cname_py3)
2940
2941         self._serialize_modulenode_as_function(node, py2_attrs)
2942         self._serialize_modulenode_as_function(node, py3_attrs)
2943
2944     def _serialize_modulenode_as_function(self, node, attrs):
2945         self.tb.start('Function', attrs=attrs)
2946
2947         self.tb.start('Locals')
2948         self.serialize_local_variables(node.scope.entries)
2949         self.tb.end('Locals')
2950
2951         self.tb.start('Arguments')
2952         self.tb.end('Arguments')
2953
2954         self.tb.start('StepIntoFunctions')
2955         self.register_stepinto = True
2956         self.visitchildren(node)
2957         self.register_stepinto = False
2958         self.tb.end('StepIntoFunctions')
2959
2960         self.tb.end('Function')
2961
2962     def serialize_local_variables(self, entries):
2963         for entry in entries.values():
2964             if not entry.cname:
2965                 # not a local variable
2966                 continue
2967             if entry.type.is_pyobject:
2968                 vartype = 'PythonObject'
2969             else:
2970                 vartype = 'CObject'
2971
2972             if entry.from_closure:
2973                 # We're dealing with a closure where a variable from an outer
2974                 # scope is accessed, get it from the scope object.
2975                 cname = '%s->%s' % (Naming.cur_scope_cname,
2976                                     entry.outer_entry.cname)
2977
2978                 qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
2979                                       entry.scope.name,
2980                                       entry.name)
2981             elif entry.in_closure:
2982                 cname = '%s->%s' % (Naming.cur_scope_cname,
2983                                     entry.cname)
2984                 qname = entry.qualified_name
2985             else:
2986                 cname = entry.cname
2987                 qname = entry.qualified_name
2988
2989             if not entry.pos:
2990                 # this happens for variables that are not in the user's code,
2991                 # e.g. for the global __builtins__, __doc__, etc. We can just
2992                 # set the lineno to 0 for those.
2993                 lineno = '0'
2994             else:
2995                 lineno = str(entry.pos[1])
2996
2997             attrs = dict(
2998                 name=entry.name,
2999                 cname=cname,
3000                 qualified_name=qname,
3001                 type=vartype,
3002                 lineno=lineno)
3003
3004             self.tb.start('LocalVar', attrs)
3005             self.tb.end('LocalVar')