• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import itertools
2from time import time
3
4import Errors
5import DebugFlags
6import Options
7from Visitor import CythonTransform
8from Errors import CompileError, InternalError, AbortError
9import Naming
10
11#
12# Really small pipeline stages
13#
14def dumptree(t):
15    # For quick debugging in pipelines
16    print t.dump()
17    return t
18
19def abort_on_errors(node):
20    # Stop the pipeline if there are any errors.
21    if Errors.num_errors != 0:
22        raise AbortError("pipeline break")
23    return node
24
25def parse_stage_factory(context):
26    def parse(compsrc):
27        source_desc = compsrc.source_desc
28        full_module_name = compsrc.full_module_name
29        initial_pos = (source_desc, 1, 0)
30        saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False
31        scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0,
32                                    check_module_name = not Options.embed)
33        Options.cimport_from_pyx = saved_cimport_from_pyx
34        tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
35        tree.compilation_source = compsrc
36        tree.scope = scope
37        tree.is_pxd = False
38        return tree
39    return parse
40
41def parse_pxd_stage_factory(context, scope, module_name):
42    def parse(source_desc):
43        tree = context.parse(source_desc, scope, pxd=True,
44                             full_module_name=module_name)
45        tree.scope = scope
46        tree.is_pxd = True
47        return tree
48    return parse
49
50def generate_pyx_code_stage_factory(options, result):
51    def generate_pyx_code_stage(module_node):
52        module_node.process_implementation(options, result)
53        result.compilation_source = module_node.compilation_source
54        return result
55    return generate_pyx_code_stage
56
57def inject_pxd_code_stage_factory(context):
58    def inject_pxd_code_stage(module_node):
59        from textwrap import dedent
60        stats = module_node.body.stats
61        for name, (statlistnode, scope) in context.pxds.iteritems():
62            module_node.merge_in(statlistnode, scope)
63        return module_node
64    return inject_pxd_code_stage
65
66def use_utility_code_definitions(scope, target, seen=None):
67    if seen is None:
68        seen = set()
69
70    for entry in scope.entries.itervalues():
71        if entry in seen:
72            continue
73
74        seen.add(entry)
75        if entry.used and entry.utility_code_definition:
76            target.use_utility_code(entry.utility_code_definition)
77            for required_utility in entry.utility_code_definition.requires:
78                target.use_utility_code(required_utility)
79        elif entry.as_module:
80            use_utility_code_definitions(entry.as_module, target, seen)
81
82def inject_utility_code_stage_factory(context):
83    def inject_utility_code_stage(module_node):
84        use_utility_code_definitions(context.cython_scope, module_node.scope)
85        added = []
86        # Note: the list might be extended inside the loop (if some utility code
87        # pulls in other utility code, explicitly or implicitly)
88        for utilcode in module_node.scope.utility_code_list:
89            if utilcode in added: continue
90            added.append(utilcode)
91            if utilcode.requires:
92                for dep in utilcode.requires:
93                    if not dep in added and not dep in module_node.scope.utility_code_list:
94                        module_node.scope.utility_code_list.append(dep)
95            tree = utilcode.get_tree()
96            if tree:
97                module_node.merge_in(tree.body, tree.scope, merge_scope=True)
98        return module_node
99    return inject_utility_code_stage
100
101class UseUtilityCodeDefinitions(CythonTransform):
102    # Temporary hack to use any utility code in nodes' "utility_code_definitions".
103    # This should be moved to the code generation phase of the relevant nodes once
104    # it is safe to generate CythonUtilityCode at code generation time.
105    def __call__(self, node):
106        self.scope = node.scope
107        return super(UseUtilityCodeDefinitions, self).__call__(node)
108
109    def process_entry(self, entry):
110        if entry:
111            for utility_code in (entry.utility_code, entry.utility_code_definition):
112                if utility_code:
113                    self.scope.use_utility_code(utility_code)
114
115    def visit_AttributeNode(self, node):
116        self.process_entry(node.entry)
117        return node
118
119    def visit_NameNode(self, node):
120        self.process_entry(node.entry)
121        self.process_entry(node.type_entry)
122        return node
123
124#
125# Pipeline factories
126#
127
128def create_pipeline(context, mode, exclude_classes=()):
129    assert mode in ('pyx', 'py', 'pxd')
130    from Visitor import PrintTree
131    from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
132    from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
133    from ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes
134    from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
135    from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
136    from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
137    from ParseTreeTransforms import CalculateQualifiedNamesTransform
138    from TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
139    from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
140    from ParseTreeTransforms import RemoveUnreachableCode, GilCheck
141    from FlowControl import ControlFlowAnalysis
142    from AnalysedTreeTransforms import AutoTestDictTransform
143    from AutoDocTransforms import EmbedSignature
144    from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
145    from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
146    from Optimize import InlineDefNodeCalls
147    from Optimize import ConstantFolding, FinalOptimizePhase
148    from Optimize import DropRefcountingTransform
149    from Optimize import ConsolidateOverflowCheck
150    from Buffer import IntroduceBufferAuxiliaryVars
151    from ModuleNode import check_c_declarations, check_c_declarations_pxd
152
153
154    if mode == 'pxd':
155        _check_c_declarations = check_c_declarations_pxd
156        _specific_post_parse = PxdPostParse(context)
157    else:
158        _check_c_declarations = check_c_declarations
159        _specific_post_parse = None
160
161    if mode == 'py':
162        _align_function_definitions = AlignFunctionDefinitions(context)
163    else:
164        _align_function_definitions = None
165
166    # NOTE: This is the "common" parts of the pipeline, which is also
167    # code in pxd files. So it will be run multiple times in a
168    # compilation stage.
169    stages = [
170        NormalizeTree(context),
171        PostParse(context),
172        _specific_post_parse,
173        InterpretCompilerDirectives(context, context.compiler_directives),
174        ParallelRangeTransform(context),
175        AdjustDefByDirectives(context),
176        MarkClosureVisitor(context),
177        _align_function_definitions,
178        RemoveUnreachableCode(context),
179        ConstantFolding(),
180        FlattenInListTransform(),
181        WithTransform(context),
182        DecoratorTransform(context),
183        ForwardDeclareTypes(context),
184        AnalyseDeclarationsTransform(context),
185        AutoTestDictTransform(context),
186        EmbedSignature(context),
187        EarlyReplaceBuiltinCalls(context),  ## Necessary?
188        TransformBuiltinMethods(context),  ## Necessary?
189        MarkParallelAssignments(context),
190        ControlFlowAnalysis(context),
191        RemoveUnreachableCode(context),
192        # MarkParallelAssignments(context),
193        MarkOverflowingArithmetic(context),
194        IntroduceBufferAuxiliaryVars(context),
195        _check_c_declarations,
196        InlineDefNodeCalls(context),
197        AnalyseExpressionsTransform(context),
198        FindInvalidUseOfFusedTypes(context),
199        ExpandInplaceOperators(context),
200        OptimizeBuiltinCalls(context),  ## Necessary?
201        CreateClosureClasses(context),  ## After all lookups and type inference
202        CalculateQualifiedNamesTransform(context),
203        ConsolidateOverflowCheck(context),
204        IterationTransform(context),
205        SwitchTransform(),
206        DropRefcountingTransform(),
207        FinalOptimizePhase(context),
208        GilCheck(),
209        UseUtilityCodeDefinitions(context),
210        ]
211    filtered_stages = []
212    for s in stages:
213        if s.__class__ not in exclude_classes:
214            filtered_stages.append(s)
215    return filtered_stages
216
217def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
218    if py:
219        mode = 'py'
220    else:
221        mode = 'pyx'
222    test_support = []
223    if options.evaluate_tree_assertions:
224        from Cython.TestUtils import TreeAssertVisitor
225        test_support.append(TreeAssertVisitor())
226
227    if options.gdb_debug:
228        from Cython.Debugger import DebugWriter # requires Py2.5+
229        from ParseTreeTransforms import DebugTransform
230        context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
231            options.output_dir)
232        debug_transform = [DebugTransform(context, options, result)]
233    else:
234        debug_transform = []
235
236    return list(itertools.chain(
237        [parse_stage_factory(context)],
238        create_pipeline(context, mode, exclude_classes=exclude_classes),
239        test_support,
240        [inject_pxd_code_stage_factory(context),
241         inject_utility_code_stage_factory(context),
242         abort_on_errors],
243        debug_transform,
244        [generate_pyx_code_stage_factory(options, result)]))
245
246def create_pxd_pipeline(context, scope, module_name):
247    from CodeGeneration import ExtractPxdCode
248
249    # The pxd pipeline ends up with a CCodeWriter containing the
250    # code of the pxd, as well as a pxd scope.
251    return [
252        parse_pxd_stage_factory(context, scope, module_name)
253        ] + create_pipeline(context, 'pxd') + [
254        ExtractPxdCode()
255        ]
256
257def create_py_pipeline(context, options, result):
258    return create_pyx_pipeline(context, options, result, py=True)
259
260def create_pyx_as_pxd_pipeline(context, result):
261    from ParseTreeTransforms import AlignFunctionDefinitions, \
262        MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform
263    from Optimize import ConstantFolding, FlattenInListTransform
264    from Nodes import StatListNode
265    pipeline = []
266    pyx_pipeline = create_pyx_pipeline(context, context.options, result,
267                                       exclude_classes=[
268                                           AlignFunctionDefinitions,
269                                           MarkClosureVisitor,
270                                           ConstantFolding,
271                                           FlattenInListTransform,
272                                           WithTransform
273                                           ])
274    for stage in pyx_pipeline:
275        pipeline.append(stage)
276        if isinstance(stage, AnalyseDeclarationsTransform):
277            # This is the last stage we need.
278            break
279    def fake_pxd(root):
280        for entry in root.scope.entries.values():
281            if not entry.in_cinclude:
282                entry.defined_in_pxd = 1
283                if entry.name == entry.cname and entry.visibility != 'extern':
284                    # Always mangle non-extern cimported entries.
285                    entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name)
286        return StatListNode(root.pos, stats=[]), root.scope
287    pipeline.append(fake_pxd)
288    return pipeline
289
290def insert_into_pipeline(pipeline, transform, before=None, after=None):
291    """
292    Insert a new transform into the pipeline after or before an instance of
293    the given class. e.g.
294
295        pipeline = insert_into_pipeline(pipeline, transform,
296                                        after=AnalyseDeclarationsTransform)
297    """
298    assert before or after
299
300    cls = before or after
301    for i, t in enumerate(pipeline):
302        if isinstance(t, cls):
303            break
304
305    if after:
306        i += 1
307
308    return pipeline[:i] + [transform] + pipeline[i:]
309
310#
311# Running a pipeline
312#
313
314def run_pipeline(pipeline, source, printtree=True):
315    from Cython.Compiler.Visitor import PrintTree
316
317    error = None
318    data = source
319    try:
320        try:
321            for phase in pipeline:
322                if phase is not None:
323                    if DebugFlags.debug_verbose_pipeline:
324                        t = time()
325                        print "Entering pipeline phase %r" % phase
326                    if not printtree and isinstance(phase, PrintTree):
327                        continue
328                    data = phase(data)
329                    if DebugFlags.debug_verbose_pipeline:
330                        print "    %.3f seconds" % (time() - t)
331        except CompileError, err:
332            # err is set
333            Errors.report_error(err)
334            error = err
335    except InternalError, err:
336        # Only raise if there was not an earlier error
337        if Errors.num_errors == 0:
338            raise
339        error = err
340    except AbortError, err:
341        error = err
342    return (error, data)
343