• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2019 The Dawn Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Module to create generators that render multiple Jinja2 templates for GN.
16
17A helper module that can be used to create generator scripts (clients)
18that expand one or more Jinja2 templates, without outputs usable from
19GN and Ninja build-based systems. See generator_lib.gni as well.
20
21Clients should create a Generator sub-class, then call run_generator()
22with a proper derived class instance.
23
24Clients specify a list of FileRender operations, each one of them will
25output a file into a temporary output directory through Jinja2 expansion.
26All temporary output files are then grouped and written to into a single JSON
27file, that acts as a convenient single GN output target. Use extract_json.py
28to extract the output files from the JSON tarball in another GN action.
29
30--depfile can be used to specify an output Ninja dependency file for the
31JSON tarball, to ensure it is regenerated any time one of its dependencies
32changes.
33
34Finally, --expected-output-files can be used to check the list of generated
35output files.
36"""
37
38import argparse, json, os, re, sys
39from collections import namedtuple
40
41# A FileRender represents a single Jinja2 template render operation:
42#
43#   template: Jinja2 template name, relative to --template-dir path.
44#
45#   output: Output file path, relative to temporary output directory.
46#
47#   params_dicts: iterable of (name:string -> value:string) dictionaries.
48#       All of them will be merged before being sent as Jinja2 template
49#       expansion parameters.
50#
51# Example:
52#   FileRender('api.c', 'src/project_api.c', [{'PROJECT_VERSION': '1.0.0'}])
53#
54FileRender = namedtuple('FileRender', ['template', 'output', 'params_dicts'])
55
56
57# The interface that must be implemented by generators.
58class Generator:
59    def get_description(self):
60        """Return generator description for --help."""
61        return ""
62
63    def add_commandline_arguments(self, parser):
64        """Add generator-specific argparse arguments."""
65        pass
66
67    def get_file_renders(self, args):
68        """Return the list of FileRender objects to process."""
69        return []
70
71    def get_dependencies(self, args):
72        """Return a list of extra input dependencies."""
73        return []
74
75
76# Allow custom Jinja2 installation path through an additional python
77# path from the arguments if present. This isn't done through the regular
78# argparse because PreprocessingLoader uses jinja2 in the global scope before
79# "main" gets to run.
80#
81# NOTE: If this argument appears several times, this only uses the first
82#       value, while argparse would typically keep the last one!
83kJinja2Path = '--jinja2-path'
84try:
85    jinja2_path_argv_index = sys.argv.index(kJinja2Path)
86    # Add parent path for the import to succeed.
87    path = os.path.join(sys.argv[jinja2_path_argv_index + 1], os.pardir)
88    sys.path.insert(1, path)
89except ValueError:
90    # --jinja2-path isn't passed, ignore the exception and just import Jinja2
91    # assuming it already is in the Python PATH.
92    pass
93
94import jinja2
95
96
97# A custom Jinja2 template loader that removes the extra indentation
98# of the template blocks so that the output is correctly indented
99class _PreprocessingLoader(jinja2.BaseLoader):
100    def __init__(self, path):
101        self.path = path
102
103    def get_source(self, environment, template):
104        path = os.path.join(self.path, template)
105        if not os.path.exists(path):
106            raise jinja2.TemplateNotFound(template)
107        mtime = os.path.getmtime(path)
108        with open(path) as f:
109            source = self.preprocess(f.read())
110        return source, path, lambda: mtime == os.path.getmtime(path)
111
112    blockstart = re.compile('{%-?\s*(if|elif|else|for|block|macro)[^}]*%}')
113    blockend = re.compile('{%-?\s*(end(if|for|block|macro)|elif|else)[^}]*%}')
114
115    def preprocess(self, source):
116        lines = source.split('\n')
117
118        # Compute the current indentation level of the template blocks and
119        # remove their indentation
120        result = []
121        indentation_level = 0
122
123        # Filter lines that are pure comments. line_comment_prefix is not
124        # enough because it removes the comment but doesn't completely remove
125        # the line, resulting in more verbose output.
126        lines = filter(lambda line: not line.strip().startswith('//*'), lines)
127
128        # Remove indentation templates have for the Jinja control flow.
129        for line in lines:
130            # The capture in the regex adds one element per block start or end,
131            # so we divide by two. There is also an extra line chunk
132            # corresponding to the line end, so we subtract it.
133            numends = (len(self.blockend.split(line)) - 1) // 2
134            indentation_level -= numends
135
136            result.append(self.remove_indentation(line, indentation_level))
137
138            numstarts = (len(self.blockstart.split(line)) - 1) // 2
139            indentation_level += numstarts
140
141        return '\n'.join(result) + '\n'
142
143    def remove_indentation(self, line, n):
144        for _ in range(n):
145            if line.startswith(' '):
146                line = line[4:]
147            elif line.startswith('\t'):
148                line = line[1:]
149            else:
150                assert line.strip() == ''
151        return line
152
153
154_FileOutput = namedtuple('FileOutput', ['name', 'content'])
155
156
157def _do_renders(renders, template_dir):
158    loader = _PreprocessingLoader(template_dir)
159    env = jinja2.Environment(loader=loader,
160                             lstrip_blocks=True,
161                             trim_blocks=True,
162                             line_comment_prefix='//*')
163
164    def do_assert(expr):
165        assert expr
166        return ''
167
168    def debug(text):
169        print(text)
170
171    base_params = {
172        'enumerate': enumerate,
173        'format': format,
174        'len': len,
175        'debug': debug,
176        'assert': do_assert,
177    }
178
179    outputs = []
180    for render in renders:
181        params = {}
182        params.update(base_params)
183        for param_dict in render.params_dicts:
184            params.update(param_dict)
185        content = env.get_template(render.template).render(**params)
186        outputs.append(_FileOutput(render.output, content))
187
188    return outputs
189
190
191# Compute the list of imported, non-system Python modules.
192# It assumes that any path outside of the root directory is system.
193def _compute_python_dependencies(root_dir=None):
194    if not root_dir:
195        # Assume this script is under generator/ by default.
196        root_dir = os.path.join(os.path.dirname(__file__), os.pardir)
197    root_dir = os.path.abspath(root_dir)
198
199    module_paths = (module.__file__ for module in sys.modules.values()
200                    if module and hasattr(module, '__file__'))
201
202    paths = set()
203    for path in module_paths:
204        # Builtin/namespaced modules may return None for the file path.
205        if not path:
206            continue
207
208        path = os.path.abspath(path)
209
210        if not path.startswith(root_dir):
211            continue
212
213        if (path.endswith('.pyc')
214                or (path.endswith('c') and not os.path.splitext(path)[1])):
215            path = path[:-1]
216
217        paths.add(path)
218
219    return paths
220
221
222def run_generator(generator):
223    parser = argparse.ArgumentParser(
224        description=generator.get_description(),
225        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
226    )
227
228    generator.add_commandline_arguments(parser)
229    parser.add_argument('--template-dir',
230                        default='templates',
231                        type=str,
232                        help='Directory with template files.')
233    parser.add_argument(
234        kJinja2Path,
235        default=None,
236        type=str,
237        help='Additional python path to set before loading Jinja2')
238    parser.add_argument(
239        '--output-json-tarball',
240        default=None,
241        type=str,
242        help=('Name of the "JSON tarball" to create (tar is too annoying '
243              'to use in python).'))
244    parser.add_argument(
245        '--depfile',
246        default=None,
247        type=str,
248        help='Name of the Ninja depfile to create for the JSON tarball')
249    parser.add_argument(
250        '--expected-outputs-file',
251        default=None,
252        type=str,
253        help="File to compare outputs with and fail if it doesn't match")
254    parser.add_argument(
255        '--root-dir',
256        default=None,
257        type=str,
258        help=('Optional source root directory for Python dependency '
259              'computations'))
260    parser.add_argument(
261        '--allowed-output-dirs-file',
262        default=None,
263        type=str,
264        help=("File containing a list of allowed directories where files "
265              "can be output."))
266    parser.add_argument(
267        '--print-cmake-dependencies',
268        default=False,
269        action="store_true",
270        help=("Prints a semi-colon separated list of dependencies to "
271              "stdout and exits."))
272    parser.add_argument(
273        '--print-cmake-outputs',
274        default=False,
275        action="store_true",
276        help=("Prints a semi-colon separated list of outputs to "
277              "stdout and exits."))
278    parser.add_argument('--output-dir',
279                        default=None,
280                        type=str,
281                        help='Directory where to output generate files.')
282
283    args = parser.parse_args()
284
285    renders = generator.get_file_renders(args)
286
287    # Output a list of all dependencies for CMake or the tarball for GN/Ninja.
288    if args.depfile != None or args.print_cmake_dependencies:
289        dependencies = generator.get_dependencies(args)
290        dependencies += [
291            args.template_dir + os.path.sep + render.template
292            for render in renders
293        ]
294        dependencies += _compute_python_dependencies(args.root_dir)
295
296        if args.depfile != None:
297            with open(args.depfile, 'w') as f:
298                f.write(args.output_json_tarball + ": " +
299                        " ".join(dependencies))
300
301        if args.print_cmake_dependencies:
302            sys.stdout.write(";".join(dependencies))
303            return 0
304
305    # The caller wants to assert that the outputs are what it expects.
306    # Load the file and compare with our renders.
307    if args.expected_outputs_file != None:
308        with open(args.expected_outputs_file) as f:
309            expected = set([line.strip() for line in f.readlines()])
310
311        actual = {render.output for render in renders}
312
313        if actual != expected:
314            print("Wrong expected outputs, caller expected:\n    " +
315                  repr(sorted(expected)))
316            print("Actual output:\n    " + repr(sorted(actual)))
317            return 1
318
319    # Print the list of all the outputs for cmake.
320    if args.print_cmake_outputs:
321        sys.stdout.write(";".join([
322            os.path.join(args.output_dir, render.output) for render in renders
323        ]))
324        return 0
325
326    outputs = _do_renders(renders, args.template_dir)
327
328    # The caller wants to assert that the outputs are only in specific
329    # directories.
330    if args.allowed_output_dirs_file != None:
331        with open(args.allowed_output_dirs_file) as f:
332            allowed_dirs = set([line.strip() for line in f.readlines()])
333
334        for directory in allowed_dirs:
335            if not directory.endswith('/'):
336                print('Allowed directory entry "{}" doesn\'t '
337                      'end with /'.format(directory))
338                return 1
339
340        def check_in_subdirectory(path, directory):
341            return path.startswith(
342                directory) and not '/' in path[len(directory):]
343
344        for render in renders:
345            if not any(
346                    check_in_subdirectory(render.output, directory)
347                    for directory in allowed_dirs):
348                print('Output file "{}" is not in the allowed directory '
349                      'list below:'.format(render.output))
350                for directory in sorted(allowed_dirs):
351                    print('    "{}"'.format(directory))
352                return 1
353
354    # Output the JSON tarball
355    if args.output_json_tarball != None:
356        json_root = {}
357        for output in outputs:
358            json_root[output.name] = output.content
359
360        with open(args.output_json_tarball, 'w') as f:
361            f.write(json.dumps(json_root))
362
363    # Output the files directly.
364    if args.output_dir != None:
365        for output in outputs:
366            output_path = os.path.join(args.output_dir, output.name)
367
368            directory = os.path.dirname(output_path)
369            if not os.path.exists(directory):
370                os.makedirs(directory)
371
372            with open(output_path, 'w') as outfile:
373                outfile.write(output.content)
374