• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 gRPC authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import argparse
16import glob
17import os
18import shutil
19import sys
20import tempfile
21import multiprocessing
22sys.path.append(
23    os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests',
24                 'python_utils'))
25
26assert sys.argv[1:], 'run generate_projects.sh instead of this directly'
27
28import jobset
29
30os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))
31
32argp = argparse.ArgumentParser()
33argp.add_argument('build_files', nargs='+', default=[])
34argp.add_argument('--templates', nargs='+', default=[])
35argp.add_argument('--output_merged', default=None, type=str)
36argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int)
37argp.add_argument('--base', default='.', type=str)
38args = argp.parse_args()
39
40json = args.build_files
41
42test = {} if os.environ.get('TEST', 'false') == 'true' else None
43
44plugins = sorted(glob.glob('tools/buildgen/plugins/*.py'))
45
46templates = args.templates
47if not templates:
48    for root, dirs, files in os.walk('templates'):
49        for f in files:
50            templates.append(os.path.join(root, f))
51
52pre_jobs = []
53base_cmd = [sys.executable, 'tools/buildgen/mako_renderer.py']
54cmd = base_cmd[:]
55for plugin in plugins:
56    cmd.append('-p')
57    cmd.append(plugin)
58for js in json:
59    cmd.append('-d')
60    cmd.append(js)
61cmd.append('-w')
62preprocessed_build = '.preprocessed_build'
63cmd.append(preprocessed_build)
64if args.output_merged is not None:
65    cmd.append('-M')
66    cmd.append(args.output_merged)
67pre_jobs.append(
68    jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None))
69
70jobs = []
71for template in reversed(sorted(templates)):
72    root, f = os.path.split(template)
73    if os.path.splitext(f)[1] == '.template':
74        out_dir = args.base + root[len('templates'):]
75        out = out_dir + '/' + os.path.splitext(f)[0]
76        if not os.path.exists(out_dir):
77            os.makedirs(out_dir)
78        cmd = base_cmd[:]
79        cmd.append('-P')
80        cmd.append(preprocessed_build)
81        cmd.append('-o')
82        if test is None:
83            cmd.append(out)
84        else:
85            tf = tempfile.mkstemp()
86            test[out] = tf[1]
87            os.close(tf[0])
88            cmd.append(test[out])
89        cmd.append(args.base + '/' + root + '/' + f)
90        jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))
91
92jobset.run(pre_jobs, maxjobs=args.jobs)
93jobset.run(jobs, maxjobs=args.jobs)
94
95if test is not None:
96    for s, g in test.iteritems():
97        if os.path.isfile(g):
98            assert 0 == os.system('diff %s %s' % (s, g)), s
99            os.unlink(g)
100        else:
101            assert 0 == os.system('diff -r %s %s' % (s, g)), s
102            shutil.rmtree(g, ignore_errors=True)
103