• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1from __future__ import print_function
2import collections
3import itertools
4import json
5import os
6import os.path
7import re
8import shutil
9import string
10import subprocess
11import sys
12import cgi
13
14class BuildDesc:
15    def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None):
16        self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ]
17        self.variables = variables or []
18        self.build_type = build_type
19        self.generator = generator
20
21    def merged_with(self, build_desc):
22        """Returns a new BuildDesc by merging field content.
23           Prefer build_desc fields to self fields for single valued field.
24        """
25        return BuildDesc(self.prepend_envs + build_desc.prepend_envs,
26                          self.variables + build_desc.variables,
27                          build_desc.build_type or self.build_type,
28                          build_desc.generator or self.generator)
29
30    def env(self):
31        environ = os.environ.copy()
32        for values_by_name in self.prepend_envs:
33            for var, value in list(values_by_name.items()):
34                var = var.upper()
35                if type(value) is unicode:
36                    value = value.encode(sys.getdefaultencoding())
37                if var in environ:
38                    environ[var] = value + os.pathsep + environ[var]
39                else:
40                    environ[var] = value
41        return environ
42
43    def cmake_args(self):
44        args = ["-D%s" % var for var in self.variables]
45        # skip build type for Visual Studio solution as it cause warning
46        if self.build_type and 'Visual' not in self.generator:
47            args.append("-DCMAKE_BUILD_TYPE=%s" % self.build_type)
48        if self.generator:
49            args.extend(['-G', self.generator])
50        return args
51
52    def __repr__(self):
53        return "BuildDesc(%s, build_type=%s)" %  (" ".join(self.cmake_args()), self.build_type)
54
55class BuildData:
56    def __init__(self, desc, work_dir, source_dir):
57        self.desc = desc
58        self.work_dir = work_dir
59        self.source_dir = source_dir
60        self.cmake_log_path = os.path.join(work_dir, 'batchbuild_cmake.log')
61        self.build_log_path = os.path.join(work_dir, 'batchbuild_build.log')
62        self.cmake_succeeded = False
63        self.build_succeeded = False
64
65    def execute_build(self):
66        print('Build %s' % self.desc)
67        self._make_new_work_dir()
68        self.cmake_succeeded = self._generate_makefiles()
69        if self.cmake_succeeded:
70            self.build_succeeded = self._build_using_makefiles()
71        return self.build_succeeded
72
73    def _generate_makefiles(self):
74        print('  Generating makefiles: ', end=' ')
75        cmd = ['cmake'] + self.desc.cmake_args() + [os.path.abspath(self.source_dir)]
76        succeeded = self._execute_build_subprocess(cmd, self.desc.env(), self.cmake_log_path)
77        print('done' if succeeded else 'FAILED')
78        return succeeded
79
80    def _build_using_makefiles(self):
81        print('  Building:', end=' ')
82        cmd = ['cmake', '--build', self.work_dir]
83        if self.desc.build_type:
84            cmd += ['--config', self.desc.build_type]
85        succeeded = self._execute_build_subprocess(cmd, self.desc.env(), self.build_log_path)
86        print('done' if succeeded else 'FAILED')
87        return succeeded
88
89    def _execute_build_subprocess(self, cmd, env, log_path):
90        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir,
91                                    env=env)
92        stdout, _ = process.communicate()
93        succeeded = (process.returncode == 0)
94        with open(log_path, 'wb') as flog:
95            log = ' '.join(cmd) + '\n' + stdout + '\nExit code: %r\n' % process.returncode
96            flog.write(fix_eol(log))
97        return succeeded
98
99    def _make_new_work_dir(self):
100        if os.path.isdir(self.work_dir):
101            print('  Removing work directory', self.work_dir)
102            shutil.rmtree(self.work_dir, ignore_errors=True)
103        if not os.path.isdir(self.work_dir):
104            os.makedirs(self.work_dir)
105
106def fix_eol(stdout):
107    """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n).
108    """
109    return re.sub('\r*\n', os.linesep, stdout)
110
111def load_build_variants_from_config(config_path):
112    with open(config_path, 'rb') as fconfig:
113        data = json.load(fconfig)
114    variants = data[ 'cmake_variants' ]
115    build_descs_by_axis = collections.defaultdict(list)
116    for axis in variants:
117        axis_name = axis["name"]
118        build_descs = []
119        if "generators" in axis:
120            for generator_data in axis["generators"]:
121                for generator in generator_data["generator"]:
122                    build_desc = BuildDesc(generator=generator,
123                                            prepend_envs=generator_data.get("env_prepend"))
124                    build_descs.append(build_desc)
125        elif "variables" in axis:
126            for variables in axis["variables"]:
127                build_desc = BuildDesc(variables=variables)
128                build_descs.append(build_desc)
129        elif "build_types" in axis:
130            for build_type in axis["build_types"]:
131                build_desc = BuildDesc(build_type=build_type)
132                build_descs.append(build_desc)
133        build_descs_by_axis[axis_name].extend(build_descs)
134    return build_descs_by_axis
135
136def generate_build_variants(build_descs_by_axis):
137    """Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""
138    axis_names = list(build_descs_by_axis.keys())
139    build_descs = []
140    for axis_name, axis_build_descs in list(build_descs_by_axis.items()):
141        if len(build_descs):
142            # for each existing build_desc and each axis build desc, create a new build_desc
143            new_build_descs = []
144            for prototype_build_desc, axis_build_desc in itertools.product(build_descs, axis_build_descs):
145                new_build_descs.append(prototype_build_desc.merged_with(axis_build_desc))
146            build_descs = new_build_descs
147        else:
148            build_descs = axis_build_descs
149    return build_descs
150
151HTML_TEMPLATE = string.Template('''<html>
152<head>
153    <title>$title</title>
154    <style type="text/css">
155    td.failed {background-color:#f08080;}
156    td.ok {background-color:#c0eec0;}
157    </style>
158</head>
159<body>
160<table border="1">
161<thead>
162    <tr>
163        <th>Variables</th>
164        $th_vars
165    </tr>
166    <tr>
167        <th>Build type</th>
168        $th_build_types
169    </tr>
170</thead>
171<tbody>
172$tr_builds
173</tbody>
174</table>
175</body></html>''')
176
177def generate_html_report(html_report_path, builds):
178    report_dir = os.path.dirname(html_report_path)
179    # Vertical axis: generator
180    # Horizontal: variables, then build_type
181    builds_by_generator = collections.defaultdict(list)
182    variables = set()
183    build_types_by_variable = collections.defaultdict(set)
184    build_by_pos_key = {} # { (generator, var_key, build_type): build }
185    for build in builds:
186        builds_by_generator[build.desc.generator].append(build)
187        var_key = tuple(sorted(build.desc.variables))
188        variables.add(var_key)
189        build_types_by_variable[var_key].add(build.desc.build_type)
190        pos_key = (build.desc.generator, var_key, build.desc.build_type)
191        build_by_pos_key[pos_key] = build
192    variables = sorted(variables)
193    th_vars = []
194    th_build_types = []
195    for variable in variables:
196        build_types = sorted(build_types_by_variable[variable])
197        nb_build_type = len(build_types_by_variable[variable])
198        th_vars.append('<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape(' '.join(variable))))
199        for build_type in build_types:
200            th_build_types.append('<th>%s</th>' % cgi.escape(build_type))
201    tr_builds = []
202    for generator in sorted(builds_by_generator):
203        tds = [ '<td>%s</td>\n' % cgi.escape(generator) ]
204        for variable in variables:
205            build_types = sorted(build_types_by_variable[variable])
206            for build_type in build_types:
207                pos_key = (generator, variable, build_type)
208                build = build_by_pos_key.get(pos_key)
209                if build:
210                    cmake_status = 'ok' if build.cmake_succeeded else 'FAILED'
211                    build_status = 'ok' if build.build_succeeded else 'FAILED'
212                    cmake_log_url = os.path.relpath(build.cmake_log_path, report_dir)
213                    build_log_url = os.path.relpath(build.build_log_path, report_dir)
214                    td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (                        build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
215                    if build.cmake_succeeded:
216                        td += '<br><a href="%s" class="%s">Build: %s</a>' % (                            build_log_url, build_status.lower(), build_status)
217                    td += '</td>'
218                else:
219                    td = '<td></td>'
220                tds.append(td)
221        tr_builds.append('<tr>%s</tr>' % '\n'.join(tds))
222    html = HTML_TEMPLATE.substitute(        title='Batch build report',
223        th_vars=' '.join(th_vars),
224        th_build_types=' '.join(th_build_types),
225        tr_builds='\n'.join(tr_builds))
226    with open(html_report_path, 'wt') as fhtml:
227        fhtml.write(html)
228    print('HTML report generated in:', html_report_path)
229
230def main():
231    usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]
232Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run
233as described in CONFIG_JSON_PATH building in WORK_DIR.
234
235Example of call:
236python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json
237"""
238    from optparse import OptionParser
239    parser = OptionParser(usage=usage)
240    parser.allow_interspersed_args = True
241#    parser.add_option('-v', '--verbose', dest="verbose", action='store_true',
242#        help="""Be verbose.""")
243    parser.enable_interspersed_args()
244    options, args = parser.parse_args()
245    if len(args) < 3:
246        parser.error("Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH.")
247    work_dir = args[0]
248    source_dir = args[1].rstrip('/\\')
249    config_paths = args[2:]
250    for config_path in config_paths:
251        if not os.path.isfile(config_path):
252            parser.error("Can not read: %r" % config_path)
253
254    # generate build variants
255    build_descs = []
256    for config_path in config_paths:
257        build_descs_by_axis = load_build_variants_from_config(config_path)
258        build_descs.extend(generate_build_variants(build_descs_by_axis))
259    print('Build variants (%d):' % len(build_descs))
260    # assign build directory for each variant
261    if not os.path.isdir(work_dir):
262        os.makedirs(work_dir)
263    builds = []
264    with open(os.path.join(work_dir, 'matrix-dir-map.txt'), 'wt') as fmatrixmap:
265        for index, build_desc in enumerate(build_descs):
266            build_desc_work_dir = os.path.join(work_dir, '%03d' % (index+1))
267            builds.append(BuildData(build_desc, build_desc_work_dir, source_dir))
268            fmatrixmap.write('%s: %s\n' % (build_desc_work_dir, build_desc))
269    for build in builds:
270        build.execute_build()
271    html_report_path = os.path.join(work_dir, 'batchbuild-report.html')
272    generate_html_report(html_report_path, builds)
273    print('Done')
274
275
276if __name__ == '__main__':
277    main()
278
279