• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2
3"""Writes large manifest files, for manifest parser performance testing.
4
5The generated manifest files are (eerily) similar in appearance and size to the
6ones used in the Chromium project.
7
8Usage:
9  python misc/write_fake_manifests.py outdir  # Will run for about 5s.
10
11The program contains a hardcoded random seed, so it will generate the same
12output every time it runs.  By changing the seed, it's easy to generate many
13different sets of manifest files.
14"""
15
16import argparse
17import contextlib
18import os
19import random
20import sys
21
22import ninja_syntax
23
24
25def paretoint(avg, alpha):
26    """Returns a random integer that's avg on average, following a power law.
27    alpha determines the shape of the power curve. alpha has to be larger
28    than 1. The closer alpha is to 1, the higher the variation of the returned
29    numbers."""
30    return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1)))
31
32
33# Based on http://neugierig.org/software/chromium/class-name-generator.html
34def moar(avg_options, p_suffix):
35    kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url',
36              'file', 'sync', 'content', 'http', 'profile']
37    kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref',
38               'delegate', 'widget', 'proxy', 'stub', 'context',
39               'manager', 'master', 'watcher', 'service', 'file', 'data',
40               'resource', 'device', 'info', 'provider', 'internals', 'tracker',
41               'api', 'layer']
42    kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest']
43    num_options = min(paretoint(avg_options, alpha=4), 5)
44    # The original allows kOption to repeat as long as no consecutive options
45    # repeat.  This version doesn't allow any option repetition.
46    name = [random.choice(kStart)] + random.sample(kOption, num_options)
47    if random.random() < p_suffix:
48        name.append(random.choice(kOS))
49    return '_'.join(name)
50
51
52class GenRandom(object):
53    def __init__(self, src_dir):
54        self.seen_names = set([None])
55        self.seen_defines = set([None])
56        self.src_dir = src_dir
57
58    def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1):
59        s = None
60        while s in seen:
61            s = moar(avg_options, p_suffix)
62        seen.add(s)
63        return s
64
65    def _n_unique_strings(self, n):
66        seen = set([None])
67        return [self._unique_string(seen, avg_options=3, p_suffix=0.4)
68                for _ in range(n)]
69
70    def target_name(self):
71        return self._unique_string(p_suffix=0, seen=self.seen_names)
72
73    def path(self):
74        return os.path.sep.join([
75            self._unique_string(self.seen_names, avg_options=1, p_suffix=0)
76            for _ in range(1 + paretoint(0.6, alpha=4))])
77
78    def src_obj_pairs(self, path, name):
79        num_sources = paretoint(55, alpha=2) + 1
80        return [(os.path.join(self.src_dir, path, s + '.cc'),
81                 os.path.join('obj', path, '%s.%s.o' % (name, s)))
82                for s in self._n_unique_strings(num_sources)]
83
84    def defines(self):
85        return [
86            '-DENABLE_' + self._unique_string(self.seen_defines).upper()
87            for _ in range(paretoint(20, alpha=3))]
88
89
90LIB, EXE = 0, 1
91class Target(object):
92    def __init__(self, gen, kind):
93        self.name = gen.target_name()
94        self.dir_path = gen.path()
95        self.ninja_file_path = os.path.join(
96            'obj', self.dir_path, self.name + '.ninja')
97        self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name)
98        if kind == LIB:
99            self.output = os.path.join('lib' + self.name + '.a')
100        elif kind == EXE:
101            self.output = os.path.join(self.name)
102        self.defines = gen.defines()
103        self.deps = []
104        self.kind = kind
105        self.has_compile_depends = random.random() < 0.4
106
107
108def write_target_ninja(ninja, target, src_dir):
109    compile_depends = None
110    if target.has_compile_depends:
111      compile_depends = os.path.join(
112          'obj', target.dir_path, target.name + '.stamp')
113      ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0])
114      ninja.newline()
115
116    ninja.variable('defines', target.defines)
117    ninja.variable('includes', '-I' + src_dir)
118    ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
119    ninja.newline()
120
121    for src, obj in target.src_obj_pairs:
122        ninja.build(obj, 'cxx', src, implicit=compile_depends)
123    ninja.newline()
124
125    deps = [dep.output for dep in target.deps]
126    libs = [dep.output for dep in target.deps if dep.kind == LIB]
127    if target.kind == EXE:
128        ninja.variable('libs', libs)
129        if sys.platform == "darwin":
130            ninja.variable('ldflags', '-Wl,-pie')
131    link = { LIB: 'alink', EXE: 'link'}[target.kind]
132    ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
133                implicit=deps)
134
135
136def write_sources(target, root_dir):
137    need_main = target.kind == EXE
138
139    includes = []
140
141    # Include siblings.
142    for cc_filename, _ in target.src_obj_pairs:
143        h_filename = os.path.basename(os.path.splitext(cc_filename)[0] + '.h')
144        includes.append(h_filename)
145
146    # Include deps.
147    for dep in target.deps:
148        for cc_filename, _ in dep.src_obj_pairs:
149            h_filename = os.path.basename(
150                os.path.splitext(cc_filename)[0] + '.h')
151            includes.append("%s/%s" % (dep.dir_path, h_filename))
152
153    for cc_filename, _ in target.src_obj_pairs:
154        cc_path = os.path.join(root_dir, cc_filename)
155        h_path = os.path.splitext(cc_path)[0] + '.h'
156        namespace = os.path.basename(target.dir_path)
157        class_ = os.path.splitext(os.path.basename(cc_filename))[0]
158        try:
159            os.makedirs(os.path.dirname(cc_path))
160        except OSError:
161            pass
162
163        with open(h_path, 'w') as f:
164            f.write('namespace %s { struct %s { %s(); }; }' % (namespace,
165                                                               class_, class_))
166        with open(cc_path, 'w') as f:
167            for include in includes:
168                f.write('#include "%s"\n' % include)
169            f.write('\n')
170            f.write('namespace %s { %s::%s() {} }' % (namespace,
171                                                      class_, class_))
172
173            if need_main:
174                f.write('int main(int argc, char **argv) {}\n')
175                need_main = False
176
177def write_master_ninja(master_ninja, targets):
178    """Writes master build.ninja file, referencing all given subninjas."""
179    master_ninja.variable('cxx', 'c++')
180    master_ninja.variable('ld', '$cxx')
181    if sys.platform == 'darwin':
182        master_ninja.variable('alink', 'libtool -static')
183    else:
184        master_ninja.variable('alink', 'ar rcs')
185    master_ninja.newline()
186
187    master_ninja.pool('link_pool', depth=4)
188    master_ninja.newline()
189
190    master_ninja.rule('cxx', description='CXX $out',
191      command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
192      depfile='$out.d', deps='gcc')
193    master_ninja.rule('alink', description='ARCHIVE $out',
194      command='rm -f $out && $alink -o $out $in')
195    master_ninja.rule('link', description='LINK $out', pool='link_pool',
196      command='$ld $ldflags -o $out $in $libs')
197    master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
198    master_ninja.newline()
199
200    for target in targets:
201        master_ninja.subninja(target.ninja_file_path)
202    master_ninja.newline()
203
204    master_ninja.comment('Short names for targets.')
205    for target in targets:
206        if target.name != target.output:
207            master_ninja.build(target.name, 'phony', target.output)
208    master_ninja.newline()
209
210    master_ninja.build('all', 'phony', [target.output for target in targets])
211    master_ninja.default('all')
212
213
214@contextlib.contextmanager
215def FileWriter(path):
216    """Context manager for a ninja_syntax object writing to a file."""
217    try:
218        os.makedirs(os.path.dirname(path))
219    except OSError:
220        pass
221    f = open(path, 'w')
222    yield ninja_syntax.Writer(f)
223    f.close()
224
225
226def random_targets(num_targets, src_dir):
227    gen = GenRandom(src_dir)
228
229    # N-1 static libraries, and 1 executable depending on all of them.
230    targets = [Target(gen, LIB) for i in range(num_targets - 1)]
231    for i in range(len(targets)):
232        targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05]
233
234    last_target = Target(gen, EXE)
235    last_target.deps = targets[:]
236    last_target.src_obj_pairs = last_target.src_obj_pairs[0:10]  # Trim.
237    targets.append(last_target)
238    return targets
239
240
241def main():
242    parser = argparse.ArgumentParser()
243    parser.add_argument('-s', '--sources', nargs="?", const="src",
244        help='write sources to directory (relative to output directory)')
245    parser.add_argument('-t', '--targets', type=int, default=1500,
246                        help='number of targets (default: 1500)')
247    parser.add_argument('-S', '--seed', type=int, help='random seed',
248                        default=12345)
249    parser.add_argument('outdir', help='output directory')
250    args = parser.parse_args()
251    root_dir = args.outdir
252
253    random.seed(args.seed)
254
255    do_write_sources = args.sources is not None
256    src_dir = args.sources if do_write_sources else "src"
257
258    targets = random_targets(args.targets, src_dir)
259    for target in targets:
260        with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
261            write_target_ninja(n, target, src_dir)
262
263        if do_write_sources:
264            write_sources(target, root_dir)
265
266    with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
267        master_ninja.width = 120
268        write_master_ninja(master_ninja, targets)
269
270
271if __name__ == '__main__':
272    sys.exit(main())
273