3 """Writes large manifest files, for manifest parser performance testing.
5 The generated manifest files are (eerily) similar in appearance and size to the
6 ones used in the Chromium project.
9 python misc/write_fake_manifests.py outdir # Will run for about 5s.
11 The program contains a hardcoded random seed, so it will generate the same
12 output every time it runs. By changing the seed, it's easy to generate many
13 different sets of manifest files.
25 def paretoint(avg, alpha):
26 """Returns a random integer that's avg on average, following a power law.
27 alpha determines the shape of the power curve. alpha has to be larger
28 than 1. The closer alpha is to 1, the higher the variation of the returned
30 return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1)))
33 # Based on http://neugierig.org/software/chromium/class-name-generator.html
34 def moar(avg_options, p_suffix):
35 kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url',
36 'file', 'sync', 'content', 'http', 'profile']
37 kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref',
38 'delegate', 'widget', 'proxy', 'stub', 'context',
39 'manager', 'master', 'watcher', 'service', 'file', 'data',
40 'resource', 'device', 'info', 'provider', 'internals', 'tracker',
42 kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest']
43 num_options = min(paretoint(avg_options, alpha=4), 5)
44 # The original allows kOption to repeat as long as no consecutive options
45 # repeat. This version doesn't allow any option repetition.
46 name = [random.choice(kStart)] + random.sample(kOption, num_options)
47 if random.random() < p_suffix:
48 name.append(random.choice(kOS))
52 class GenRandom(object):
53 def __init__(self, src_dir):
54 self.seen_names = set([None])
55 self.seen_defines = set([None])
56 self.src_dir = src_dir
58 def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1):
61 s = moar(avg_options, p_suffix)
65 def _n_unique_strings(self, n):
67 return [self._unique_string(seen, avg_options=3, p_suffix=0.4)
70 def target_name(self):
71 return self._unique_string(p_suffix=0, seen=self.seen_names)
74 return os.path.sep.join([
75 self._unique_string(self.seen_names, avg_options=1, p_suffix=0)
76 for _ in xrange(1 + paretoint(0.6, alpha=4))])
78 def src_obj_pairs(self, path, name):
79 num_sources = paretoint(55, alpha=2) + 1
80 return [(os.path.join(self.src_dir, path, s + '.cc'),
81 os.path.join('obj', path, '%s.%s.o' % (name, s)))
82 for s in self._n_unique_strings(num_sources)]
86 '-DENABLE_' + self._unique_string(self.seen_defines).upper()
87 for _ in xrange(paretoint(20, alpha=3))]
92 def __init__(self, gen, kind):
93 self.name = gen.target_name()
94 self.dir_path = gen.path()
95 self.ninja_file_path = os.path.join(
96 'obj', self.dir_path, self.name + '.ninja')
97 self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name)
99 self.output = os.path.join('lib' + self.name + '.a')
101 self.output = os.path.join(self.name)
102 self.defines = gen.defines()
105 self.has_compile_depends = random.random() < 0.4
108 def write_target_ninja(ninja, target, src_dir):
109 compile_depends = None
110 if target.has_compile_depends:
111 compile_depends = os.path.join(
112 'obj', target.dir_path, target.name + '.stamp')
113 ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0])
116 ninja.variable('defines', target.defines)
117 ninja.variable('includes', '-I' + src_dir)
118 ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
121 for src, obj in target.src_obj_pairs:
122 ninja.build(obj, 'cxx', src, implicit=compile_depends)
125 deps = [dep.output for dep in target.deps]
126 libs = [dep.output for dep in target.deps if dep.kind == LIB]
127 if target.kind == EXE:
128 ninja.variable('libs', libs)
129 if sys.platform == "darwin":
130 ninja.variable('ldflags', '-Wl,-pie')
131 link = { LIB: 'alink', EXE: 'link'}[target.kind]
132 ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
136 def write_sources(target, root_dir):
137 need_main = target.kind == EXE
142 for cc_filename, _ in target.src_obj_pairs:
143 h_filename = os.path.basename(os.path.splitext(cc_filename)[0] + '.h')
144 includes.append(h_filename)
147 for dep in target.deps:
148 for cc_filename, _ in dep.src_obj_pairs:
149 h_filename = os.path.basename(
150 os.path.splitext(cc_filename)[0] + '.h')
151 includes.append("%s/%s" % (dep.dir_path, h_filename))
153 for cc_filename, _ in target.src_obj_pairs:
154 cc_path = os.path.join(root_dir, cc_filename)
155 h_path = os.path.splitext(cc_path)[0] + '.h'
156 namespace = os.path.basename(target.dir_path)
157 class_ = os.path.splitext(os.path.basename(cc_filename))[0]
159 os.makedirs(os.path.dirname(cc_path))
163 with open(h_path, 'w') as f:
164 f.write('namespace %s { struct %s { %s(); }; }' % (namespace,
166 with open(cc_path, 'w') as f:
167 for include in includes:
168 f.write('#include "%s"\n' % include)
170 f.write('namespace %s { %s::%s() {} }' % (namespace,
174 f.write('int main(int argc, char **argv) {}\n')
177 def write_master_ninja(master_ninja, targets):
178 """Writes master build.ninja file, referencing all given subninjas."""
179 master_ninja.variable('cxx', 'c++')
180 master_ninja.variable('ld', '$cxx')
181 if sys.platform == 'darwin':
182 master_ninja.variable('alink', 'libtool -static')
184 master_ninja.variable('alink', 'ar rcs')
185 master_ninja.newline()
187 master_ninja.pool('link_pool', depth=4)
188 master_ninja.newline()
190 master_ninja.rule('cxx', description='CXX $out',
191 command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
192 depfile='$out.d', deps='gcc')
193 master_ninja.rule('alink', description='ARCHIVE $out',
194 command='rm -f $out && $alink -o $out $in')
195 master_ninja.rule('link', description='LINK $out', pool='link_pool',
196 command='$ld $ldflags -o $out $in $libs')
197 master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
198 master_ninja.newline()
200 for target in targets:
201 master_ninja.subninja(target.ninja_file_path)
202 master_ninja.newline()
204 master_ninja.comment('Short names for targets.')
205 for target in targets:
206 if target.name != target.output:
207 master_ninja.build(target.name, 'phony', target.output)
208 master_ninja.newline()
210 master_ninja.build('all', 'phony', [target.output for target in targets])
211 master_ninja.default('all')
214 @contextlib.contextmanager
215 def FileWriter(path):
216 """Context manager for a ninja_syntax object writing to a file."""
218 os.makedirs(os.path.dirname(path))
222 yield ninja_syntax.Writer(f)
226 def random_targets(num_targets, src_dir):
227 gen = GenRandom(src_dir)
229 # N-1 static libraries, and 1 executable depending on all of them.
230 targets = [Target(gen, LIB) for i in xrange(num_targets - 1)]
231 for i in range(len(targets)):
232 targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05]
234 last_target = Target(gen, EXE)
235 last_target.deps = targets[:]
236 last_target.src_obj_pairs = last_target.src_obj_pairs[0:10] # Trim.
237 targets.append(last_target)
242 parser = argparse.ArgumentParser()
243 parser.add_argument('-s', '--sources', nargs="?", const="src",
244 help='write sources to directory (relative to output directory)')
245 parser.add_argument('-t', '--targets', type=int, default=1500,
246 help='number of targets (default: 1500)')
247 parser.add_argument('-S', '--seed', type=int, help='random seed',
249 parser.add_argument('outdir', help='output directory')
250 args = parser.parse_args()
251 root_dir = args.outdir
253 random.seed(args.seed)
255 do_write_sources = args.sources is not None
256 src_dir = args.sources if do_write_sources else "src"
258 targets = random_targets(args.targets, src_dir)
259 for target in targets:
260 with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
261 write_target_ninja(n, target, src_dir)
264 write_sources(target, root_dir)
266 with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
267 master_ninja.width = 120
268 write_master_ninja(master_ninja, targets)
271 if __name__ == '__main__':