1 # Copyright 2014 The Chromium Authors
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Helper functions useful when writing scripts that integrate with GN.
7 The main functions are ToGNString() and FromGNString(), to convert between
8 serialized GN veriables and Python variables.
10 To use in an arbitrary Python file in the build:
15 sys.path.append(os.path.join(os.path.dirname(__file__),
16 os.pardir, os.pardir, 'build'))
19 Where the sequence of parameters to join is the relative path from your source
20 file to the build directory.
31 _CHROMIUM_ROOT = os.path.abspath(
32 os.path.join(os.path.dirname(__file__), os.pardir))
34 BUILD_VARS_FILENAME = 'build_vars.json'
35 IMPORT_RE = re.compile(r'^import\("//(\S+)"\)')
38 class GNError(Exception):
42 # Computes ASCII code of an element of encoded Python 2 str / Python 3 bytes.
43 _Ord = ord if sys.version_info.major < 3 else lambda c: c
46 def _TranslateToGnChars(s):
47 for decoded_ch in s.encode('utf-8'): # str in Python 2, bytes in Python 3.
48 code = _Ord(decoded_ch) # int
49 if code in (34, 36, 92): # For '"', '$', or '\\'.
50 yield '\\' + chr(code)
51 elif 32 <= code < 127:
54 yield '$0x%02X' % code
57 def ToGNString(value, pretty=False):
58 """Returns a stringified GN equivalent of a Python value.
61 value: The Python value to convert.
62 pretty: Whether to pretty print. If true, then non-empty lists are rendered
63 recursively with one item per line, with indents. Otherwise lists are
64 rendered without new line.
66 The stringified GN equivalent to |value|.
69 GNError: |value| cannot be printed to GN.
72 if sys.version_info.major < 3:
73 basestring_compat = basestring
75 basestring_compat = str
77 # Emits all output tokens without intervening whitespaces.
78 def GenerateTokens(v, level):
79 if isinstance(v, basestring_compat):
80 yield '"' + ''.join(_TranslateToGnChars(v)) + '"'
82 elif isinstance(v, bool):
83 yield 'true' if v else 'false'
85 elif isinstance(v, int):
88 elif isinstance(v, list):
90 for i, item in enumerate(v):
93 for tok in GenerateTokens(item, level + 1):
97 elif isinstance(v, dict):
100 for key in sorted(v):
101 if not isinstance(key, basestring_compat):
102 raise GNError('Dictionary key is not a string.')
103 if not key or key[0].isdigit() or not key.replace('_', '').isalnum():
104 raise GNError('Dictionary key is not a valid GN identifier.')
105 yield key # No quotations.
107 for tok in GenerateTokens(v[key], level + 1):
112 else: # Not supporting float: Add only when needed.
113 raise GNError('Unsupported type when printing to GN.')
115 can_start = lambda tok: tok and tok not in ',}]='
116 can_end = lambda tok: tok and tok not in ',{[='
118 # Adds whitespaces, trying to keep everything (except dicts) in 1 line.
121 for i, tok in enumerate(gen):
123 if can_end(prev_tok) and can_start(tok):
124 yield '\n' # New dict item.
125 elif prev_tok == '[' and tok == ']':
126 yield ' ' # Special case for [].
132 # Adds whitespaces so non-empty lists can span multiple lines, with indent.
136 for i, tok in enumerate(gen):
138 if can_end(prev_tok) and can_start(tok):
139 yield '\n' + ' ' * level # New dict item.
140 elif tok == '=' or prev_tok in '=':
141 yield ' ' # Separator before and after '=', on same line.
144 # Exclude '[]' and '{}' cases.
145 if int(prev_tok == '[') + int(tok == ']') == 1 or \
146 int(prev_tok == '{') + int(tok == '}') == 1:
147 yield '\n' + ' ' * level
152 yield '\n' + ' ' * level
155 token_gen = GenerateTokens(value, 0)
156 ret = ''.join((PrettyGlue if pretty else PlainGlue)(token_gen))
157 # Add terminating '\n' for dict |value| or multi-line output.
158 if isinstance(value, dict) or '\n' in ret:
163 def FromGNString(input_string):
164 """Converts the input string from a GN serialized value to Python values.
166 For details on supported types see GNValueParser.Parse() below.
168 If your GN script did:
169 something = [ "file1", "file2" ]
170 args = [ "--values=$something" ]
171 The command line would look something like:
172 --values="[ \"file1\", \"file2\" ]"
173 Which when interpreted as a command line gives the value:
176 You can parse this into a Python list using GN rules with:
177 input_values = FromGNValues(options.values)
178 Although the Python 'ast' module will parse many forms of such input, it
179 will not handle GN escaping properly, nor GN booleans. You should use this
183 A NOTE ON STRING HANDLING:
185 If you just pass a string on the command line to your Python script, or use
186 string interpolation on a string variable, the strings will not be quoted:
188 args = [ str, "--value=$str" ]
189 Will yield the command line:
191 The unquoted asdf string will not be valid input to this function, which
192 accepts only quoted strings like GN scripts. In such cases, you can just use
193 the Python string literal directly.
195 The main use cases for this is for other types, in particular lists. When
196 using string interpolation on a list (as in the top example) the embedded
197 strings will be quoted and escaped according to GN rules so the list can be
198 re-parsed to get the same result.
200 parser = GNValueParser(input_string)
201 return parser.Parse()
204 def FromGNArgs(input_string):
205 """Converts a string with a bunch of gn arg assignments into a Python dict.
207 Given a whitespace-separated list of
209 <ident> = (integer | string | boolean | <list of the former>)
211 gn assignments, this returns a Python dict, i.e.:
213 FromGNArgs('foo=true\nbar=1\n') -> { 'foo': True, 'bar': 1 }.
215 Only simple types and lists supported; variables, structs, calls
216 and other, more complicated things are not.
218 This routine is meant to handle only the simple sorts of values that
219 arise in parsing --args.
221 parser = GNValueParser(input_string)
222 return parser.ParseArgs()
225 def UnescapeGNString(value):
226 """Given a string with GN escaping, returns the unescaped string.
228 Be careful not to feed with input from a Python parsing function like
229 'ast' because it will do Python unescaping, which will be incorrect when
230 fed into the GN unescaper.
233 value: Input string to unescape.
237 while i < len(value):
239 if i < len(value) - 1:
240 next_char = value[i + 1]
241 if next_char in ('$', '"', '\\'):
242 # These are the escaped characters GN supports.
246 # Any other backslash is a literal.
254 def _IsDigitOrMinus(char):
255 return char in '-0123456789'
258 class GNValueParser(object):
259 """Duplicates GN parsing of values and converts to Python types.
261 Normally you would use the wrapper function FromGNValue() below.
263 If you expect input as a specific type, you can also call one of the Parse*
264 functions directly. All functions throw GNError on invalid input.
267 def __init__(self, string, checkout_root=_CHROMIUM_ROOT):
270 self.checkout_root = checkout_root
273 return self.cur == len(self.input)
275 def ReplaceImports(self):
276 """Replaces import(...) lines with the contents of the imports.
278 Recurses on itself until there are no imports remaining, in the case of
281 lines = self.input.splitlines()
282 if not any(line.startswith('import(') for line in lines):
285 if not line.startswith('import('):
287 regex_match = IMPORT_RE.match(line)
289 raise GNError('Not a valid import string: %s' % line)
290 import_path = os.path.join(self.checkout_root, regex_match.group(1))
291 with open(import_path) as f:
292 imported_args = f.read()
293 self.input = self.input.replace(line, imported_args)
294 # Call ourselves again if we've just replaced an import() with additional
296 self.ReplaceImports()
299 def _ConsumeWhitespace(self):
300 while not self.IsDone() and self.input[self.cur] in ' \t\n':
303 def ConsumeCommentAndWhitespace(self):
304 self._ConsumeWhitespace()
306 # Consume each comment, line by line.
307 while not self.IsDone() and self.input[self.cur] == '#':
308 # Consume the rest of the comment, up until the end of the line.
309 while not self.IsDone() and self.input[self.cur] != '\n':
311 # Move the cursor to the next line (if there is one).
312 if not self.IsDone():
315 self._ConsumeWhitespace()
318 """Converts a string representing a printed GN value to the Python type.
320 See additional usage notes on FromGNString() above.
322 * GN booleans ('true', 'false') will be converted to Python booleans.
324 * GN numbers ('123') will be converted to Python numbers.
326 * GN strings (double-quoted as in '"asdf"') will be converted to Python
327 strings with GN escaping rules. GN string interpolation (embedded
328 variables preceded by $) are not supported and will be returned as
331 * GN lists ('[1, "asdf", 3]') will be converted to Python lists.
333 * GN scopes ('{ ... }') are not supported.
336 GNError: Parse fails.
338 result = self._ParseAllowTrailing()
339 self.ConsumeCommentAndWhitespace()
340 if not self.IsDone():
341 raise GNError("Trailing input after parsing:\n " + self.input[self.cur:])
345 """Converts a whitespace-separated list of ident=literals to a dict.
347 See additional usage notes on FromGNArgs(), above.
350 GNError: Parse fails.
354 self.ReplaceImports()
355 self.ConsumeCommentAndWhitespace()
357 while not self.IsDone():
358 ident = self._ParseIdent()
359 self.ConsumeCommentAndWhitespace()
360 if self.input[self.cur] != '=':
361 raise GNError("Unexpected token: " + self.input[self.cur:])
363 self.ConsumeCommentAndWhitespace()
364 val = self._ParseAllowTrailing()
365 self.ConsumeCommentAndWhitespace()
370 def _ParseAllowTrailing(self):
371 """Internal version of Parse() that doesn't check for trailing stuff."""
372 self.ConsumeCommentAndWhitespace()
374 raise GNError("Expected input to parse.")
376 next_char = self.input[self.cur]
378 return self.ParseList()
379 elif next_char == '{':
380 return self.ParseScope()
381 elif _IsDigitOrMinus(next_char):
382 return self.ParseNumber()
383 elif next_char == '"':
384 return self.ParseString()
385 elif self._ConstantFollows('true'):
387 elif self._ConstantFollows('false'):
390 raise GNError("Unexpected token: " + self.input[self.cur:])
392 def _ParseIdent(self):
395 next_char = self.input[self.cur]
396 if not next_char.isalpha() and not next_char=='_':
397 raise GNError("Expected an identifier: " + self.input[self.cur:])
402 next_char = self.input[self.cur]
403 while next_char.isalpha() or next_char.isdigit() or next_char=='_':
406 next_char = self.input[self.cur]
410 def ParseNumber(self):
411 self.ConsumeCommentAndWhitespace()
413 raise GNError('Expected number but got nothing.')
417 # The first character can include a negative sign.
418 if not self.IsDone() and _IsDigitOrMinus(self.input[self.cur]):
420 while not self.IsDone() and self.input[self.cur].isdigit():
423 number_string = self.input[begin:self.cur]
424 if not len(number_string) or number_string == '-':
425 raise GNError('Not a valid number.')
426 return int(number_string)
428 def ParseString(self):
429 self.ConsumeCommentAndWhitespace()
431 raise GNError('Expected string but got nothing.')
433 if self.input[self.cur] != '"':
434 raise GNError('Expected string beginning in a " but got:\n ' +
435 self.input[self.cur:])
436 self.cur += 1 # Skip over quote.
439 while not self.IsDone() and self.input[self.cur] != '"':
440 if self.input[self.cur] == '\\':
441 self.cur += 1 # Skip over the backslash.
443 raise GNError('String ends in a backslash in:\n ' + self.input)
447 raise GNError('Unterminated string:\n ' + self.input[begin:])
450 self.cur += 1 # Consume trailing ".
452 return UnescapeGNString(self.input[begin:end])
455 self.ConsumeCommentAndWhitespace()
457 raise GNError('Expected list but got nothing.')
459 # Skip over opening '['.
460 if self.input[self.cur] != '[':
461 raise GNError('Expected [ for list but got:\n ' + self.input[self.cur:])
463 self.ConsumeCommentAndWhitespace()
465 raise GNError('Unterminated list:\n ' + self.input)
468 previous_had_trailing_comma = True
469 while not self.IsDone():
470 if self.input[self.cur] == ']':
471 self.cur += 1 # Skip over ']'.
474 if not previous_had_trailing_comma:
475 raise GNError('List items not separated by comma.')
477 list_result += [ self._ParseAllowTrailing() ]
478 self.ConsumeCommentAndWhitespace()
482 # Consume comma if there is one.
483 previous_had_trailing_comma = self.input[self.cur] == ','
484 if previous_had_trailing_comma:
487 self.ConsumeCommentAndWhitespace()
489 raise GNError('Unterminated list:\n ' + self.input)
491 def ParseScope(self):
492 self.ConsumeCommentAndWhitespace()
494 raise GNError('Expected scope but got nothing.')
496 # Skip over opening '{'.
497 if self.input[self.cur] != '{':
498 raise GNError('Expected { for scope but got:\n ' + self.input[self.cur:])
500 self.ConsumeCommentAndWhitespace()
502 raise GNError('Unterminated scope:\n ' + self.input)
505 while not self.IsDone():
506 if self.input[self.cur] == '}':
510 ident = self._ParseIdent()
511 self.ConsumeCommentAndWhitespace()
512 if self.input[self.cur] != '=':
513 raise GNError("Unexpected token: " + self.input[self.cur:])
515 self.ConsumeCommentAndWhitespace()
516 val = self._ParseAllowTrailing()
517 self.ConsumeCommentAndWhitespace()
518 scope_result[ident] = val
520 raise GNError('Unterminated scope:\n ' + self.input)
522 def _ConstantFollows(self, constant):
523 """Checks and maybe consumes a string constant at current input location.
526 constant: The string constant to check.
529 True if |constant| follows immediately at the current location in the
530 input. In this case, the string is consumed as a side effect. Otherwise,
531 returns False and the current position is unchanged.
533 end = self.cur + len(constant)
534 if end > len(self.input):
535 return False # Not enough room.
536 if self.input[self.cur:end] == constant:
542 def ReadBuildVars(output_directory):
543 """Parses $output_directory/build_vars.json into a dict."""
544 with open(os.path.join(output_directory, BUILD_VARS_FILENAME)) as f:
548 def CreateBuildCommand(output_directory):
549 """Returns [cmd, -C, output_directory], where |cmd| is auto{siso,ninja}."""
550 suffix = '.bat' if sys.platform.startswith('win32') else ''
551 # Prefer the version on PATH, but fallback to known version if PATH doesn't
552 # have one (e.g. on bots).
553 if not shutil.which(f'autoninja{suffix}'):
554 third_party_prefix = os.path.join(_CHROMIUM_ROOT, 'third_party')
555 ninja_prefix = os.path.join(third_party_prefix, 'ninja', '')
556 siso_prefix = os.path.join(third_party_prefix, 'siso', '')
557 # Also - bots configure reclient manually, and so do not use the "auto"
559 ninja_cmd = [f'{ninja_prefix}ninja{suffix}']
560 siso_cmd = [f'{siso_prefix}siso{suffix}', 'ninja']
562 ninja_cmd = [f'autoninja{suffix}']
563 siso_cmd = [f'autosiso{suffix}']
565 if output_directory and os.path.relpath(output_directory) != '.':
566 ninja_cmd += ['-C', output_directory]
567 siso_cmd += ['-C', output_directory]
568 siso_deps = os.path.exists(os.path.join(output_directory, '.siso_deps'))
569 ninja_deps = os.path.exists(os.path.join(output_directory, '.ninja_deps'))
570 if siso_deps and ninja_deps:
571 raise Exception('Found both .siso_deps and .ninja_deps in '
572 f'{output_directory}. Not sure which build tool to use. '
573 'Please delete one, or better, run "gn clean".')