1 #!/usr/bin/env vpython3
2 # Copyright 2019 The Chromium Authors
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Helper script used to manage locale-related files in Chromium.
8 This script is used to check, and potentially fix, many locale-related files
9 in your Chromium workspace, such as:
11 - GRIT input files (.grd) and the corresponding translations (.xtb).
13 - BUILD.gn files listing Android localized resource string resource .xml
14 generated by GRIT for all supported Chrome locales. These correspond to
15 <output> elements that use the type="android" attribute.
17 The --scan-dir <dir> option can be used to check for all files under a specific
18 directory, and the --fix-inplace option can be used to try fixing any file
19 that doesn't pass the check.
21 This can be very handy to avoid tedious and repetitive work when adding new
22 translations / locales to the Chrome code base, since this script can update
23 said input files for you.
25 Important note: checks and fix may fail on some input files. For example
26 remoting/resources/remoting_strings.grd contains an in-line comment element
27 inside its <outputs> section that breaks the script. The check will fail, and
28 trying to fix it too, but at least the file will not be modified.
41 # Assume this script is under build/
42 _SCRIPT_DIR = os.path.dirname(__file__)
43 _SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
44 _TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')
46 # Need to import android/gyp/util/resource_utils.py here.
47 sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))
49 from util import build_utils
50 from util import resource_utils
53 # This locale is the default and doesn't have translations.
54 _DEFAULT_LOCALE = 'en-US'
56 # Misc terminal codes to provide human friendly progress output.
57 _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
58 _CONSOLE_CODE_ERASE_LINE = '\x1b[K'
59 _CONSOLE_START_LINE = (
60 _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)
62 ##########################################################################
63 ##########################################################################
65 ##### G E N E R I C H E L P E R F U N C T I O N S
67 ##########################################################################
68 ##########################################################################
70 def _FixChromiumLangAttribute(lang):
71 """Map XML "lang" attribute values to Chromium locale names."""
72 _CHROMIUM_LANG_FIXES = {
73 'en': 'en-US', # For now, Chromium doesn't have an 'en' locale.
74 'iw': 'he', # 'iw' is the obsolete form of ISO 639-1 for Hebrew
75 'no': 'nb', # 'no' is used by the Translation Console for Norwegian (nb).
77 return _CHROMIUM_LANG_FIXES.get(lang, lang)
80 def _FixTranslationConsoleLocaleName(locale):
82 'nb': 'no', # Norwegian.
85 return _FIXES.get(locale, locale)
88 def _CompareLocaleLists(list_a, list_expected, list_name):
89 """Compare two lists of locale names. Print errors if they differ.
92 list_a: First list of locales.
93 list_expected: Second list of locales, as expected.
94 list_name: Name of list printed in error messages.
96 On success, return False. On error, print error messages and return True.
99 missing_locales = sorted(set(list_a) - set(list_expected))
101 errors.append('Missing locales: %s' % missing_locales)
103 extra_locales = sorted(set(list_expected) - set(list_a))
105 errors.append('Unexpected locales: %s' % extra_locales)
108 print('Errors in %s definition:' % list_name)
110 print(' %s\n' % error)
116 def _BuildIntervalList(input_list, predicate):
117 """Find ranges of contiguous list items that pass a given predicate.
120 input_list: An input list of items of any type.
121 predicate: A function that takes a list item and return True if it
124 A list of (start_pos, end_pos) tuples, where all items in
125 [start_pos, end_pos) pass the predicate.
128 size = len(input_list)
131 # Find first item in list that passes the predicate.
132 while start < size and not predicate(input_list[start]):
138 # Find first item in the rest of the list that does not pass the
141 while end < size and predicate(input_list[end]):
144 result.append((start, end))
148 def _SortListSubRange(input_list, start, end, key_func):
149 """Sort an input list's sub-range according to a specific key function.
152 input_list: An input list.
153 start: Sub-range starting position in list.
154 end: Sub-range limit position in list.
155 key_func: A function that extracts a sort key from a line.
157 A copy of |input_list|, with all items in [|start|, |end|) sorted
158 according to |key_func|.
160 result = input_list[:start]
162 for pos in xrange(start, end):
163 line = input_list[pos]
165 inputs.append((key, line))
167 for _, line in sorted(inputs):
170 result += input_list[end:]
174 def _SortElementsRanges(lines, element_predicate, element_key):
175 """Sort all elements of a given type in a list of lines by a given key.
179 element_predicate: predicate function to select elements to sort.
180 element_key: lambda returning a comparison key for each element that
181 passes the predicate.
183 A new list of input lines, with lines [start..end) sorted.
185 intervals = _BuildIntervalList(lines, element_predicate)
186 for start, end in intervals:
187 lines = _SortListSubRange(lines, start, end, element_key)
192 def _ProcessFile(input_file, locales, check_func, fix_func):
193 """Process a given input file, potentially fixing it.
196 input_file: Input file path.
197 locales: List of Chrome locales to consider / expect.
198 check_func: A lambda called to check the input file lines with
199 (input_lines, locales) argument. It must return an list of error
200 messages, or None on success.
201 fix_func: None, or a lambda called to fix the input file lines with
202 (input_lines, locales). It must return the new list of lines for
203 the input file, and may raise an Exception in case of error.
207 print('%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), end=' ')
209 with open(input_file) as f:
210 input_lines = f.readlines()
211 errors = check_func(input_file, input_lines, locales)
213 print('\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)))
216 input_lines = fix_func(input_file, input_lines, locales)
217 output = ''.join(input_lines)
218 with open(input_file, 'wt') as f:
220 print('Fixed %s.' % input_file)
221 except Exception as e: # pylint: disable=broad-except
222 print('Skipped %s: %s' % (input_file, e))
227 def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
228 """Scan a directory for files that match a given predicate.
231 scan_dir: A list of top-level directories to start scan in.
232 file_predicate: lambda function which is passed the file's base name
233 and returns True if its full path, relative to |scan_dir|, should be
234 passed in the result.
236 A list of file full paths.
239 for src_dir in scan_dirs:
240 for root, _, files in os.walk(src_dir):
241 result.extend(os.path.join(root, f) for f in files if file_predicate(f))
245 def _WriteFile(file_path, file_data):
246 """Write |file_data| to |file_path|."""
247 with open(file_path, 'w') as f:
251 def _FindGnExecutable():
252 """Locate the real GN executable used by this Chromium checkout.
254 This is needed because the depot_tools 'gn' wrapper script will look
255 for .gclient and other things we really don't need here.
258 Path of real host GN executable from current Chromium src/ checkout.
260 # Simply scan buildtools/*/gn and return the first one found so we don't
261 # have to guess the platform-specific sub-directory name (e.g. 'linux64'
262 # for 64-bit Linux machines).
263 buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
264 for subdir in os.listdir(buildtools_dir):
265 subdir_path = os.path.join(buildtools_dir, subdir)
266 if not os.path.isdir(subdir_path):
268 gn_path = os.path.join(subdir_path, 'gn')
269 if os.path.exists(gn_path):
274 def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
276 input_str = ', '.join(input_list)
277 while len(input_str) > available_width:
278 pos = input_str.rfind(',', 0, available_width)
279 result.append(input_str[:pos + 1])
280 input_str = input_str[pos + 1:].lstrip()
281 if trailing_comma and input_str:
283 result.append(input_str)
287 class _PrettyPrintListAsLinesTest(unittest.TestCase):
289 def test_empty_list(self):
290 self.assertListEqual([''], _PrettyPrintListAsLines([], 10))
292 def test_wrapping(self):
293 input_list = ['foo', 'bar', 'zoo', 'tool']
294 self.assertListEqual(
295 _PrettyPrintListAsLines(input_list, 8),
296 ['foo,', 'bar,', 'zoo,', 'tool'])
297 self.assertListEqual(
298 _PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
299 self.assertListEqual(
300 _PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])
302 def test_trailing_comma(self):
303 input_list = ['foo', 'bar', 'zoo', 'tool']
304 self.assertListEqual(
305 _PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
306 ['foo,', 'bar,', 'zoo,', 'tool,'])
307 self.assertListEqual(
308 _PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
309 ['foo, bar,', 'zoo, tool,'])
310 self.assertListEqual(
311 _PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
312 ['foo, bar, zoo, tool,'])
315 ##########################################################################
316 ##########################################################################
318 ##### L O C A L E S L I S T S
320 ##########################################################################
321 ##########################################################################
323 # Various list of locales that will be extracted from build/config/locales.gni
324 # Do not use these directly, use ChromeLocales(), and IosUnsupportedLocales()
325 # instead to access these lists.
326 _INTERNAL_CHROME_LOCALES = []
327 _INTERNAL_IOS_UNSUPPORTED_LOCALES = []
331 """Return the list of all locales supported by Chrome."""
332 if not _INTERNAL_CHROME_LOCALES:
333 _ExtractAllChromeLocalesLists()
334 return _INTERNAL_CHROME_LOCALES
337 def IosUnsupportedLocales():
338 """Return the list of locales that are unsupported on iOS."""
339 if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
340 _ExtractAllChromeLocalesLists()
341 return _INTERNAL_IOS_UNSUPPORTED_LOCALES
344 def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
345 """Populate an empty directory with a tiny set of working GN config files.
347 This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
348 to generate files containing the locales list. This takes about 300ms on
349 a decent machine, instead of more than 5 seconds when running the equivalent
350 commands from a real Chromium workspace, which requires regenerating more
354 work_dir: target working directory.
355 out_subdir_name: Name of output sub-directory.
357 Full path of output directory created inside |work_dir|.
359 # Create top-level .gn file that must point to the BUILDCONFIG.gn.
360 _WriteFile(os.path.join(work_dir, '.gn'),
361 'buildconfig = "//BUILDCONFIG.gn"\n')
362 # Create BUILDCONFIG.gn which must set a default toolchain. Also add
363 # all variables that may be used in locales.gni in a declare_args() block.
365 os.path.join(work_dir, 'BUILDCONFIG.gn'),
366 r'''set_default_toolchain("toolchain")
373 # Create fake toolchain required by BUILDCONFIG.gn.
374 os.mkdir(os.path.join(work_dir, 'toolchain'))
375 _WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
376 r'''toolchain("toolchain") {
378 command = "touch {{output}}" # Required by action()
383 # Create top-level BUILD.gn, GN requires at least one target to build so do
384 # that with a fake action which will never be invoked. Also write the locales
385 # to misc files in the output directory.
387 os.path.join(work_dir, 'BUILD.gn'), r'''import("//locales.gni")
389 action("create_foo") { # fake action to avoid GN complaints.
390 script = "//build/create_foo.py"
392 outputs = [ "$target_out_dir/$target_name" ]
395 # Write the locales lists to files in the output directory.
396 _filename = root_build_dir + "/foo"
397 write_file(_filename + ".locales", locales, "json")
398 write_file(_filename + ".ios_unsupported_locales",
399 ios_unsupported_locales,
403 # Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
404 shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
405 os.path.join(work_dir, 'locales.gni'))
407 # Create output directory.
408 out_path = os.path.join(work_dir, out_subdir_name)
411 # And ... we're good.
415 # Set this global variable to the path of a given temporary directory
416 # before calling _ExtractAllChromeLocalesLists() if you want to debug
417 # the locales list extraction process.
418 _DEBUG_LOCALES_WORK_DIR = None
421 def _ReadJsonList(file_path):
422 """Read a JSON file that must contain a list, and return it."""
423 with open(file_path) as f:
425 assert isinstance(data, list), "JSON file %s is not a list!" % file_path
426 return [item.encode('utf8') for item in data]
429 def _ExtractAllChromeLocalesLists():
430 with build_utils.TempDir() as tmp_path:
431 if _DEBUG_LOCALES_WORK_DIR:
432 tmp_path = _DEBUG_LOCALES_WORK_DIR
433 build_utils.DeleteDirectory(tmp_path)
434 build_utils.MakeDirectory(tmp_path)
436 out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')
438 # NOTE: The file suffixes used here should be kept in sync with
439 # build/config/locales.gni
440 gn_executable = _FindGnExecutable()
442 subprocess.check_output(
443 [gn_executable, 'gen', out_path, '--root=' + tmp_path])
444 except subprocess.CalledProcessError as e:
448 global _INTERNAL_CHROME_LOCALES
449 _INTERNAL_CHROME_LOCALES = _ReadJsonList(
450 os.path.join(out_path, 'foo.locales'))
452 global _INTERNAL_IOS_UNSUPPORTED_LOCALES
453 _INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
454 os.path.join(out_path, 'foo.ios_unsupported_locales'))
457 ##########################################################################
458 ##########################################################################
460 ##### G R D H E L P E R F U N C T I O N S
462 ##########################################################################
463 ##########################################################################
467 # Even though .grd files are XML, an xml parser library is not used in order
468 # to preserve the original file's structure after modification. ElementTree
469 # tends to re-order attributes in each element when re-writing an XML
470 # document tree, which is undesirable here.
472 # Thus simple line-based regular expression matching is used instead.
475 # Misc regular expressions used to match elements and their attributes.
476 _RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
477 _RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
478 _RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
479 _RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
480 _RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
481 _RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')
485 def _IsGritInputFile(input_file):
486 """Returns True iff this is a GRIT input file."""
487 return input_file.endswith('.grd')
490 def _GetXmlLangAttribute(xml_line):
491 """Extract the lang attribute value from an XML input line."""
492 m = _RE_LANG_ATTRIBUTE.search(xml_line)
498 class _GetXmlLangAttributeTest(unittest.TestCase):
504 '<something lang="foo bar" />': 'foo bar',
505 '<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
508 def test_GetXmlLangAttribute(self):
509 for test_line, expected in self.TEST_DATA.items():
510 self.assertEquals(_GetXmlLangAttribute(test_line), expected)
513 def _SortGrdElementsRanges(grd_lines, element_predicate):
514 """Sort all .grd elements of a given type by their lang attribute."""
515 return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)
518 def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
519 """Check the element 'lang' attributes in specific .grd lines range.
521 This really checks the following:
522 - Each item has a correct 'lang' attribute.
523 - There are no duplicated lines for the same 'lang' attribute.
524 - That there are no extra locales that Chromium doesn't want.
525 - That no wanted locale is missing.
528 grd_lines: Input .grd lines.
529 start: Sub-range start position in input line list.
530 end: Sub-range limit position in input line list.
531 wanted_locales: Set of wanted Chromium locale names.
533 List of error message strings for this input. Empty on success.
537 for pos in xrange(start, end):
538 line = grd_lines[pos]
539 lang = _GetXmlLangAttribute(line)
541 errors.append('%d: Missing "lang" attribute in <output> element' % pos +
544 cr_locale = _FixChromiumLangAttribute(lang)
545 if cr_locale in locales:
547 '%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
548 locales.add(cr_locale)
550 extra_locales = locales.difference(wanted_locales)
552 errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
553 sorted(extra_locales)))
555 missing_locales = wanted_locales.difference(locales)
557 errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
558 sorted(missing_locales)))
563 ##########################################################################
564 ##########################################################################
566 ##### G R D A N D R O I D O U T P U T S
568 ##########################################################################
569 ##########################################################################
571 def _IsGrdAndroidOutputLine(line):
572 """Returns True iff this is an Android-specific <output> line."""
573 m = _RE_OUTPUT_ELEMENT.search(line)
575 return 'type="android"' in m.group(1)
578 assert _IsGrdAndroidOutputLine(' <output type="android"/>')
580 # Many of the functions below have unused arguments due to genericity.
581 # pylint: disable=unused-argument
583 def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
585 """Check all <output> elements in specific input .grd lines range.
587 This really checks the following:
588 - Filenames exist for each listed locale.
589 - Filenames are well-formed.
592 grd_lines: Input .grd lines.
593 start: Sub-range start position in input line list.
594 end: Sub-range limit position in input line list.
595 wanted_locales: Set of wanted Chromium locale names.
597 List of error message strings for this input. Empty on success.
600 for pos in xrange(start, end):
601 line = grd_lines[pos]
602 lang = _GetXmlLangAttribute(line)
605 cr_locale = _FixChromiumLangAttribute(lang)
607 m = _RE_FILENAME_ATTRIBUTE.search(line)
609 errors.append('%d: Missing filename attribute in <output> element' % pos +
612 filename = m.group(1)
613 if not filename.endswith('.xml'):
615 '%d: Filename should end with ".xml": %s' % (pos + 1, filename))
617 dirname = os.path.basename(os.path.dirname(filename))
618 prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
619 if cr_locale != _DEFAULT_LOCALE else 'values')
620 if dirname != prefix:
622 '%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))
627 def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
628 """Check all <output> elements related to Android.
631 grd_file: Input .grd file path.
632 grd_lines: List of input .grd lines.
633 wanted_locales: set of wanted Chromium locale names.
635 List of error message strings. Empty on success.
637 intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
639 for start, end in intervals:
640 errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
641 errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
646 def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
647 """Fix an input .grd line by adding missing Android outputs.
650 grd_file: Input .grd file path.
651 grd_lines: Input .grd line list.
652 wanted_locales: set of Chromium locale names.
654 A new list of .grd lines, containing new <output> elements when needed
655 for locales from |wanted_locales| that were not part of the input.
657 intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
658 for start, end in reversed(intervals):
660 for pos in xrange(start, end):
661 lang = _GetXmlLangAttribute(grd_lines[pos])
662 locale = _FixChromiumLangAttribute(lang)
665 missing_locales = wanted_locales.difference(locales)
666 if not missing_locales:
670 src_lang_attribute = 'lang="%s"' % src_locale
672 for pos in xrange(start, end):
673 if src_lang_attribute in grd_lines[pos]:
674 src_line = grd_lines[pos]
679 'Cannot find <output> element with "%s" lang attribute' % src_locale)
682 for locale in missing_locales:
683 android_locale = resource_utils.ToAndroidLocaleName(locale)
684 dst_line = src_line.replace(
685 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
686 'values-%s/' % src_locale, 'values-%s/' % android_locale)
687 grd_lines.insert(line_count, dst_line)
690 # Sort the new <output> elements.
691 return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)
694 ##########################################################################
695 ##########################################################################
697 ##### G R D T R A N S L A T I O N S
699 ##########################################################################
700 ##########################################################################
703 def _IsTranslationGrdOutputLine(line):
704 """Returns True iff this is an output .xtb <file> element."""
705 m = _RE_TRANSLATION_ELEMENT.search(line)
709 class _IsTranslationGrdOutputLineTest(unittest.TestCase):
711 def test_GrdTranslationOutputLines(self):
712 _VALID_INPUT_LINES = [
713 '<file path="foo/bar.xtb" />',
714 '<file path="foo/bar.xtb"/>',
715 '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
716 '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
717 ' <file path="translations/aw_strings_ar.xtb" lang="ar" />',
719 _INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']
721 for line in _VALID_INPUT_LINES:
723 _IsTranslationGrdOutputLine(line),
724 '_IsTranslationGrdOutputLine() returned False for [%s]' % line)
726 for line in _INVALID_INPUT_LINES:
728 _IsTranslationGrdOutputLine(line),
729 '_IsTranslationGrdOutputLine() returned True for [%s]' % line)
732 def _CheckGrdTranslationElementRange(grd_lines, start, end,
734 """Check all <translations> sub-elements in specific input .grd lines range.
736 This really checks the following:
737 - Each item has a 'path' attribute.
738 - Each such path value ends up with '.xtb'.
741 grd_lines: Input .grd lines.
742 start: Sub-range start position in input line list.
743 end: Sub-range limit position in input line list.
744 wanted_locales: Set of wanted Chromium locale names.
746 List of error message strings for this input. Empty on success.
749 for pos in xrange(start, end):
750 line = grd_lines[pos]
751 lang = _GetXmlLangAttribute(line)
754 m = _RE_PATH_ATTRIBUTE.search(line)
756 errors.append('%d: Missing path attribute in <file> element' % pos +
759 filename = m.group(1)
760 if not filename.endswith('.xtb'):
762 '%d: Path should end with ".xtb": %s' % (pos + 1, filename))
767 def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
768 """Check all <file> elements that correspond to an .xtb output file.
771 grd_file: Input .grd file path.
772 grd_lines: List of input .grd lines.
773 wanted_locales: set of wanted Chromium locale names.
775 List of error message strings. Empty on success.
777 wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
778 intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
780 for start, end in intervals:
781 errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
782 errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
787 # Regular expression used to replace the lang attribute inside .xtb files.
788 _RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')
791 def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
792 """Create a fake .xtb file.
795 src_xtb_path: Path to source .xtb file to copy from.
796 dst_xtb_path: Path to destination .xtb file to write to.
797 dst_locale: Destination locale, the lang attribute in the source file
798 will be substituted with this value before its lines are written
799 to the destination file.
801 with open(src_xtb_path) as f:
802 src_xtb_lines = f.readlines()
804 def replace_xtb_lang_attribute(line):
805 m = _RE_TRANSLATIONBUNDLE.search(line)
808 return line[:m.start(1)] + dst_locale + line[m.end(1):]
810 dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
811 with build_utils.AtomicOutput(dst_xtb_path) as tmp:
812 tmp.writelines(dst_xtb_lines)
815 def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
816 """Fix an input .grd line by adding missing Android outputs.
818 This also creates fake .xtb files from the one provided for 'en-GB'.
821 grd_file: Input .grd file path.
822 grd_lines: Input .grd line list.
823 wanted_locales: set of Chromium locale names.
825 A new list of .grd lines, containing new <output> elements when needed
826 for locales from |wanted_locales| that were not part of the input.
828 wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
829 intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
830 for start, end in reversed(intervals):
832 for pos in xrange(start, end):
833 lang = _GetXmlLangAttribute(grd_lines[pos])
834 locale = _FixChromiumLangAttribute(lang)
837 missing_locales = wanted_locales.difference(locales)
838 if not missing_locales:
842 src_lang_attribute = 'lang="%s"' % src_locale
844 for pos in xrange(start, end):
845 if src_lang_attribute in grd_lines[pos]:
846 src_line = grd_lines[pos]
851 'Cannot find <file> element with "%s" lang attribute' % src_locale)
853 src_path = os.path.join(
854 os.path.dirname(grd_file),
855 _RE_PATH_ATTRIBUTE.search(src_line).group(1))
858 for locale in missing_locales:
859 dst_line = src_line.replace(
860 'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
861 '_%s.xtb' % src_locale, '_%s.xtb' % locale)
862 grd_lines.insert(line_count, dst_line)
865 dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
866 _CreateFakeXtbFileFrom(src_path, dst_path, locale)
869 # Sort the new <output> elements.
870 return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)
873 ##########################################################################
874 ##########################################################################
876 ##### G N A N D R O I D O U T P U T S
878 ##########################################################################
879 ##########################################################################
881 _RE_GN_VALUES_LIST_LINE = re.compile(
882 r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')
884 def _IsBuildGnInputFile(input_file):
885 """Returns True iff this is a BUILD.gn file."""
886 return os.path.basename(input_file) == 'BUILD.gn'
889 def _GetAndroidGnOutputLocale(line):
890 """Check a GN list, and return its Android locale if it is an output .xml"""
891 m = _RE_GN_VALUES_LIST_LINE.match(line)
895 if m.group(1): # First group is optional and contains group 2.
898 return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)
901 def _IsAndroidGnOutputLine(line):
902 """Returns True iff this is an Android-specific localized .xml output."""
903 return _GetAndroidGnOutputLocale(line) != None
906 def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
907 """Check that a range of GN lines corresponds to localized strings.
909 Special case: Some BUILD.gn files list several non-localized .xml files
910 that should be ignored by this function, e.g. in
911 components/cronet/android/BUILD.gn, the following appears:
915 "sample/res/layout/activity_main.xml",
916 "sample/res/layout/dialog_url.xml",
917 "sample/res/values/dimens.xml",
918 "sample/res/values/strings.xml",
922 These are non-localized strings, and should be ignored. This function is
923 used to detect them quickly.
925 for pos in xrange(start, end):
926 if not 'values/' in gn_lines[pos]:
931 def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
932 if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
937 for pos in xrange(start, end):
939 android_locale = _GetAndroidGnOutputLocale(line)
940 assert android_locale != None
941 cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
942 if cr_locale in locales:
943 errors.append('%s: Redefinition of output for "%s" locale' %
944 (pos + 1, android_locale))
945 locales.add(cr_locale)
947 extra_locales = locales.difference(wanted_locales)
949 errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
950 sorted(extra_locales)))
952 missing_locales = wanted_locales.difference(locales)
954 errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
955 sorted(missing_locales)))
960 def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
961 intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
963 for start, end in intervals:
964 errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
968 def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
969 intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
970 # NOTE: Since this may insert new lines to each interval, process the
971 # list in reverse order to maintain valid (start,end) positions during
973 for start, end in reversed(intervals):
974 if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
978 for pos in xrange(start, end):
979 lang = _GetAndroidGnOutputLocale(gn_lines[pos])
980 locale = resource_utils.ToChromiumLocaleName(lang)
983 missing_locales = wanted_locales.difference(locales)
984 if not missing_locales:
988 src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
990 for pos in xrange(start, end):
991 if src_values in gn_lines[pos]:
992 src_line = gn_lines[pos]
997 'Cannot find output list item with "%s" locale' % src_locale)
1000 for locale in missing_locales:
1001 if locale == _DEFAULT_LOCALE:
1002 dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
1004 dst_line = src_line.replace(
1005 'values-%s/' % src_locale,
1006 'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
1007 gn_lines.insert(line_count, dst_line)
1010 gn_lines = _SortListSubRange(
1011 gn_lines, start, line_count,
1012 lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))
1017 ##########################################################################
1018 ##########################################################################
1020 ##### T R A N S L A T I O N E X P E C T A T I O N S
1022 ##########################################################################
1023 ##########################################################################
1025 _EXPECTATIONS_FILENAME = 'translation_expectations.pyl'
1027 # Technical note: the format of translation_expectations.pyl
1028 # is a 'Python literal', which defines a python dictionary, so should
1029 # be easy to parse. However, when modifying it, care should be taken
1030 # to respect the line comments and the order of keys within the text
1034 def _ReadPythonLiteralFile(pyl_path):
1035 """Read a .pyl file into a Python data structure."""
1036 with open(pyl_path) as f:
1037 pyl_content = f.read()
1038 # Evaluate as a Python data structure, use an empty global
1039 # and local dictionary.
1040 return eval(pyl_content, dict(), dict())
1043 def _UpdateLocalesInExpectationLines(pyl_lines,
1045 available_width=79):
1046 """Update the locales list(s) found in an expectations file.
1049 pyl_lines: Iterable of input lines from the file.
1050 wanted_locales: Set or list of new locale names.
1051 available_width: Optional, number of character colums used
1052 to word-wrap the new list items.
1054 New list of updated lines.
1056 locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
1058 line_count = len(pyl_lines)
1060 DICT_START = '"languages": ['
1061 while line_num < line_count:
1062 line = pyl_lines[line_num]
1065 # Look for start of "languages" dictionary.
1066 pos = line.find(DICT_START)
1071 start_line = line_num
1072 # Skip over all lines from the list.
1073 while (line_num < line_count and
1074 not pyl_lines[line_num].rstrip().endswith('],')):
1078 if line_num == line_count:
1079 raise Exception('%d: Missing list termination!' % start_line)
1081 # Format the new list according to the new margin.
1082 locale_width = available_width - (start_margin + 2)
1083 locale_lines = _PrettyPrintListAsLines(
1084 locales_list, locale_width, trailing_comma=True)
1085 for locale_line in locale_lines:
1086 result.append(' ' * (start_margin + 2) + locale_line)
1087 result.append(' ' * start_margin + '],')
1093 class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):
1095 def test_simple(self):
1098 # This comment should be preserved
1099 # 23456789012345678901234567890123456789
1103 "aa", "bb", "cc", "dd", "ee",
1104 "ff", "gg", "hh", "ii", "jj",
1107 # Example with bad indentation in input.
1110 "aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
1115 expected_text = r'''
1116 # This comment should be preserved
1117 # 23456789012345678901234567890123456789
1121 "A2", "AA", "BB", "CC", "DD",
1122 "E2", "EE", "FF", "GG", "HH",
1123 "I2", "II", "JJ", "KK",
1126 # Example with bad indentation in input.
1129 "A2", "AA", "BB", "CC", "DD",
1130 "E2", "EE", "FF", "GG", "HH",
1131 "I2", "II", "JJ", "KK",
1136 input_lines = input_text.splitlines()
1138 'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
1141 expected_lines = expected_text.splitlines()
1142 self.assertListEqual(
1143 _UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
1146 def test_missing_list_termination(self):
1149 "aa", "bb", "cc", "dd"
1151 with self.assertRaises(Exception) as cm:
1152 _UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)
1154 self.assertEqual(str(cm.exception), '2: Missing list termination!')
1157 def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
1158 """Update all locales listed in a given expectations file.
1161 pyl_path: Path to .pyl file to update.
1162 wanted_locales: List of locales that need to be written to
1166 _FixTranslationConsoleLocaleName(locale)
1167 for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
1170 with open(pyl_path) as f:
1171 input_lines = [l.rstrip() for l in f.readlines()]
1173 updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
1174 with build_utils.AtomicOutput(pyl_path) as f:
1175 f.writelines('\n'.join(updated_lines) + '\n')
1178 ##########################################################################
1179 ##########################################################################
1181 ##### C H E C K E V E R Y T H I N G
1183 ##########################################################################
1184 ##########################################################################
1186 # pylint: enable=unused-argument
1189 def _IsAllInputFile(input_file):
1190 return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)
1193 def _CheckAllFiles(input_file, input_lines, wanted_locales):
1195 if _IsGritInputFile(input_file):
1196 errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
1197 errors += _CheckGrdAndroidOutputElements(
1198 input_file, input_lines, wanted_locales)
1199 elif _IsBuildGnInputFile(input_file):
1200 errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
1204 def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
1205 if _IsGritInputFile(input_file):
1206 lines = _AddMissingLocalesInGrdTranslations(
1207 input_file, input_lines, wanted_locales)
1208 lines = _AddMissingLocalesInGrdAndroidOutputs(
1209 input_file, lines, wanted_locales)
1210 elif _IsBuildGnInputFile(input_file):
1211 lines = _AddMissingLocalesInGnAndroidOutputs(
1212 input_file, input_lines, wanted_locales)
1216 ##########################################################################
1217 ##########################################################################
1219 ##### C O M M A N D H A N D L I N G
1221 ##########################################################################
1222 ##########################################################################
1224 class _Command(object):
1225 """A base class for all commands recognized by this script.
1227 Usage is the following:
1228 1) Derived classes must re-define the following class-based fields:
1229 - name: Command name (e.g. 'list-locales')
1230 - description: Command short description.
1231 - long_description: Optional. Command long description.
1232 NOTE: As a convenience, if the first character is a newline,
1233 it will be omitted in the help output.
1235 2) Derived classes for commands that take arguments should override
1236 RegisterExtraArgs(), which receives a corresponding argparse
1237 sub-parser as argument.
1239 3) Derived classes should implement a Run() command, which can read
1240 the current arguments from self.args.
1244 long_description = None
1250 def RegisterExtraArgs(self, subparser):
1253 def RegisterArgs(self, parser):
1254 subp = parser.add_parser(
1255 self.name, help=self.description,
1256 description=self.long_description or self.description,
1257 formatter_class=argparse.RawDescriptionHelpFormatter)
1259 subp.set_defaults(command=self)
1260 group = subp.add_argument_group('%s arguments' % self.name)
1261 self.RegisterExtraArgs(group)
1263 def ProcessArgs(self, args):
1267 class _ListLocalesCommand(_Command):
1268 """Implement the 'list-locales' command to list locale lists of interest."""
1269 name = 'list-locales'
1270 description = 'List supported Chrome locales'
1271 long_description = r'''
1272 List locales of interest, by default this prints all locales supported by
1273 Chrome, but `--type=ios_unsupported` can be used for the list of locales
1276 These values are extracted directly from build/config/locales.gni.
1278 Additionally, use the --as-json argument to print the list as a JSON list,
1279 instead of the default format (which is a space-separated list of locale names).
1282 # Maps type argument to a function returning the corresponding locales list.
1284 'all': ChromeLocales,
1285 'ios_unsupported': IosUnsupportedLocales,
1288 def RegisterExtraArgs(self, group):
1291 action='store_true',
1292 help='Output as JSON list.')
1295 choices=tuple(self.TYPE_MAP.viewkeys()),
1297 help='Select type of locale list to print.')
1300 locale_list = self.TYPE_MAP[self.args.type]()
1301 if self.args.as_json:
1302 print('[%s]' % ", ".join("'%s'" % loc for loc in locale_list))
1304 print(' '.join(locale_list))
1307 class _CheckInputFileBaseCommand(_Command):
1308 """Used as a base for other _Command subclasses that check input files.
1310 Subclasses should also define the following class-level variables:
1313 A predicate that receives a file name (not path) and return True if it
1314 should be selected for inspection. Used when scanning directories with
1319 Two functions passed as parameters to _ProcessFile(), see relevant
1320 documentation in this function's definition.
1322 select_file_func = None
1326 def RegisterExtraArgs(self, group):
1330 help='Optional directory to scan for input files recursively.')
1334 help='Input file(s) to check.')
1337 action='store_true',
1338 help='Try to fix the files in-place too.')
1341 help='Space-separated list of additional locales to use')
1347 input_files = args.input
1349 input_files.extend(_ScanDirectoriesForFiles(
1350 args.scan_dir, self.select_file_func.__func__))
1351 locales = ChromeLocales()
1352 if args.add_locales:
1353 locales.extend(args.add_locales.split(' '))
1355 locales = set(locales)
1357 for input_file in input_files:
1358 _ProcessFile(input_file,
1360 self.check_func.__func__,
1361 self.fix_func.__func__ if args.fix_inplace else None)
1362 print('%sDone.' % (_CONSOLE_START_LINE))
1365 class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
1366 name = 'check-grd-android-outputs'
1368 'Check the Android resource (.xml) files outputs in GRIT input files.')
1369 long_description = r'''
1370 Check the Android .xml files outputs in one or more input GRIT (.grd) files
1371 for the following conditions:
1373 - Each item has a correct 'lang' attribute.
1374 - There are no duplicated lines for the same 'lang' attribute.
1375 - That there are no extra locales that Chromium doesn't want.
1376 - That no wanted locale is missing.
1377 - Filenames exist for each listed locale.
1378 - Filenames are well-formed.
1380 select_file_func = _IsGritInputFile
1381 check_func = _CheckGrdAndroidOutputElements
1382 fix_func = _AddMissingLocalesInGrdAndroidOutputs
1385 class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
1386 name = 'check-grd-translations'
1388 'Check the translation (.xtb) files outputted by .grd input files.')
1389 long_description = r'''
1390 Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
1391 for the following conditions:
1393 - Each item has a correct 'lang' attribute.
1394 - There are no duplicated lines for the same 'lang' attribute.
1395 - That there are no extra locales that Chromium doesn't want.
1396 - That no wanted locale is missing.
1397 - Each item has a 'path' attribute.
1398 - Each such path value ends up with '.xtb'.
1400 select_file_func = _IsGritInputFile
1401 check_func = _CheckGrdTranslations
1402 fix_func = _AddMissingLocalesInGrdTranslations
1405 class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
1406 name = 'check-gn-android-outputs'
1407 description = 'Check the Android .xml file lists in GN build files.'
1408 long_description = r'''
1409 Check one or more BUILD.gn file, looking for lists of Android resource .xml
1410 files, and checking that:
1412 - There are no duplicated output files in the list.
1413 - Each output file belongs to a wanted Chromium locale.
1414 - There are no output files for unwanted Chromium locales.
1416 select_file_func = _IsBuildGnInputFile
1417 check_func = _CheckGnAndroidOutputs
1418 fix_func = _AddMissingLocalesInGnAndroidOutputs
1421 class _CheckAllCommand(_CheckInputFileBaseCommand):
1423 description = 'Check everything.'
1424 long_description = 'Equivalent to calling all other check-xxx commands.'
1425 select_file_func = _IsAllInputFile
1426 check_func = _CheckAllFiles
1427 fix_func = _AddMissingLocalesInAllFiles
1430 class _UpdateExpectationsCommand(_Command):
1431 name = 'update-expectations'
1432 description = 'Update translation expectations file.'
1433 long_description = r'''
1434 Update %s files to match the current list of locales supported by Chromium.
1435 This is especially useful to add new locales before updating any GRIT or GN
1436 input file with the --add-locales option.
1437 ''' % _EXPECTATIONS_FILENAME
1439 def RegisterExtraArgs(self, group):
1442 help='Space-separated list of additional locales to use.')
1445 locales = ChromeLocales()
1446 add_locales = self.args.add_locales
1448 locales.extend(add_locales.split(' '))
1450 expectation_paths = [
1451 'tools/gritsettings/translation_expectations.pyl',
1452 'clank/tools/translation_expectations.pyl',
1454 missing_expectation_files = []
1455 for path in enumerate(expectation_paths):
1456 file_path = os.path.join(_TOP_SRC_DIR, path)
1457 if not os.path.exists(file_path):
1458 missing_expectation_files.append(file_path)
1460 _UpdateLocalesInExpectationFile(file_path, locales)
1462 if missing_expectation_files:
1463 sys.stderr.write('WARNING: Missing file(s): %s\n' %
1464 (', '.join(missing_expectation_files)))
1467 class _UnitTestsCommand(_Command):
1469 description = 'Run internal unit-tests for this script'
1471 def RegisterExtraArgs(self, group):
1473 '-v', '--verbose', action='count', help='Increase test verbosity.')
1474 group.add_argument('args', nargs=argparse.REMAINDER)
1477 argv = [_SCRIPT_NAME] + self.args.args
1478 unittest.main(argv=argv, verbosity=self.args.verbose)
1481 # List of all commands supported by this script.
1483 _ListLocalesCommand,
1484 _CheckGrdAndroidOutputsCommand,
1485 _CheckGrdTranslationsCommand,
1486 _CheckGnAndroidOutputsCommand,
1488 _UpdateExpectationsCommand,
1494 parser = argparse.ArgumentParser(
1495 description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
1497 subparsers = parser.add_subparsers()
1498 commands = [clazz() for clazz in _COMMANDS]
1499 for command in commands:
1500 command.RegisterArgs(subparsers)
1505 args = parser.parse_args(argv)
1506 args.command.ProcessArgs(args)
1510 if __name__ == "__main__":