3 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Move config options from headers to defconfig files.
11 Since Kconfig was introduced to U-Boot, we have worked on moving
12 config options from headers to Kconfig (defconfig).
14 This tool intends to help this tremendous work.
20 This tool takes one input file. (let's say 'recipe' file here.)
21 The recipe describes the list of config options you want to move.
22 Each line takes the form:
23 <config_name> <type> <default>
24 (the fields must be separated with whitespaces.)
26 <config_name> is the name of config option.
28 <type> is the type of the option. It must be one of bool, tristate,
31 <default> is the default value of the option. It must be appropriate
32 value corresponding to the option type. It must be either y or n for
33 the bool type. Tristate options can also take m (although U-Boot has
34 not supported the module feature).
36 You can add two or more lines in the recipe file, so you can move
37 multiple options at once.
39 Let's say, for example, you want to move CONFIG_CMD_USB and
42 The type should be bool, hex, respectively. So, the recipe file
43 should look like this:
47 CONFIG_SYS_TEXT_BASE hex 0x00000000
49 Next you must edit the Kconfig to add the menu entries for the configs
52 And then run this tool giving the file name of the recipe
54 $ tools/moveconfig.py recipe
56 The tool walks through all the defconfig files to move the config
57 options specified by the recipe file.
59 The log is also displayed on the terminal.
61 Each line is printed in the format
62 <defconfig_name> : <action>
64 <defconfig_name> is the name of the defconfig
65 (without the suffix _defconfig).
67 <action> shows what the tool did for that defconfig.
68 It looks like one of the followings:
71 This config option was moved to the defconfig
73 - Default value 'CONFIG_...'. Do nothing.
74 The value of this option is the same as default.
75 We do not have to add it to the defconfig.
77 - 'CONFIG_...' already exists in Kconfig. Do nothing.
78 This config option is already defined in Kconfig.
79 We do not need/want to touch it.
81 - Undefined. Do nothing.
82 This config option was not found in the config header.
85 - Failed to process. Skip.
86 An error occurred during processing this defconfig. Skipped.
87 (If -e option is passed, the tool exits immediately on error.)
89 Finally, you will be asked, Clean up headers? [y/n]:
91 If you say 'y' here, the unnecessary config defines are removed
92 from the config headers (include/configs/*.h).
93 It just uses the regex method, so you should not rely on it.
94 Just in case, please do 'git diff' to see what happened.
100 This tool runs configuration and builds include/autoconf.mk for every
101 defconfig. The config options defined in Kconfig appear in the .config
102 file (unless they are hidden because of unmet dependency.)
103 On the other hand, the config options defined by board headers are seen
104 in include/autoconf.mk. The tool looks for the specified options in both
105 of them to decide the appropriate action for the options. If the option
106 is found in the .config or the value is the same as the specified default,
107 the option does not need to be touched. If the option is found in
108 include/autoconf.mk, but not in the .config, and the value is different
109 from the default, the tools adds the option to the defconfig.
111 For faster processing, this tool handles multi-threading. It creates
112 separate build directories where the out-of-tree build is run. The
113 temporary build directories are automatically created and deleted as
114 needed. The number of threads are chosen based on the number of the CPU
115 cores of your system although you can change it via -j (--jobs) option.
121 Appropriate toolchain are necessary to generate include/autoconf.mk
122 for all the architectures supported by U-Boot. Most of them are available
123 at the kernel.org site, some are not provided by kernel.org.
125 The default per-arch CROSS_COMPILE used by this tool is specified by
126 the list below, CROSS_COMPILE. You may wish to update the list to
127 use your own. Instead of modifying the list directly, you can give
128 them via environments.
135 Surround each portion of the log with escape sequences to display it
136 in color on the terminal.
139 Specify a file containing a list of defconfigs to move
142 Peform a trial run that does not make any changes. It is useful to
143 see what is going to happen before one actually runs it.
146 Exit immediately if Make exits with a non-zero status while processing
150 Specify the number of threads to run simultaneously. If not specified,
151 the number of threads is the same as the number of CPU cores.
153 To see the complete list of supported options, run
155 $ tools/moveconfig.py -h
160 import multiprocessing
170 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
173 # Here is the list of cross-tools I use.
174 # Most of them are available at kernel.org
175 # (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the followings:
176 # arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases
177 # blackfin: http://sourceforge.net/projects/adi-toolchain/files/
178 # nds32: http://osdk.andestech.com/packages/
179 # nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545
180 # sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu
183 'aarch64': 'aarch64-linux-',
184 'arm': 'arm-unknown-linux-gnueabi-',
185 'avr32': 'avr32-linux-',
186 'blackfin': 'bfin-elf-',
187 'm68k': 'm68k-linux-',
188 'microblaze': 'microblaze-linux-',
189 'mips': 'mips-linux-',
190 'nds32': 'nds32le-linux-',
191 'nios2': 'nios2-linux-gnu-',
192 'openrisc': 'or32-linux-',
193 'powerpc': 'powerpc-linux-',
194 'sh': 'sh-linux-gnu-',
195 'sparc': 'sparc-linux-',
202 STATE_SAVEDEFCONFIG = 3
205 ACTION_DEFAULT_VALUE = 1
206 ACTION_ALREADY_EXIST = 2
214 COLOR_PURPLE = '0;35'
216 COLOR_LIGHT_GRAY = '0;37'
217 COLOR_DARK_GRAY = '1;30'
218 COLOR_LIGHT_RED = '1;31'
219 COLOR_LIGHT_GREEN = '1;32'
220 COLOR_YELLOW = '1;33'
221 COLOR_LIGHT_BLUE = '1;34'
222 COLOR_LIGHT_PURPLE = '1;35'
223 COLOR_LIGHT_CYAN = '1;36'
226 ### helper functions ###
228 """Get the file object of '/dev/null' device."""
230 devnull = subprocess.DEVNULL # py3k
231 except AttributeError:
232 devnull = open(os.devnull, 'wb')
235 def check_top_directory():
236 """Exit if we are not at the top of source directory."""
237 for f in ('README', 'Licenses'):
238 if not os.path.exists(f):
239 sys.exit('Please run at the top of source directory.')
242 """Get the command name of GNU Make.
244 U-Boot needs GNU Make for building, but the command name is not
245 necessarily "make". (for example, "gmake" on FreeBSD).
246 Returns the most appropriate command name on your system.
248 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
249 ret = process.communicate()
250 if process.returncode:
251 sys.exit('GNU Make not found')
252 return ret[0].rstrip()
254 def color_text(color_enabled, color, string):
255 """Return colored string."""
257 return '\033[' + color + 'm' + string + '\033[0m'
261 def log_msg(color_enabled, color, defconfig, msg):
262 """Return the formated line for the log."""
263 return defconfig[:-len('_defconfig')].ljust(37) + ': ' + \
264 color_text(color_enabled, color, msg) + '\n'
266 def update_cross_compile():
267 """Update per-arch CROSS_COMPILE via enviroment variables
269 The default CROSS_COMPILE values are available
270 in the CROSS_COMPILE list above.
272 You can override them via enviroment variables
273 CROSS_COMPILE_{ARCH}.
275 For example, if you want to override toolchain prefixes
276 for ARM and PowerPC, you can do as follows in your shell:
278 export CROSS_COMPILE_ARM=...
279 export CROSS_COMPILE_POWERPC=...
283 for arch in os.listdir('arch'):
284 if os.path.exists(os.path.join('arch', arch, 'Makefile')):
287 # arm64 is a special case
288 archs.append('aarch64')
291 env = 'CROSS_COMPILE_' + arch.upper()
292 cross_compile = os.environ.get(env)
294 CROSS_COMPILE[arch] = cross_compile
296 def cleanup_one_header(header_path, patterns, dry_run):
297 """Clean regex-matched lines away from a file.
300 header_path: path to the cleaned file.
301 patterns: list of regex patterns. Any lines matching to these
302 patterns are deleted.
303 dry_run: make no changes, but still display log.
305 with open(header_path) as f:
306 lines = f.readlines()
309 for i, line in enumerate(lines):
310 for pattern in patterns:
311 m = pattern.search(line)
313 print '%s: %s: %s' % (header_path, i + 1, line),
317 if dry_run or not matched:
320 with open(header_path, 'w') as f:
321 for i, line in enumerate(lines):
325 def cleanup_headers(config_attrs, dry_run):
326 """Delete config defines from board headers.
329 config_attrs: A list of dictionaris, each of them includes the name,
330 the type, and the default value of the target config.
331 dry_run: make no changes, but still display log.
334 choice = raw_input('Clean up headers? [y/n]: ').lower()
336 if choice == 'y' or choice == 'n':
343 for config_attr in config_attrs:
344 config = config_attr['config']
345 patterns.append(re.compile(r'#\s*define\s+%s\W' % config))
346 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config))
348 for (dirpath, dirnames, filenames) in os.walk('include'):
349 for filename in filenames:
350 if not fnmatch.fnmatch(filename, '*~'):
351 cleanup_one_header(os.path.join(dirpath, filename), patterns,
357 """A parser of .config and include/autoconf.mk."""
359 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
360 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
362 def __init__(self, config_attrs, options, build_dir):
363 """Create a new parser.
366 config_attrs: A list of dictionaris, each of them includes the name,
367 the type, and the default value of the target config.
368 options: option flags.
369 build_dir: Build directory.
371 self.config_attrs = config_attrs
372 self.options = options
373 self.build_dir = build_dir
375 def get_cross_compile(self):
376 """Parse .config file and return CROSS_COMPILE.
379 A string storing the compiler prefix for the architecture.
383 dotconfig = os.path.join(self.build_dir, '.config')
384 for line in open(dotconfig):
385 m = self.re_arch.match(line)
389 m = self.re_cpu.match(line)
393 assert arch, 'Error: arch is not defined in %s' % defconfig
396 if arch == 'arm' and cpu == 'armv8':
399 return CROSS_COMPILE.get(arch, '')
401 def parse_one_config(self, config_attr, defconfig_lines, autoconf_lines):
402 """Parse .config, defconfig, include/autoconf.mk for one config.
404 This function looks for the config options in the lines from
405 defconfig, .config, and include/autoconf.mk in order to decide
406 which action should be taken for this defconfig.
409 config_attr: A dictionary including the name, the type,
410 and the default value of the target config.
411 defconfig_lines: lines from the original defconfig file.
412 autoconf_lines: lines from the include/autoconf.mk file.
415 A tupple of the action for this defconfig and the line
416 matched for the config.
418 config = config_attr['config']
419 not_set = '# %s is not set' % config
421 if config_attr['type'] in ('bool', 'tristate') and \
422 config_attr['default'] == 'n':
425 default = config + '=' + config_attr['default']
427 for line in defconfig_lines:
429 if line.startswith(config + '=') or line == not_set:
430 return (ACTION_ALREADY_EXIST, line)
432 if config_attr['type'] in ('bool', 'tristate'):
435 value = '(undefined)'
437 for line in autoconf_lines:
439 if line.startswith(config + '='):
444 action = ACTION_DEFAULT_VALUE
445 elif value == '(undefined)':
446 action = ACTION_UNDEFINED
450 return (action, value)
452 def update_defconfig(self, defconfig):
453 """Parse files for the config options and update the defconfig.
455 This function parses the given defconfig, the generated .config
456 and include/autoconf.mk searching the target options.
457 Move the config option(s) to the defconfig or do nothing if unneeded.
458 Also, display the log to show what happened to this defconfig.
461 defconfig: defconfig name.
464 defconfig_path = os.path.join('configs', defconfig)
465 dotconfig_path = os.path.join(self.build_dir, '.config')
466 autoconf_path = os.path.join(self.build_dir, 'include', 'autoconf.mk')
469 with open(defconfig_path) as f:
470 defconfig_lines = f.readlines()
472 with open(autoconf_path) as f:
473 autoconf_lines = f.readlines()
475 for config_attr in self.config_attrs:
476 result = self.parse_one_config(config_attr, defconfig_lines,
478 results.append(result)
482 for (action, value) in results:
483 if action == ACTION_MOVE:
484 actlog = "Move '%s'" % value
485 log_color = COLOR_LIGHT_GREEN
486 elif action == ACTION_DEFAULT_VALUE:
487 actlog = "Default value '%s'. Do nothing." % value
488 log_color = COLOR_LIGHT_BLUE
489 elif action == ACTION_ALREADY_EXIST:
490 actlog = "'%s' already defined in Kconfig. Do nothing." % value
491 log_color = COLOR_LIGHT_PURPLE
492 elif action == ACTION_UNDEFINED:
493 actlog = "Undefined. Do nothing."
494 log_color = COLOR_DARK_GRAY
496 sys.exit("Internal Error. This should not happen.")
498 log += log_msg(self.options.color, log_color, defconfig, actlog)
500 # Some threads are running in parallel.
501 # Print log in one shot to not mix up logs from different threads.
504 if not self.options.dry_run:
505 with open(dotconfig_path, 'a') as f:
506 for (action, value) in results:
507 if action == ACTION_MOVE:
508 f.write(value + '\n')
510 os.remove(os.path.join(self.build_dir, 'include', 'config', 'auto.conf'))
511 os.remove(autoconf_path)
515 """A slot to store a subprocess.
517 Each instance of this class handles one subprocess.
518 This class is useful to control multiple threads
519 for faster processing.
522 def __init__(self, config_attrs, options, devnull, make_cmd):
523 """Create a new process slot.
526 config_attrs: A list of dictionaris, each of them includes the name,
527 the type, and the default value of the target config.
528 options: option flags.
529 devnull: A file object of '/dev/null'.
530 make_cmd: command name of GNU Make.
532 self.options = options
533 self.build_dir = tempfile.mkdtemp()
534 self.devnull = devnull
535 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
536 self.parser = KconfigParser(config_attrs, options, self.build_dir)
537 self.state = STATE_IDLE
538 self.failed_boards = []
541 """Delete the working directory
543 This function makes sure the temporary directory is cleaned away
544 even if Python suddenly dies due to error. It should be done in here
545 because it is guranteed the destructor is always invoked when the
546 instance of the class gets unreferenced.
548 If the subprocess is still running, wait until it finishes.
550 if self.state != STATE_IDLE:
551 while self.ps.poll() == None:
553 shutil.rmtree(self.build_dir)
555 def add(self, defconfig):
556 """Assign a new subprocess for defconfig and add it to the slot.
558 If the slot is vacant, create a new subprocess for processing the
559 given defconfig and add it to the slot. Just returns False if
560 the slot is occupied (i.e. the current subprocess is still running).
563 defconfig: defconfig name.
566 Return True on success or False on failure
568 if self.state != STATE_IDLE:
570 cmd = list(self.make_cmd)
571 cmd.append(defconfig)
572 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
573 self.defconfig = defconfig
574 self.state = STATE_DEFCONFIG
578 """Check the status of the subprocess and handle it as needed.
580 Returns True if the slot is vacant (i.e. in idle state).
581 If the configuration is successfully finished, assign a new
582 subprocess to build include/autoconf.mk.
583 If include/autoconf.mk is generated, invoke the parser to
584 parse the .config and the include/autoconf.mk, and then set the
585 slot back to the idle state.
588 Return True if the subprocess is terminated, False otherwise
590 if self.state == STATE_IDLE:
593 if self.ps.poll() == None:
596 if self.ps.poll() != 0:
598 print >> sys.stderr, log_msg(self.options.color,
601 "failed to process.")
602 if self.options.exit_on_error:
603 sys.exit("Exit on error.")
605 # If --exit-on-error flag is not set,
606 # skip this board and continue.
607 # Record the failed board.
608 self.failed_boards.append(self.defconfig)
609 self.state = STATE_IDLE
612 if self.state == STATE_AUTOCONF:
613 self.parser.update_defconfig(self.defconfig)
615 """Save off the defconfig in a consistent way"""
616 cmd = list(self.make_cmd)
617 cmd.append('savedefconfig')
618 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
620 self.state = STATE_SAVEDEFCONFIG
623 if self.state == STATE_SAVEDEFCONFIG:
624 defconfig_path = os.path.join(self.build_dir, 'defconfig')
625 shutil.move(defconfig_path,
626 os.path.join('configs', self.defconfig))
627 self.state = STATE_IDLE
630 cross_compile = self.parser.get_cross_compile()
631 cmd = list(self.make_cmd)
633 cmd.append('CROSS_COMPILE=%s' % cross_compile)
634 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
635 cmd.append('include/config/auto.conf')
636 self.ps = subprocess.Popen(cmd, stdout=self.devnull)
637 self.state = STATE_AUTOCONF
640 def get_failed_boards(self):
641 """Returns a list of failed boards (defconfigs) in this slot.
643 return self.failed_boards
647 """Controller of the array of subprocess slots."""
649 def __init__(self, config_attrs, options):
650 """Create a new slots controller.
653 config_attrs: A list of dictionaris containing the name, the type,
654 and the default value of the target CONFIG.
655 options: option flags.
657 self.options = options
659 devnull = get_devnull()
660 make_cmd = get_make_cmd()
661 for i in range(options.jobs):
662 self.slots.append(Slot(config_attrs, options, devnull, make_cmd))
664 def add(self, defconfig):
665 """Add a new subprocess if a vacant slot is found.
668 defconfig: defconfig name to be put into.
671 Return True on success or False on failure
673 for slot in self.slots:
674 if slot.add(defconfig):
679 """Check if there is a vacant slot.
682 Return True if at lease one vacant slot is found, False otherwise.
684 for slot in self.slots:
690 """Check if all slots are vacant.
693 Return True if all the slots are vacant, False otherwise.
696 for slot in self.slots:
701 def show_failed_boards(self):
702 """Display all of the failed boards (defconfigs)."""
705 for slot in self.slots:
706 failed_boards += slot.get_failed_boards()
708 if len(failed_boards) > 0:
709 msg = [ "The following boards were not processed due to error:" ]
712 print >> sys.stderr, color_text(self.options.color,
713 COLOR_LIGHT_RED, line)
715 def move_config(config_attrs, options):
716 """Move config options to defconfig files.
719 config_attrs: A list of dictionaris, each of them includes the name,
720 the type, and the default value of the target config.
721 options: option flags
723 check_top_directory()
725 if len(config_attrs) == 0:
726 print 'Nothing to do. exit.'
729 print 'Move the following CONFIG options (jobs: %d)' % options.jobs
730 for config_attr in config_attrs:
731 print ' %s (type: %s, default: %s)' % (config_attr['config'],
733 config_attr['default'])
735 if options.defconfigs:
736 defconfigs = [line.strip() for line in open(options.defconfigs)]
737 for i, defconfig in enumerate(defconfigs):
738 if not defconfig.endswith('_defconfig'):
739 defconfigs[i] = defconfig + '_defconfig'
740 if not os.path.exists(os.path.join('configs', defconfigs[i])):
741 sys.exit('%s - defconfig does not exist. Stopping.' %
744 # All the defconfig files to be processed
746 for (dirpath, dirnames, filenames) in os.walk('configs'):
747 dirpath = dirpath[len('configs') + 1:]
748 for filename in fnmatch.filter(filenames, '*_defconfig'):
749 defconfigs.append(os.path.join(dirpath, filename))
751 slots = Slots(config_attrs, options)
753 # Main loop to process defconfig files:
754 # Add a new subprocess into a vacant slot.
755 # Sleep if there is no available slot.
756 for defconfig in defconfigs:
757 while not slots.add(defconfig):
758 while not slots.available():
759 # No available slot: sleep for a while
760 time.sleep(SLEEP_TIME)
762 # wait until all the subprocesses finish
763 while not slots.empty():
764 time.sleep(SLEEP_TIME)
766 slots.show_failed_boards()
768 cleanup_headers(config_attrs, options.dry_run)
770 def bad_recipe(filename, linenum, msg):
771 """Print error message with the file name and the line number and exit."""
772 sys.exit("%s: line %d: error : " % (filename, linenum) + msg)
774 def parse_recipe(filename):
775 """Parse the recipe file and retrieve the config attributes.
777 This function parses the given recipe file and gets the name,
778 the type, and the default value of the target config options.
781 filename: path to file to be parsed.
783 A list of dictionaris, each of them includes the name,
784 the type, and the default value of the target config.
789 for line in open(filename):
790 tokens = line.split()
792 bad_recipe(filename, linenum,
793 "%d fields in this line. Each line must contain 3 fields"
796 (config, type, default) = tokens
798 # prefix the option name with CONFIG_ if missing
799 if not config.startswith('CONFIG_'):
800 config = 'CONFIG_' + config
802 # sanity check of default values
804 if not default in ('y', 'n'):
805 bad_recipe(filename, linenum,
806 "default for bool type must be either y or n")
807 elif type == 'tristate':
808 if not default in ('y', 'm', 'n'):
809 bad_recipe(filename, linenum,
810 "default for tristate type must be y, m, or n")
811 elif type == 'string':
812 if default[0] != '"' or default[-1] != '"':
813 bad_recipe(filename, linenum,
814 "default for string type must be surrounded by double-quotations")
819 bad_recipe(filename, linenum,
820 "type is int, but default value is not decimal")
822 if len(default) < 2 or default[:2] != '0x':
823 bad_recipe(filename, linenum,
824 "default for hex type must be prefixed with 0x")
828 bad_recipe(filename, linenum,
829 "type is hex, but default value is not hexadecimal")
831 bad_recipe(filename, linenum,
832 "unsupported type '%s'. type must be one of bool, tristate, string, int, hex"
835 config_attrs.append({'config': config, 'type': type, 'default': default})
842 cpu_count = multiprocessing.cpu_count()
843 except NotImplementedError:
846 parser = optparse.OptionParser()
848 parser.add_option('-c', '--color', action='store_true', default=False,
849 help='display the log in color')
850 parser.add_option('-d', '--defconfigs', type='string',
851 help='a file containing a list of defconfigs to move')
852 parser.add_option('-n', '--dry-run', action='store_true', default=False,
853 help='perform a trial run (show log with no changes)')
854 parser.add_option('-e', '--exit-on-error', action='store_true',
856 help='exit immediately on any error')
857 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
858 help='the number of jobs to run simultaneously')
859 parser.usage += ' recipe_file\n\n' + \
860 'The recipe_file should describe config options you want to move.\n' + \
861 'Each line should contain config_name, type, default_value\n\n' + \
863 'CONFIG_FOO bool n\n' + \
864 'CONFIG_BAR int 100\n' + \
865 'CONFIG_BAZ string "hello"\n'
867 (options, args) = parser.parse_args()
873 config_attrs = parse_recipe(args[0])
875 update_cross_compile()
877 move_config(config_attrs, options)
879 if __name__ == '__main__':