2 # SPDX-License-Identifier: GPL-2.0+
4 # Author: Masahiro Yamada <yamada.masahiro@socionext.com>
8 Move config options from headers to defconfig files.
10 See doc/develop/moveconfig.rst for documentation.
13 from argparse import ArgumentParser
16 from contextlib import ExitStack
23 import multiprocessing
35 from buildman import bsettings
36 from buildman import kconfiglib
37 from buildman import toolchain
39 SHOW_GNU_MAKE = 'scripts/show-gnu-make'
45 STATE_SAVEDEFCONFIG = 3
49 ACTION_NO_ENTRY_WARN = 2
59 COLOR_LIGHT_GRAY = '0;37'
60 COLOR_DARK_GRAY = '1;30'
61 COLOR_LIGHT_RED = '1;31'
62 COLOR_LIGHT_GREEN = '1;32'
64 COLOR_LIGHT_BLUE = '1;34'
65 COLOR_LIGHT_PURPLE = '1;35'
66 COLOR_LIGHT_CYAN = '1;36'
69 AUTO_CONF_PATH = 'include/config/auto.conf'
70 CONFIG_DATABASE = 'moveconfig.db'
72 CONFIG_LEN = len('CONFIG_')
75 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
76 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
77 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
78 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
79 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
80 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
81 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
82 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
83 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
84 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
85 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
86 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
87 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
88 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
89 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
90 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
94 RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
96 # CONFIG symbols present in the build system (from Linux) but not actually used
97 # in U-Boot; KCONFIG symbols
98 IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
99 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
100 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
101 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
102 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
103 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
104 'VAL', '_UNDEFINED', 'SPL_BUILD', ]
106 SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
108 ### helper functions ###
109 def check_top_directory():
110 """Exit if we are not at the top of source directory."""
111 for fname in 'README', 'Licenses':
112 if not os.path.exists(fname):
113 sys.exit('Please run at the top of source directory.')
115 def check_clean_directory():
116 """Exit if the source tree is not clean."""
117 for fname in '.config', 'include/config':
118 if os.path.exists(fname):
119 sys.exit("source tree is not clean, please run 'make mrproper'")
122 """Get the command name of GNU Make.
124 U-Boot needs GNU Make for building, but the command name is not
125 necessarily "make". (for example, "gmake" on FreeBSD).
126 Returns the most appropriate command name on your system.
128 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
129 ret = proc.communicate()
131 sys.exit('GNU Make not found')
132 return ret[0].rstrip()
134 def get_matched_defconfig(line):
135 """Get the defconfig files that match a pattern
138 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
139 'k2*_defconfig'. If no directory is provided, 'configs/' is
143 list of str: a list of matching defconfig files
145 dirname = os.path.dirname(line)
149 pattern = os.path.join('configs', line)
150 return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
152 def get_matched_defconfigs(defconfigs_file):
153 """Get all the defconfig files that match the patterns in a file.
156 defconfigs_file (str): File containing a list of defconfigs to process,
157 or '-' to read the list from stdin
160 list of str: A list of paths to defconfig files, with no duplicates
163 with ExitStack() as stack:
164 if defconfigs_file == '-':
166 defconfigs_file = 'stdin'
168 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
169 for i, line in enumerate(inf):
172 continue # skip blank lines silently
174 line = line.split(' ')[0] # handle 'git log' input
175 matched = get_matched_defconfig(line)
177 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
180 defconfigs += matched
182 # use set() to drop multiple matching
183 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
185 def get_all_defconfigs():
186 """Get all the defconfig files under the configs/ directory.
189 list of str: List of paths to defconfig files
192 for (dirpath, _, filenames) in os.walk('configs'):
193 dirpath = dirpath[len('configs') + 1:]
194 for filename in fnmatch.filter(filenames, '*_defconfig'):
195 defconfigs.append(os.path.join(dirpath, filename))
199 def color_text(color_enabled, color, string):
200 """Return colored string."""
202 # LF should not be surrounded by the escape sequence.
203 # Otherwise, additional whitespace or line-feed might be printed.
204 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
205 for s in string.split('\n') ])
208 def show_diff(alines, blines, file_path, color_enabled):
209 """Show unidified diff.
212 alines (list of str): A list of lines (before)
213 blines (list of str): A list of lines (after)
214 file_path (str): Path to the file
215 color_enabled (bool): Display the diff in color
217 diff = difflib.unified_diff(alines, blines,
218 fromfile=os.path.join('a', file_path),
219 tofile=os.path.join('b', file_path))
222 if line.startswith('-') and not line.startswith('--'):
223 print(color_text(color_enabled, COLOR_RED, line))
224 elif line.startswith('+') and not line.startswith('++'):
225 print(color_text(color_enabled, COLOR_GREEN, line))
229 def extend_matched_lines(lines, matched, pre_patterns, post_patterns,
230 extend_pre, extend_post):
231 """Extend matched lines if desired patterns are found before/after already
235 lines (list of str): list of lines handled.
236 matched (list of int): list of line numbers that have been already
237 matched (will be updated by this function)
238 pre_patterns (list of re.Pattern): list of regular expression that should
239 be matched as preamble
240 post_patterns (list of re.Pattern): list of regular expression that should
241 be matched as postamble
242 extend_pre (bool): Add the line number of matched preamble to the matched
244 extend_post (bool): Add the line number of matched postamble to the
247 extended_matched = []
260 for pat in pre_patterns:
261 if pat.search(lines[i - 1]):
267 for pat in post_patterns:
268 if pat.search(lines[j]):
275 extended_matched.append(i - 1)
277 extended_matched.append(j)
279 matched += extended_matched
282 def confirm(args, prompt):
283 """Ask the user to confirm something
286 args (Namespace ): program arguments
289 bool: True to confirm, False to cancel/stop
293 choice = input(f'{prompt} [y/n]: ')
294 choice = choice.lower()
296 if choice in ('y', 'n'):
304 def write_file(fname, data):
305 """Write data to a file
308 fname (str): Filename to write to
309 data (list of str): Lines to write (with or without trailing newline);
312 with open(fname, 'w', encoding='utf-8') as out:
313 if isinstance(data, list):
315 print(line.rstrip('\n'), file=out)
319 def read_file(fname, as_lines=True, skip_unicode=False):
320 """Read a file and return the contents
323 fname (str): Filename to read from
324 as_lines: Return file contents as a list of lines
325 skip_unicode (bool): True to report unicode errors and continue
328 iter of str: List of ;ines from the file with newline removed; str if
329 as_lines is False with newlines intact; or None if a unicode error
333 UnicodeDecodeError: Unicode error occurred when reading
335 with open(fname, encoding='utf-8') as inf:
338 return [line.rstrip('\n') for line in inf.readlines()]
341 except UnicodeDecodeError as e:
344 print("Failed on file %s': %s" % (fname, e))
347 def cleanup_empty_blocks(header_path, args):
348 """Clean up empty conditional blocks
351 header_path (str): path to the cleaned file.
352 args (Namespace): program arguments
354 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M)
355 data = read_file(header_path, as_lines=False, skip_unicode=True)
359 new_data = pattern.sub('\n', data)
361 show_diff(data.splitlines(True), new_data.splitlines(True), header_path,
368 write_file(header_path, new_data)
370 def cleanup_one_header(header_path, patterns, args):
371 """Clean regex-matched lines away from a file.
374 header_path: path to the cleaned file.
375 patterns: list of regex patterns. Any lines matching to these
376 patterns are deleted.
377 args (Namespace): program arguments
379 lines = read_file(header_path, skip_unicode=True)
384 for i, line in enumerate(lines):
385 if i - 1 in matched and lines[i - 1].endswith('\\'):
388 for pattern in patterns:
389 if pattern.search(line):
396 # remove empty #ifdef ... #endif, successive blank lines
397 pattern_if = re.compile(r'#\s*if(def|ndef)?\b') # #if, #ifdef, #ifndef
398 pattern_elif = re.compile(r'#\s*el(if|se)\b') # #elif, #else
399 pattern_endif = re.compile(r'#\s*endif\b') # #endif
400 pattern_blank = re.compile(r'^\s*$') # empty line
403 old_matched = copy.copy(matched)
404 extend_matched_lines(lines, matched, [pattern_if],
405 [pattern_endif], True, True)
406 extend_matched_lines(lines, matched, [pattern_elif],
407 [pattern_elif, pattern_endif], True, False)
408 extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
409 [pattern_blank], False, True)
410 extend_matched_lines(lines, matched, [pattern_blank],
411 [pattern_elif, pattern_endif], True, False)
412 extend_matched_lines(lines, matched, [pattern_blank],
413 [pattern_blank], True, False)
414 if matched == old_matched:
417 tolines = copy.copy(lines)
419 for i in reversed(matched):
422 show_diff(lines, tolines, header_path, args.color)
427 write_file(header_path, tolines)
429 def cleanup_headers(configs, args):
430 """Delete config defines from board headers.
433 configs: A list of CONFIGs to remove.
434 args (Namespace): program arguments
436 if not confirm(args, 'Clean up headers?'):
440 for config in configs:
441 patterns.append(re.compile(r'#\s*define\s+%s\b' % config))
442 patterns.append(re.compile(r'#\s*undef\s+%s\b' % config))
444 for dir in 'include', 'arch', 'board':
445 for (dirpath, dirnames, filenames) in os.walk(dir):
446 if dirpath == os.path.join('include', 'generated'):
448 for filename in filenames:
449 if not filename.endswith(('~', '.dts', '.dtsi', '.bin',
450 '.elf','.aml','.dat')):
451 header_path = os.path.join(dirpath, filename)
452 # This file contains UTF-16 data and no CONFIG symbols
453 if header_path == 'include/video_font_data.h':
455 cleanup_one_header(header_path, patterns, args)
456 cleanup_empty_blocks(header_path, args)
458 def find_matching(patterns, line):
464 def cleanup_readme(configs, args):
465 """Delete config description in README
468 configs: A list of CONFIGs to remove.
469 args (Namespace): program arguments
471 if not confirm(args, 'Clean up README?'):
475 for config in configs:
476 patterns.append(re.compile(r'^\s+%s' % config))
478 lines = read_file('README')
484 found = find_matching(patterns, line)
488 if found and re.search(r'^\s+CONFIG', line):
492 newlines.append(line)
494 write_file('README', newlines)
496 def try_expand(line):
497 """If value looks like an expression, try expanding it
498 Otherwise just return the existing value
500 if line.find('=') == -1:
504 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
505 cfg, val = re.split("=", line)
507 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
508 newval = hex(aeval(val))
509 print('\tExpanded expression %s to %s' % (val, newval))
510 return cfg+'='+newval
512 print('\tFailed to expand expression in %s' % line)
520 """Progress Indicator"""
522 def __init__(self, total):
523 """Create a new progress indicator.
526 total: A number of defconfig files to process.
532 """Increment the number of processed defconfig files."""
537 """Display the progress."""
538 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
542 class KconfigScanner:
543 """Kconfig scanner."""
546 """Scan all the Kconfig files and create a Config object."""
547 # Define environment variables referenced from Kconfig
548 os.environ['srctree'] = os.getcwd()
549 os.environ['UBOOTVERSION'] = 'dummy'
550 os.environ['KCONFIG_OBJDIR'] = ''
551 os.environ['CC'] = 'gcc'
552 self.conf = kconfiglib.Kconfig()
557 """A parser of .config and include/autoconf.mk."""
559 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
560 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
562 def __init__(self, configs, args, build_dir):
563 """Create a new parser.
566 configs: A list of CONFIGs to move.
567 args (Namespace): program arguments
568 build_dir: Build directory.
570 self.configs = configs
572 self.dotconfig = os.path.join(build_dir, '.config')
573 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
574 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
576 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
577 self.defconfig = os.path.join(build_dir, 'defconfig')
580 """Parse .config file and return the architecture.
583 Architecture name (e.g. 'arm').
587 for line in read_file(self.dotconfig):
588 m = self.re_arch.match(line)
592 m = self.re_cpu.match(line)
600 if arch == 'arm' and cpu == 'armv8':
605 def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
606 """Parse .config, defconfig, include/autoconf.mk for one config.
608 This function looks for the config options in the lines from
609 defconfig, .config, and include/autoconf.mk in order to decide
610 which action should be taken for this defconfig.
613 config: CONFIG name to parse.
614 dotconfig_lines: lines from the .config file.
615 autoconf_lines: lines from the include/autoconf.mk file.
618 A tupple of the action for this defconfig and the line
619 matched for the config.
621 not_set = '# %s is not set' % config
623 for line in autoconf_lines:
625 if line.startswith(config + '='):
631 new_val = try_expand(new_val)
633 for line in dotconfig_lines:
635 if line.startswith(config + '=') or line == not_set:
639 if new_val == not_set:
640 return (ACTION_NO_ENTRY, config)
642 return (ACTION_NO_ENTRY_WARN, config)
644 # If this CONFIG is neither bool nor trisate
645 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
646 # tools/scripts/define2mk.sed changes '1' to 'y'.
647 # This is a problem if the CONFIG is int type.
648 # Check the type in Kconfig and handle it correctly.
649 if new_val[-2:] == '=y':
650 new_val = new_val[:-1] + '1'
652 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
655 def update_dotconfig(self):
656 """Parse files for the config options and update the .config.
658 This function parses the generated .config and include/autoconf.mk
659 searching the target options.
660 Move the config option(s) to the .config as needed.
663 defconfig: defconfig name.
666 Return a tuple of (updated flag, log string).
667 The "updated flag" is True if the .config was updated, False
668 otherwise. The "log string" shows what happend to the .config.
674 rm_files = [self.config_autoconf, self.autoconf]
677 if os.path.exists(self.spl_autoconf):
678 autoconf_path = self.spl_autoconf
679 rm_files.append(self.spl_autoconf)
683 return (updated, suspicious,
684 color_text(self.args.color, COLOR_BROWN,
685 "SPL is not enabled. Skipped.") + '\n')
687 autoconf_path = self.autoconf
689 dotconfig_lines = read_file(self.dotconfig)
691 autoconf_lines = read_file(autoconf_path)
693 for config in self.configs:
694 result = self.parse_one_config(config, dotconfig_lines,
696 results.append(result)
700 for (action, value) in results:
701 if action == ACTION_MOVE:
702 actlog = "Move '%s'" % value
703 log_color = COLOR_LIGHT_GREEN
704 elif action == ACTION_NO_ENTRY:
705 actlog = '%s is not defined in Kconfig. Do nothing.' % value
706 log_color = COLOR_LIGHT_BLUE
707 elif action == ACTION_NO_ENTRY_WARN:
708 actlog = '%s is not defined in Kconfig (suspicious). Do nothing.' % value
709 log_color = COLOR_YELLOW
711 elif action == ACTION_NO_CHANGE:
712 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
714 log_color = COLOR_LIGHT_PURPLE
716 sys.exit('Internal Error. This should not happen.')
718 log += color_text(self.args.color, log_color, actlog) + '\n'
720 with open(self.dotconfig, 'a', encoding='utf-8') as out:
721 for (action, value) in results:
722 if action == ACTION_MOVE:
723 out.write(value + '\n')
726 self.results = results
730 return (updated, suspicious, log)
732 def check_defconfig(self):
733 """Check the defconfig after savedefconfig
736 Return additional log if moved CONFIGs were removed again by
737 'make savedefconfig'.
742 defconfig_lines = read_file(self.defconfig)
744 for (action, value) in self.results:
745 if action != ACTION_MOVE:
747 if not value in defconfig_lines:
748 log += color_text(self.args.color, COLOR_YELLOW,
749 "'%s' was removed by savedefconfig.\n" %
755 class DatabaseThread(threading.Thread):
756 """This thread processes results from Slot threads.
758 It collects the data in the master config directary. There is only one
759 result thread, and this helps to serialise the build output.
761 def __init__(self, config_db, db_queue):
762 """Set up a new result thread
765 builder: Builder which will be sent each result
767 threading.Thread.__init__(self)
768 self.config_db = config_db
769 self.db_queue= db_queue
772 """Called to start up the result thread.
774 We collect the next result job and pass it on to the build.
777 defconfig, configs = self.db_queue.get()
778 self.config_db[defconfig] = configs
779 self.db_queue.task_done()
784 """A slot to store a subprocess.
786 Each instance of this class handles one subprocess.
787 This class is useful to control multiple threads
788 for faster processing.
791 def __init__(self, toolchains, configs, args, progress, devnull,
792 make_cmd, reference_src_dir, db_queue):
793 """Create a new process slot.
796 toolchains: Toolchains object containing toolchains.
797 configs: A list of CONFIGs to move.
798 args: Program arguments
799 progress: A progress indicator.
800 devnull: A file object of '/dev/null'.
801 make_cmd: command name of GNU Make.
802 reference_src_dir: Determine the true starting config state from this
804 db_queue: output queue to write config info for the database
806 self.toolchains = toolchains
808 self.progress = progress
809 self.build_dir = tempfile.mkdtemp()
810 self.devnull = devnull
811 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
812 self.reference_src_dir = reference_src_dir
813 self.db_queue = db_queue
814 self.parser = KconfigParser(configs, args, self.build_dir)
815 self.state = STATE_IDLE
816 self.failed_boards = set()
817 self.suspicious_boards = set()
820 """Delete the working directory
822 This function makes sure the temporary directory is cleaned away
823 even if Python suddenly dies due to error. It should be done in here
824 because it is guaranteed the destructor is always invoked when the
825 instance of the class gets unreferenced.
827 If the subprocess is still running, wait until it finishes.
829 if self.state != STATE_IDLE:
830 while self.ps.poll() == None:
832 shutil.rmtree(self.build_dir)
834 def add(self, defconfig):
835 """Assign a new subprocess for defconfig and add it to the slot.
837 If the slot is vacant, create a new subprocess for processing the
838 given defconfig and add it to the slot. Just returns False if
839 the slot is occupied (i.e. the current subprocess is still running).
842 defconfig: defconfig name.
845 Return True on success or False on failure
847 if self.state != STATE_IDLE:
850 self.defconfig = defconfig
852 self.current_src_dir = self.reference_src_dir
857 """Check the status of the subprocess and handle it as needed.
859 Returns True if the slot is vacant (i.e. in idle state).
860 If the configuration is successfully finished, assign a new
861 subprocess to build include/autoconf.mk.
862 If include/autoconf.mk is generated, invoke the parser to
863 parse the .config and the include/autoconf.mk, moving
864 config options to the .config as needed.
865 If the .config was updated, run "make savedefconfig" to sync
866 it, update the original defconfig, and then set the slot back
870 Return True if the subprocess is terminated, False otherwise
872 if self.state == STATE_IDLE:
875 if self.ps.poll() == None:
878 if self.ps.poll() != 0:
880 elif self.state == STATE_DEFCONFIG:
881 if self.reference_src_dir and not self.current_src_dir:
882 self.do_savedefconfig()
885 elif self.state == STATE_AUTOCONF:
886 if self.current_src_dir:
887 self.current_src_dir = None
889 elif self.args.build_db:
892 self.do_savedefconfig()
893 elif self.state == STATE_SAVEDEFCONFIG:
894 self.update_defconfig()
896 sys.exit('Internal Error. This should not happen.')
898 return True if self.state == STATE_IDLE else False
900 def handle_error(self):
901 """Handle error cases."""
903 self.log += color_text(self.args.color, COLOR_LIGHT_RED,
904 'Failed to process.\n')
905 if self.args.verbose:
906 self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
907 self.ps.stderr.read().decode())
910 def do_defconfig(self):
911 """Run 'make <board>_defconfig' to create the .config file."""
913 cmd = list(self.make_cmd)
914 cmd.append(self.defconfig)
915 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
916 stderr=subprocess.PIPE,
917 cwd=self.current_src_dir)
918 self.state = STATE_DEFCONFIG
920 def do_autoconf(self):
921 """Run 'make AUTO_CONF_PATH'."""
923 arch = self.parser.get_arch()
925 toolchain = self.toolchains.Select(arch)
927 self.log += color_text(self.args.color, COLOR_YELLOW,
928 "Tool chain for '%s' is missing. Do nothing.\n" % arch)
931 env = toolchain.MakeEnvironment(False)
933 cmd = list(self.make_cmd)
934 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
935 cmd.append(AUTO_CONF_PATH)
936 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env,
937 stderr=subprocess.PIPE,
938 cwd=self.current_src_dir)
939 self.state = STATE_AUTOCONF
941 def do_build_db(self):
942 """Add the board to the database"""
944 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
945 if line.startswith('CONFIG'):
946 config, value = line.split('=', 1)
947 configs[config] = value.rstrip()
948 self.db_queue.put([self.defconfig, configs])
951 def do_savedefconfig(self):
952 """Update the .config and run 'make savedefconfig'."""
954 (updated, suspicious, log) = self.parser.update_dotconfig()
956 self.suspicious_boards.add(self.defconfig)
959 if not self.args.force_sync and not updated:
963 self.log += color_text(self.args.color, COLOR_LIGHT_GREEN,
964 'Syncing by savedefconfig...\n')
966 self.log += 'Syncing by savedefconfig (forced by option)...\n'
968 cmd = list(self.make_cmd)
969 cmd.append('savedefconfig')
970 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
971 stderr=subprocess.PIPE)
972 self.state = STATE_SAVEDEFCONFIG
974 def update_defconfig(self):
975 """Update the input defconfig and go back to the idle state."""
977 log = self.parser.check_defconfig()
979 self.suspicious_boards.add(self.defconfig)
981 orig_defconfig = os.path.join('configs', self.defconfig)
982 new_defconfig = os.path.join(self.build_dir, 'defconfig')
983 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
986 self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
987 'defconfig was updated.\n')
989 if not self.args.dry_run and updated:
990 shutil.move(new_defconfig, orig_defconfig)
993 def finish(self, success):
994 """Display log along with progress and go to the idle state.
997 success: Should be True when the defconfig was processed
998 successfully, or False when it fails.
1000 # output at least 30 characters to hide the "* defconfigs out of *".
1001 log = self.defconfig.ljust(30) + '\n'
1003 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
1004 # Some threads are running in parallel.
1005 # Print log atomically to not mix up logs from different threads.
1006 print(log, file=(sys.stdout if success else sys.stderr))
1009 if self.args.exit_on_error:
1010 sys.exit('Exit on error.')
1011 # If --exit-on-error flag is not set, skip this board and continue.
1012 # Record the failed board.
1013 self.failed_boards.add(self.defconfig)
1016 self.progress.show()
1017 self.state = STATE_IDLE
1019 def get_failed_boards(self):
1020 """Returns a set of failed boards (defconfigs) in this slot.
1022 return self.failed_boards
1024 def get_suspicious_boards(self):
1025 """Returns a set of boards (defconfigs) with possible misconversion.
1027 return self.suspicious_boards - self.failed_boards
1031 """Controller of the array of subprocess slots."""
1033 def __init__(self, toolchains, configs, args, progress,
1034 reference_src_dir, db_queue):
1035 """Create a new slots controller.
1038 toolchains: Toolchains object containing toolchains.
1039 configs: A list of CONFIGs to move.
1040 args: Program arguments
1041 progress: A progress indicator.
1042 reference_src_dir: Determine the true starting config state from this
1044 db_queue: output queue to write config info for the database
1048 devnull = subprocess.DEVNULL
1049 make_cmd = get_make_cmd()
1050 for i in range(args.jobs):
1051 self.slots.append(Slot(toolchains, configs, args, progress,
1052 devnull, make_cmd, reference_src_dir,
1055 def add(self, defconfig):
1056 """Add a new subprocess if a vacant slot is found.
1059 defconfig: defconfig name to be put into.
1062 Return True on success or False on failure
1064 for slot in self.slots:
1065 if slot.add(defconfig):
1069 def available(self):
1070 """Check if there is a vacant slot.
1073 Return True if at lease one vacant slot is found, False otherwise.
1075 for slot in self.slots:
1081 """Check if all slots are vacant.
1084 Return True if all the slots are vacant, False otherwise.
1087 for slot in self.slots:
1092 def show_failed_boards(self):
1093 """Display all of the failed boards (defconfigs)."""
1095 output_file = 'moveconfig.failed'
1097 for slot in self.slots:
1098 boards |= slot.get_failed_boards()
1101 boards = '\n'.join(boards) + '\n'
1102 msg = 'The following boards were not processed due to error:\n'
1104 msg += '(the list has been saved in %s)\n' % output_file
1105 print(color_text(self.args.color, COLOR_LIGHT_RED,
1106 msg), file=sys.stderr)
1108 write_file(output_file, boards)
1110 def show_suspicious_boards(self):
1111 """Display all boards (defconfigs) with possible misconversion."""
1113 output_file = 'moveconfig.suspicious'
1115 for slot in self.slots:
1116 boards |= slot.get_suspicious_boards()
1119 boards = '\n'.join(boards) + '\n'
1120 msg = 'The following boards might have been converted incorrectly.\n'
1121 msg += 'It is highly recommended to check them manually:\n'
1123 msg += '(the list has been saved in %s)\n' % output_file
1124 print(color_text(self.args.color, COLOR_YELLOW,
1125 msg), file=sys.stderr)
1127 write_file(output_file, boards)
1129 class ReferenceSource:
1131 """Reference source against which original configs should be parsed."""
1133 def __init__(self, commit):
1134 """Create a reference source directory based on a specified commit.
1137 commit: commit to git-clone
1139 self.src_dir = tempfile.mkdtemp()
1140 print('Cloning git repo to a separate work directory...')
1141 subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
1143 print("Checkout '%s' to build the original autoconf.mk." % \
1144 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
1145 subprocess.check_output(['git', 'checkout', commit],
1146 stderr=subprocess.STDOUT, cwd=self.src_dir)
1149 """Delete the reference source directory
1151 This function makes sure the temporary directory is cleaned away
1152 even if Python suddenly dies due to error. It should be done in here
1153 because it is guaranteed the destructor is always invoked when the
1154 instance of the class gets unreferenced.
1156 shutil.rmtree(self.src_dir)
1159 """Return the absolute path to the reference source directory."""
1163 def move_config(toolchains, configs, args, db_queue):
1164 """Move config options to defconfig files.
1167 configs: A list of CONFIGs to move.
1168 args: Program arguments
1170 if len(configs) == 0:
1172 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
1174 print('Building %s database' % CONFIG_DATABASE)
1176 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
1178 print('Move ' + ', '.join(configs), end=' ')
1179 print('(jobs: %d)\n' % args.jobs)
1182 reference_src = ReferenceSource(args.git_ref)
1183 reference_src_dir = reference_src.get_dir()
1185 reference_src_dir = None
1188 defconfigs = get_matched_defconfigs(args.defconfigs)
1190 defconfigs = get_all_defconfigs()
1192 progress = Progress(len(defconfigs))
1193 slots = Slots(toolchains, configs, args, progress, reference_src_dir,
1196 # Main loop to process defconfig files:
1197 # Add a new subprocess into a vacant slot.
1198 # Sleep if there is no available slot.
1199 for defconfig in defconfigs:
1200 while not slots.add(defconfig):
1201 while not slots.available():
1202 # No available slot: sleep for a while
1203 time.sleep(SLEEP_TIME)
1205 # wait until all the subprocesses finish
1206 while not slots.empty():
1207 time.sleep(SLEEP_TIME)
1210 slots.show_failed_boards()
1211 slots.show_suspicious_boards()
1213 def find_kconfig_rules(kconf, config, imply_config):
1214 """Check whether a config has a 'select' or 'imply' keyword
1217 kconf: Kconfiglib.Kconfig object
1218 config: Name of config to check (without CONFIG_ prefix)
1219 imply_config: Implying config (without CONFIG_ prefix) which may or
1220 may not have an 'imply' for 'config')
1223 Symbol object for 'config' if found, else None
1225 sym = kconf.syms.get(imply_config)
1227 for sel, cond in (sym.selects + sym.implies):
1228 if sel.name == config:
1232 def check_imply_rule(kconf, config, imply_config):
1233 """Check if we can add an 'imply' option
1235 This finds imply_config in the Kconfig and looks to see if it is possible
1236 to add an 'imply' for 'config' to that part of the Kconfig.
1239 kconf: Kconfiglib.Kconfig object
1240 config: Name of config to check (without CONFIG_ prefix)
1241 imply_config: Implying config (without CONFIG_ prefix) which may or
1242 may not have an 'imply' for 'config')
1246 filename of Kconfig file containing imply_config, or None if none
1247 line number within the Kconfig file, or 0 if none
1248 message indicating the result
1250 sym = kconf.syms.get(imply_config)
1252 return 'cannot find sym'
1255 return '%d locations' % len(nodes)
1257 fname, linenum = node.filename, node.linenr
1259 if cwd and fname.startswith(cwd):
1260 fname = fname[len(cwd) + 1:]
1261 file_line = ' at %s:%d' % (fname, linenum)
1262 data = read_file(fname)
1263 if data[linenum - 1] != 'config %s' % imply_config:
1264 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line)
1265 return fname, linenum, 'adding%s' % file_line
1267 def add_imply_rule(config, fname, linenum):
1268 """Add a new 'imply' option to a Kconfig
1271 config: config option to add an imply for (without CONFIG_ prefix)
1272 fname: Kconfig filename to update
1273 linenum: Line number to place the 'imply' before
1276 Message indicating the result
1278 file_line = ' at %s:%d' % (fname, linenum)
1279 data = read_file(fname)
1282 for offset, line in enumerate(data[linenum:]):
1283 if line.strip().startswith('help') or not line:
1284 data.insert(linenum + offset, '\timply %s' % config)
1285 write_file(fname, data)
1286 return 'added%s' % file_line
1288 return 'could not insert%s'
1290 (IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
1294 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
1295 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
1296 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
1298 IMPLY_NON_ARCH_BOARD,
1299 'Allow Kconfig options outside arch/ and /board/ to imply'],
1303 def read_database():
1304 """Read in the config database
1308 set of all config options seen (each a str)
1309 set of all defconfigs seen (each a str)
1310 dict of configs for each defconfig:
1311 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
1314 value: Value of option
1315 dict of defconfigs for each config:
1317 value: set of boards using that option
1322 # key is defconfig name, value is dict of (CONFIG_xxx, value)
1325 # Set of all config options we have seen
1328 # Set of all defconfigs we have seen
1329 all_defconfigs = set()
1331 defconfig_db = collections.defaultdict(set)
1332 for line in read_file(CONFIG_DATABASE):
1333 line = line.rstrip()
1334 if not line: # Separator between defconfigs
1335 config_db[defconfig] = configs
1336 all_defconfigs.add(defconfig)
1338 elif line[0] == ' ': # CONFIG line
1339 config, value = line.strip().split('=', 1)
1340 configs[config] = value
1341 defconfig_db[config].add(defconfig)
1342 all_configs.add(config)
1343 else: # New defconfig
1346 return all_configs, all_defconfigs, config_db, defconfig_db
1349 def do_imply_config(config_list, add_imply, imply_flags, skip_added,
1350 check_kconfig=True, find_superset=False):
1351 """Find CONFIG options which imply those in the list
1353 Some CONFIG options can be implied by others and this can help to reduce
1354 the size of the defconfig files. For example, CONFIG_X86 implies
1355 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
1356 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
1357 each of the x86 defconfig files.
1359 This function uses the moveconfig database to find such options. It
1360 displays a list of things that could possibly imply those in the list.
1361 The algorithm ignores any that start with CONFIG_TARGET since these
1362 typically refer to only a few defconfigs (often one). It also does not
1363 display a config with less than 5 defconfigs.
1365 The algorithm works using sets. For each target config in config_list:
1366 - Get the set 'defconfigs' which use that target config
1367 - For each config (from a list of all configs):
1368 - Get the set 'imply_defconfig' of defconfigs which use that config
1370 - If imply_defconfigs contains anything not in defconfigs then
1371 this config does not imply the target config
1374 config_list: List of CONFIG options to check (each a string)
1375 add_imply: Automatically add an 'imply' for each config.
1376 imply_flags: Flags which control which implying configs are allowed
1378 skip_added: Don't show options which already have an imply added.
1379 check_kconfig: Check if implied symbols already have an 'imply' or
1380 'select' for the target config, and show this information if so.
1381 find_superset: True to look for configs which are a superset of those
1382 already found. So for example if CONFIG_EXYNOS5 implies an option,
1383 but CONFIG_EXYNOS covers a larger set of defconfigs and also
1384 implies that option, this will drop the former in favour of the
1385 latter. In practice this option has not proved very used.
1387 Note the terminoloy:
1388 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
1389 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
1391 kconf = KconfigScanner().conf if check_kconfig else None
1392 if add_imply and add_imply != 'all':
1393 add_imply = add_imply.split(',')
1395 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
1397 # Work through each target config option in turn, independently
1398 for config in config_list:
1399 defconfigs = defconfig_db.get(config)
1401 print('%s not found in any defconfig' % config)
1404 # Get the set of defconfigs without this one (since a config cannot
1406 non_defconfigs = all_defconfigs - defconfigs
1407 num_defconfigs = len(defconfigs)
1408 print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
1411 # This will hold the results: key=config, value=defconfigs containing it
1413 rest_configs = all_configs - set([config])
1415 # Look at every possible config, except the target one
1416 for imply_config in rest_configs:
1417 if 'ERRATUM' in imply_config:
1419 if not imply_flags & IMPLY_CMD:
1420 if 'CONFIG_CMD' in imply_config:
1422 if not imply_flags & IMPLY_TARGET:
1423 if 'CONFIG_TARGET' in imply_config:
1426 # Find set of defconfigs that have this config
1427 imply_defconfig = defconfig_db[imply_config]
1429 # Get the intersection of this with defconfigs containing the
1431 common_defconfigs = imply_defconfig & defconfigs
1433 # Get the set of defconfigs containing this config which DO NOT
1434 # also contain the taret config. If this set is non-empty it means
1435 # that this config affects other defconfigs as well as (possibly)
1436 # the ones affected by the target config. This means it implies
1437 # things we don't want to imply.
1438 not_common_defconfigs = imply_defconfig & non_defconfigs
1439 if not_common_defconfigs:
1442 # If there are common defconfigs, imply_config may be useful
1443 if common_defconfigs:
1446 for prev in list(imply_configs.keys()):
1447 prev_count = len(imply_configs[prev])
1448 count = len(common_defconfigs)
1449 if (prev_count > count and
1450 (imply_configs[prev] & common_defconfigs ==
1451 common_defconfigs)):
1452 # skip imply_config because prev is a superset
1455 elif count > prev_count:
1456 # delete prev because imply_config is a superset
1457 del imply_configs[prev]
1459 imply_configs[imply_config] = common_defconfigs
1461 # Now we have a dict imply_configs of configs which imply each config
1462 # The value of each dict item is the set of defconfigs containing that
1463 # config. Rank them so that we print the configs that imply the largest
1464 # number of defconfigs first.
1465 ranked_iconfigs = sorted(imply_configs,
1466 key=lambda k: len(imply_configs[k]), reverse=True)
1469 add_list = collections.defaultdict(list)
1470 for iconfig in ranked_iconfigs:
1471 num_common = len(imply_configs[iconfig])
1473 # Don't bother if there are less than 5 defconfigs affected.
1474 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
1476 missing = defconfigs - imply_configs[iconfig]
1477 missing_str = ', '.join(missing) if missing else 'all'
1481 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
1482 iconfig[CONFIG_LEN:])
1487 fname, linenum = nodes[0].filename, nodes[0].linenr
1488 if cwd and fname.startswith(cwd):
1489 fname = fname[len(cwd) + 1:]
1490 kconfig_info = '%s:%d' % (fname, linenum)
1494 sym = kconf.syms.get(iconfig[CONFIG_LEN:])
1499 fname, linenum = nodes[0].filename, nodes[0].linenr
1500 if cwd and fname.startswith(cwd):
1501 fname = fname[len(cwd) + 1:]
1502 in_arch_board = not sym or (fname.startswith('arch') or
1503 fname.startswith('board'))
1504 if (not in_arch_board and
1505 not imply_flags & IMPLY_NON_ARCH_BOARD):
1508 if add_imply and (add_imply == 'all' or
1509 iconfig in add_imply):
1510 fname, linenum, kconfig_info = (check_imply_rule(kconf,
1511 config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
1513 add_list[fname].append(linenum)
1515 if show and kconfig_info != 'skip':
1516 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
1517 kconfig_info, missing_str))
1519 # Having collected a list of things to add, now we add them. We process
1520 # each file from the largest line number to the smallest so that
1521 # earlier additions do not affect our line numbers. E.g. if we added an
1522 # imply at line 20 it would change the position of each line after
1524 for fname, linenums in add_list.items():
1525 for linenum in sorted(linenums, reverse=True):
1526 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
1528 def defconfig_matches(configs, re_match):
1529 """Check if any CONFIG option matches a regex
1531 The match must be complete, i.e. from the start to end of the CONFIG option.
1534 configs (dict): Dict of CONFIG options:
1536 value: Value of option
1537 re_match (re.Pattern): Match to check
1540 bool: True if any CONFIG matches the regex
1543 if re_match.fullmatch(cfg):
1547 def do_find_config(config_list):
1548 """Find boards with a given combination of CONFIGs
1551 config_list: List of CONFIG options to check (each a regex consisting
1552 of a config option, with or without a CONFIG_ prefix. If an option
1553 is preceded by a tilde (~) then it must be false, otherwise it must
1556 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
1558 # Start with all defconfigs
1559 out = all_defconfigs
1561 # Work through each config in turn
1562 for item in config_list:
1563 # Get the real config name and whether we want this config or not
1570 # Search everything that is still in the running. If it has a config
1571 # that we want, or doesn't have one that we don't, add it into the
1572 # running for the next stage
1575 re_match = re.compile(cfg)
1576 for defc in in_list:
1577 has_cfg = defconfig_matches(config_db[defc], re_match)
1580 print(f'{len(out)} matches')
1581 print(' '.join(item.split('_defconfig')[0] for item in out))
1584 def prefix_config(cfg):
1585 """Prefix a config with CONFIG_ if needed
1587 This handles ~ operator, which indicates that the CONFIG should be disabled
1589 >>> prefix_config('FRED')
1591 >>> prefix_config('CONFIG_FRED')
1593 >>> prefix_config('~FRED')
1595 >>> prefix_config('~CONFIG_FRED')
1597 >>> prefix_config('A123')
1604 if not cfg.startswith('CONFIG_'):
1605 cfg = 'CONFIG_' + cfg
1609 RE_MK_CONFIGS = re.compile('CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
1610 RE_IFDEF = re.compile('(ifdef|ifndef)')
1611 RE_C_CONFIGS = re.compile('CONFIG_([A-Za-z0-9_]*)')
1612 RE_CONFIG_IS = re.compile('CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
1615 def __init__(self, cfg, is_spl, fname, rest):
1617 self.is_spl = is_spl
1622 return hash((self.cfg, self.is_spl))
1624 def scan_makefiles(fnames):
1625 """Scan Makefiles looking for Kconfig options
1627 Looks for uses of CONFIG options in Makefiles
1630 fnames (list of tuple):
1631 str: Makefile filename where the option was found
1632 str: Line of the Makefile
1637 key (ConfigUse): object
1638 value (list of str): matching lines
1639 dict: Uses by filename
1641 value (set of ConfigUse): uses in that filename
1643 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
1645 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
1647 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
1648 ('$(SPL_TPL_)', 'MARY')
1650 all_uses = collections.defaultdict(list)
1652 for fname, rest in fnames:
1653 m_iter = RE_MK_CONFIGS.finditer(rest)
1657 real_opt = m.group(2)
1663 use = ConfigUse(real_opt, is_spl, fname, rest)
1664 if fname not in fname_uses:
1665 fname_uses[fname] = set()
1666 fname_uses[fname].add(use)
1667 all_uses[use].append(rest)
1668 return all_uses, fname_uses
1671 def scan_src_files(fnames):
1672 """Scan source files (other than Makefiles) looking for Kconfig options
1674 Looks for uses of CONFIG options
1677 fnames (list of tuple):
1678 str: Makefile filename where the option was found
1679 str: Line of the Makefile
1684 key (ConfigUse): object
1685 value (list of str): matching lines
1686 dict: Uses by filename
1688 value (set of ConfigUse): uses in that filename
1690 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
1692 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
1694 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
1697 def add_uses(m_iter, is_spl):
1700 real_opt = m.group(1)
1703 use = ConfigUse(real_opt, is_spl, fname, rest)
1704 if fname not in fname_uses:
1705 fname_uses[fname] = set()
1706 fname_uses[fname].add(use)
1707 all_uses[use].append(rest)
1709 all_uses = collections.defaultdict(list)
1711 for fname, rest in fnames:
1712 m_iter = RE_C_CONFIGS.finditer(rest)
1713 add_uses(m_iter, False)
1715 m_iter2 = RE_CONFIG_IS.finditer(rest)
1716 add_uses(m_iter2, True)
1718 return all_uses, fname_uses
1721 MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
1723 def do_scan_source(path, do_update):
1724 """Scan the source tree for Kconfig inconsistencies
1727 path (str): Path to source tree
1728 do_update (bool) : True to write to scripts/kconf_... files
1730 def is_not_proper(name):
1731 for prefix in SPL_PREFIXES:
1732 if name.startswith(prefix):
1733 return name[len(prefix):]
1736 def check_not_found(all_uses, spl_mode):
1737 """Check for Kconfig options mentioned in the source but not in Kconfig
1741 key (ConfigUse): object
1742 value (list of str): matching lines
1743 spl_mode (int): If MODE_SPL, look at source code which implies
1744 an SPL_ option, but for which there is none;
1745 for MOD_PROPER, look at source code which implies a Proper
1746 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
1747 $(SPL_TPL_) but for which there none;
1748 if MODE_NORMAL, ignore SPL
1752 key (str): CONFIG name (without 'CONFIG_' prefix
1753 value (list of ConfigUse): List of uses of this CONFIG
1755 # Make sure we know about all the options
1756 not_found = collections.defaultdict(list)
1757 for use, rest in all_uses.items():
1759 if name in IGNORE_SYMS:
1763 if spl_mode == MODE_SPL:
1766 # If it is an SPL symbol, try prepending all SPL_ prefixes to
1767 # find at least one SPL symbol
1770 for prefix in SPL_PREFIXES:
1771 try_name = prefix + name
1772 sym = kconf.syms.get(try_name)
1776 not_found[f'SPL_{name}'].append(use)
1778 elif spl_mode == MODE_PROPER:
1779 # Try to find the Proper version of this symbol, i.e. without
1781 proper_name = is_not_proper(name)
1784 elif not use.is_spl:
1788 sym = kconf.syms.get(name)
1790 proper_name = is_not_proper(name)
1793 sym = kconf.syms.get(name)
1795 for prefix in SPL_PREFIXES:
1796 try_name = prefix + name
1797 sym = kconf.syms.get(try_name)
1801 not_found[name].append(use)
1804 sym = kconf.syms.get(name)
1805 if not sym and check:
1806 not_found[name].append(use)
1809 def show_uses(uses):
1810 """Show a list of uses along with their filename and code snippet
1814 key (str): CONFIG name (without 'CONFIG_' prefix
1815 value (list of ConfigUse): List of uses of this CONFIG
1817 for name in sorted(uses):
1818 print(f'{name}: ', end='')
1819 for i, use in enumerate(uses[name]):
1820 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}')
1823 print('Scanning Kconfig')
1824 kconf = KconfigScanner().conf
1825 print(f'Scanning source in {path}')
1826 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
1827 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
1828 out, err = proc.communicate()
1829 lines = out.splitlines()
1830 re_fname = re.compile('^([^:]*):(.*)')
1834 linestr = line.decode('utf-8')
1835 m_fname = re_fname.search(linestr)
1838 fname, rest = m_fname.groups()
1839 dirname, leaf = os.path.split(fname)
1840 root, ext = os.path.splitext(leaf)
1841 if ext == '.autoconf':
1843 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
1845 src_list.append([fname, rest])
1846 elif 'Makefile' in root or ext == '.mk':
1847 mk_list.append([fname, rest])
1848 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
1850 elif 'Kconfig' in root or 'Kbuild' in root:
1852 elif 'README' in root:
1854 elif dirname in ['configs']:
1856 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
1859 print(f'Not sure how to handle file {fname}')
1861 # Scan the Makefiles
1862 all_uses, fname_uses = scan_makefiles(mk_list)
1864 spl_not_found = set()
1865 proper_not_found = set()
1867 # Make sure we know about all the options
1868 print('\nCONFIG options present in Makefiles but not Kconfig:')
1869 not_found = check_not_found(all_uses, MODE_NORMAL)
1870 show_uses(not_found)
1872 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
1873 not_found = check_not_found(all_uses, MODE_SPL)
1874 show_uses(not_found)
1875 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1877 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
1878 not_found = check_not_found(all_uses, MODE_PROPER)
1879 show_uses(not_found)
1880 proper_not_found |= set([key for key in not_found.keys()])
1882 # Scan the source code
1883 all_uses, fname_uses = scan_src_files(src_list)
1885 # Make sure we know about all the options
1886 print('\nCONFIG options present in source but not Kconfig:')
1887 not_found = check_not_found(all_uses, MODE_NORMAL)
1888 show_uses(not_found)
1890 print('\nCONFIG options present in source but not Kconfig (SPL):')
1891 not_found = check_not_found(all_uses, MODE_SPL)
1892 show_uses(not_found)
1893 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1895 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
1896 not_found = check_not_found(all_uses, MODE_PROPER)
1897 show_uses(not_found)
1898 proper_not_found |= set([key for key in not_found.keys()])
1900 print('\nCONFIG options used as SPL but without an SPL_ variant:')
1901 for item in sorted(spl_not_found):
1904 print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
1905 for item in sorted(proper_not_found):
1908 # Write out the updated information
1910 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
1911 print('# These options should not be enabled in SPL builds\n',
1913 for item in sorted(spl_not_found):
1914 print(item, file=out)
1915 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
1916 print('# These options should not be enabled in Proper builds\n',
1918 for item in sorted(proper_not_found):
1919 print(item, file=out)
1924 cpu_count = multiprocessing.cpu_count()
1925 except NotImplementedError:
1928 epilog = '''Move config options from headers to defconfig files. See
1929 doc/develop/moveconfig.rst for documentation.'''
1931 parser = ArgumentParser(epilog=epilog)
1932 # Add arguments here
1933 parser.add_argument('-a', '--add-imply', type=str, default='',
1934 help='comma-separated list of CONFIG options to add '
1935 "an 'imply' statement to for the CONFIG in -i")
1936 parser.add_argument('-A', '--skip-added', action='store_true', default=False,
1937 help="don't show options which are already marked as "
1939 parser.add_argument('-b', '--build-db', action='store_true', default=False,
1940 help='build a CONFIG database')
1941 parser.add_argument('-c', '--color', action='store_true', default=False,
1942 help='display the log in color')
1943 parser.add_argument('-C', '--commit', action='store_true', default=False,
1944 help='Create a git commit for the operation')
1945 parser.add_argument('-d', '--defconfigs', type=str,
1946 help='a file containing a list of defconfigs to move, '
1947 "one per line (for example 'snow_defconfig') "
1948 "or '-' to read from stdin")
1949 parser.add_argument('-e', '--exit-on-error', action='store_true',
1951 help='exit immediately on any error')
1952 parser.add_argument('-f', '--find', action='store_true', default=False,
1953 help='Find boards with a given config combination')
1954 parser.add_argument('-H', '--headers-only', dest='cleanup_headers_only',
1955 action='store_true', default=False,
1956 help='only cleanup the headers')
1957 parser.add_argument('-i', '--imply', action='store_true', default=False,
1958 help='find options which imply others')
1959 parser.add_argument('-I', '--imply-flags', type=str, default='',
1960 help="control the -i option ('help' for help")
1961 parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
1962 help='the number of jobs to run simultaneously')
1963 parser.add_argument('-n', '--dry-run', action='store_true', default=False,
1964 help='perform a trial run (show log with no changes)')
1965 parser.add_argument('-r', '--git-ref', type=str,
1966 help='the git ref to clone for building the autoconf.mk')
1967 parser.add_argument('-s', '--force-sync', action='store_true', default=False,
1968 help='force sync by savedefconfig')
1969 parser.add_argument('-S', '--spl', action='store_true', default=False,
1970 help='parse config options defined for SPL build')
1971 parser.add_argument('--scan-source', action='store_true', default=False,
1972 help='scan source for uses of CONFIG options')
1973 parser.add_argument('-t', '--test', action='store_true', default=False,
1974 help='run unit tests')
1975 parser.add_argument('-y', '--yes', action='store_true', default=False,
1976 help="respond 'yes' to any prompts")
1977 parser.add_argument('-u', '--update', action='store_true', default=False,
1978 help="update scripts/ files (use with --scan-source)")
1979 parser.add_argument('-v', '--verbose', action='store_true', default=False,
1980 help='show any build errors as boards are built')
1981 parser.add_argument('configs', nargs='*')
1983 args = parser.parse_args()
1984 configs = args.configs
1987 sys.argv = [sys.argv[0]]
1988 fail, count = doctest.testmod()
1993 if args.scan_source:
1994 do_scan_source(os.getcwd(), args.update)
1997 if not any((len(configs), args.force_sync, args.build_db, args.imply,
1999 parser.print_usage()
2002 # prefix the option name with CONFIG_ if missing
2003 configs = [prefix_config(cfg) for cfg in configs]
2005 check_top_directory()
2009 if args.imply_flags == 'all':
2012 elif args.imply_flags:
2013 for flag in args.imply_flags.split(','):
2014 bad = flag not in IMPLY_FLAGS
2016 print("Invalid flag '%s'" % flag)
2017 if flag == 'help' or bad:
2018 print("Imply flags: (separate with ',')")
2019 for name, info in IMPLY_FLAGS.items():
2020 print(' %-15s: %s' % (name, info[1]))
2021 parser.print_usage()
2023 imply_flags |= IMPLY_FLAGS[flag][0]
2025 do_imply_config(configs, args.add_imply, imply_flags, args.skip_added)
2029 do_find_config(configs)
2033 db_queue = queue.Queue()
2034 t = DatabaseThread(config_db, db_queue)
2038 if not args.cleanup_headers_only:
2039 check_clean_directory()
2041 toolchains = toolchain.Toolchains()
2042 toolchains.GetSettings()
2043 toolchains.Scan(verbose=False)
2044 move_config(toolchains, configs, args, db_queue)
2048 cleanup_headers(configs, args)
2049 cleanup_readme(configs, args)
2052 subprocess.call(['git', 'add', '-u'])
2054 msg = 'Convert %s %sto Kconfig' % (configs[0],
2055 'et al ' if len(configs) > 1 else '')
2056 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
2057 '\n '.join(configs))
2059 msg = 'configs: Resync with savedefconfig'
2060 msg += '\n\nRsync all defconfig files using moveconfig.py'
2061 subprocess.call(['git', 'commit', '-s', '-m', msg])
2064 with open(CONFIG_DATABASE, 'w', encoding='utf-8') as fd:
2065 for defconfig, configs in config_db.items():
2066 fd.write('%s\n' % defconfig)
2067 for config in sorted(configs.keys()):
2068 fd.write(' %s=%s\n' % (config, configs[config]))
2071 if __name__ == '__main__':