From d73fcb12e27277d5b3e80399b4b3ec41abcd80d2 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 1 Jun 2017 19:39:02 -0600 Subject: [PATCH] moveconfig: Support building a simple config database Add a -b option which scans all the defconfigs and builds a database of all the CONFIG options used by each. This is useful for querying later. At present this only works with the separate -b option, which does not move any configs. It would be possible to adjust the script to build the database automatically when moving configs, but this might not be useful as the database does not change that often. Signed-off-by: Simon Glass --- tools/moveconfig.py | 81 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/tools/moveconfig.py b/tools/moveconfig.py index eb7ada2838..00101ba43d 100755 --- a/tools/moveconfig.py +++ b/tools/moveconfig.py @@ -203,11 +203,13 @@ import glob import multiprocessing import optparse import os +import Queue import re import shutil import subprocess import sys import tempfile +import threading import time SHOW_GNU_MAKE = 'scripts/show-gnu-make' @@ -263,6 +265,7 @@ COLOR_LIGHT_CYAN = '1;36' COLOR_WHITE = '1;37' AUTO_CONF_PATH = 'include/config/auto.conf' +CONFIG_DATABASE = 'moveconfig.db' ### helper functions ### @@ -940,6 +943,34 @@ class KconfigParser: return log + +class DatabaseThread(threading.Thread): + """This thread processes results from Slot threads. + + It collects the data in the master config directary. There is only one + result thread, and this helps to serialise the build output. + """ + def __init__(self, config_db, db_queue): + """Set up a new result thread + + Args: + builder: Builder which will be sent each result + """ + threading.Thread.__init__(self) + self.config_db = config_db + self.db_queue= db_queue + + def run(self): + """Called to start up the result thread. + + We collect the next result job and pass it on to the build. + """ + while True: + defconfig, configs = self.db_queue.get() + self.config_db[defconfig] = configs + self.db_queue.task_done() + + class Slot: """A slot to store a subprocess. @@ -949,7 +980,8 @@ class Slot: for faster processing. """ - def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): + def __init__(self, configs, options, progress, devnull, make_cmd, + reference_src_dir, db_queue): """Create a new process slot. Arguments: @@ -960,6 +992,7 @@ class Slot: make_cmd: command name of GNU Make. reference_src_dir: Determine the true starting config state from this source tree. + db_queue: output queue to write config info for the database """ self.options = options self.progress = progress @@ -967,6 +1000,7 @@ class Slot: self.devnull = devnull self.make_cmd = (make_cmd, 'O=' + self.build_dir) self.reference_src_dir = reference_src_dir + self.db_queue = db_queue self.parser = KconfigParser(configs, options, self.build_dir) self.state = STATE_IDLE self.failed_boards = set() @@ -1042,6 +1076,8 @@ class Slot: if self.current_src_dir: self.current_src_dir = None self.do_defconfig() + elif self.options.build_db: + self.do_build_db() else: self.do_savedefconfig() elif self.state == STATE_SAVEDEFCONFIG: @@ -1091,6 +1127,17 @@ class Slot: cwd=self.current_src_dir) self.state = STATE_AUTOCONF + def do_build_db(self): + """Add the board to the database""" + configs = {} + with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: + for line in fd.readlines(): + if line.startswith('CONFIG'): + config, value = line.split('=', 1) + configs[config] = value.rstrip() + self.db_queue.put([self.defconfig, configs]) + self.finish(True) + def do_savedefconfig(self): """Update the .config and run 'make savedefconfig'.""" @@ -1173,7 +1220,7 @@ class Slots: """Controller of the array of subprocess slots.""" - def __init__(self, configs, options, progress, reference_src_dir): + def __init__(self, configs, options, progress, reference_src_dir, db_queue): """Create a new slots controller. Arguments: @@ -1182,6 +1229,7 @@ class Slots: progress: A progress indicator. reference_src_dir: Determine the true starting config state from this source tree. + db_queue: output queue to write config info for the database """ self.options = options self.slots = [] @@ -1189,7 +1237,7 @@ class Slots: make_cmd = get_make_cmd() for i in range(options.jobs): self.slots.append(Slot(configs, options, progress, devnull, - make_cmd, reference_src_dir)) + make_cmd, reference_src_dir, db_queue)) def add(self, defconfig): """Add a new subprocess if a vacant slot is found. @@ -1301,7 +1349,7 @@ class ReferenceSource: return self.src_dir -def move_config(configs, options): +def move_config(configs, options, db_queue): """Move config options to defconfig files. Arguments: @@ -1311,6 +1359,8 @@ def move_config(configs, options): if len(configs) == 0: if options.force_sync: print 'No CONFIG is specified. You are probably syncing defconfigs.', + elif options.build_db: + print 'Building %s database' % CONFIG_DATABASE else: print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', else: @@ -1329,7 +1379,7 @@ def move_config(configs, options): defconfigs = get_all_defconfigs() progress = Progress(len(defconfigs)) - slots = Slots(configs, options, progress, reference_src_dir) + slots = Slots(configs, options, progress, reference_src_dir, db_queue) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -1356,6 +1406,8 @@ def main(): parser = optparse.OptionParser() # Add options here + parser.add_option('-b', '--build-db', action='store_true', default=False, + help='build a CONFIG database') parser.add_option('-c', '--color', action='store_true', default=False, help='display the log in color') parser.add_option('-C', '--commit', action='store_true', default=False, @@ -1388,7 +1440,7 @@ def main(): (options, configs) = parser.parse_args() - if len(configs) == 0 and not options.force_sync: + if len(configs) == 0 and not any((options.force_sync, options.build_db)): parser.print_usage() sys.exit(1) @@ -1398,10 +1450,17 @@ def main(): check_top_directory() + config_db = {} + db_queue = Queue.Queue() + t = DatabaseThread(config_db, db_queue) + t.setDaemon(True) + t.start() + if not options.cleanup_headers_only: check_clean_directory() update_cross_compile(options.color) - move_config(configs, options) + move_config(configs, options, db_queue) + db_queue.join() if configs: cleanup_headers(configs, options) @@ -1421,5 +1480,13 @@ def main(): msg += '\n\nRsync all defconfig files using moveconfig.py' subprocess.call(['git', 'commit', '-s', '-m', msg]) + if options.build_db: + with open(CONFIG_DATABASE, 'w') as fd: + for defconfig, configs in config_db.iteritems(): + print >>fd, '%s' % defconfig + for config in sorted(configs.keys()): + print >>fd, ' %s=%s' % (config, configs[config]) + print >>fd + if __name__ == '__main__': main() -- 2.34.1