From 7d499b5ffcea20bc541854c1dc1360c4221c3c5f Mon Sep 17 00:00:00 2001 From: joey-lunarg Date: Thu, 26 Jul 2018 10:35:39 -0600 Subject: [PATCH] build: Add known-good support Change-Id: I05fe4162054fc90edd1642ab08644f6ee7950548 --- BUILD.md | 49 +++- scripts/known_good.json | 45 ++++ scripts/update_deps.py | 578 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 scripts/known_good.json create mode 100755 scripts/update_deps.py diff --git a/BUILD.md b/BUILD.md index 0ca4cba..e18f36e 100644 --- a/BUILD.md +++ b/BUILD.md @@ -110,6 +110,53 @@ child of the build directory with the name `install`. The remainder of these instructions follow this convention, although you can use any name for these directories and place them in any location. +### Building Dependent Repositories with Known-Good Revisions + +There is a Python utility script, `scripts/update_deps.py`, that you can use +to gather and build the dependent repositories mentioned above. This program +also uses information stored in the `scripts/known-good.json` file to checkout +dependent repository revisions that are known to be compatible with the +revision of this repository that you currently have checked out. + +Here is a usage example for this repository: + + git clone git@github.com:KhronosGroup/Vulkan-Tools.git + cd Vulkan-Tools + mkdir build + cd build + ../scripts/update_deps.py + cmake -C helper.cmake .. + cmake --build . + +#### Notes + +- You may need to adjust some of the CMake options based on your platform. See + the platform-specific sections later in this document. +- The `update_deps.py` script fetches and builds the dependent repositories in + the current directory when it is invoked. In this case, they are built in + the `build` directory. +- The `build` directory is also being used to build this + (Vulkan-Tools) repository. But there shouldn't be any conflicts + inside the `build` directory between the dependent repositories and the + build files for this repository. +- The `--dir` option for `update_deps.py` can be used to relocate the + dependent repositories to another arbitrary directory using an absolute or + relative path. +- The `update_deps.py` script generates a file named `helper.cmake` and places + it in the same directory as the dependent repositories (`build` in this + case). This file contains CMake commands to set the CMake `*_INSTALL_DIR` + variables that are used to point to the install artifacts of the dependent + repositories. You can use this file with the `cmake -C` option to set these + variables when you generate your build files with CMake. This lets you avoid + entering several `*_INSTALL_DIR` variable settings on the CMake command line. +- If using "MINGW" (Git For Windows), you may wish to run + `winpty update_deps.py` in order to avoid buffering all of the script's + "print" output until the end and to retain the ability to interrupt script + execution. +- Please use `update_deps.py --help` to list additional options and read the + internal documentation in `update_deps.py` for further information. + + ### Build Options When generating native platform build files through CMake, several options can @@ -186,7 +233,7 @@ See below for the details. Change your current directory to the top of the cloned repository directory, create a build directory and generate the Visual Studio project files: - cd Vulkan-Loader + cd Vulkan-Tools mkdir build cd build cmake -A x64 -DVULKAN_HEADERS_INSTALL_DIR=absolute_path_to_install_dir diff --git a/scripts/known_good.json b/scripts/known_good.json new file mode 100644 index 0000000..a55249a --- /dev/null +++ b/scripts/known_good.json @@ -0,0 +1,45 @@ +{ + "repos" : [ + { + "name" : "glslang", + "url" : "https://github.com/KhronosGroup/glslang.git", + "sub_dir" : "glslang", + "build_dir" : "glslang/build", + "install_dir" : "glslang/build/install", + "commit" : "e99a26810f65314183163c07664a40e05647c15f", + "prebuild" : [ + "python update_glslang_sources.py" + ] + }, + { + "name" : "Vulkan-Headers", + "url" : "https://github.com/KhronosGroup/Vulkan-Headers.git", + "sub_dir" : "Vulkan-Headers", + "build_dir" : "Vulkan-Headers/build", + "install_dir" : "Vulkan-Headers/build/install", + "commit" : "c4e056d365472174471a243dfefbfe66a03564af" + }, + { + "name" : "Vulkan-Loader", + "url" : "https://github.com/KhronosGroup/Vulkan-Loader.git", + "sub_dir" : "Vulkan-Loader", + "build_dir" : "Vulkan-Loader/build", + "install_dir" : "Vulkan-Loader/build/install", + "commit" : "dbf8f2cd85190ac902f1da57482a6e340f05e860", + "deps" : [ + { + "var_name" : "VULKAN_HEADERS_INSTALL_DIR", + "repo_name" : "Vulkan-Headers" + } + ], + "cmake_options" : [ + "-DBUILD_TESTS=NO" + ] + } + ], + "install_names" : { + "glslang" : "GLSLANG_INSTALL_DIR", + "Vulkan-Headers" : "VULKAN_HEADERS_INSTALL_DIR", + "Vulkan-Loader" : "VULKAN_LOADER_INSTALL_DIR" + } +} diff --git a/scripts/update_deps.py b/scripts/update_deps.py new file mode 100755 index 0000000..8da65a0 --- /dev/null +++ b/scripts/update_deps.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python + +# Copyright 2017 The Glslang Authors. All rights reserved. +# Copyright (c) 2018 Valve Corporation +# Copyright (c) 2018 LunarG, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script was heavily leveraged from KhronosGroup/glslang +# update_glslang_sources.py. +"""update_deps.py + +Get and build dependent repositories using known-good commits. + +Purpose +------- + +This program is intended to assist a developer of this repository +(the "home" repository) by gathering and building the repositories that +this home repository depend on. It also checks out each dependent +repository at a "known-good" commit in order to provide stability in +the dependent repositories. + +Python Compatibility +-------------------- + +This program can be used with Python 2.7 and Python 3. + +Known-Good JSON Database +------------------------ + +This program expects to find a file named "known-good.json" in the +same directory as the program file. This JSON file is tailored for +the needs of the home repository by including its dependent repositories. + +Program Options +--------------- + +See the help text (update_deps.py --help) for a complete list of options. + +Program Operation +----------------- + +The program uses the user's current directory at the time of program +invocation as the location for fetching and building the dependent +repositories. The user can override this by using the "--dir" option. + +For example, a directory named "build" in the repository's root directory +is a good place to put the dependent repositories because that directory +is not tracked by Git. (See the .gitignore file.) The "external" directory +may also be a suitable location. +A user can issue: + +$ cd My-Repo +$ mkdir build +$ cd build +$ ../scripts/update_deps.py + +or, to do the same thing, but using the --dir option: + +$ cd My-Repo +$ mkdir build +$ scripts/update_deps.py --dir=build + +With these commands, the "build" directory is considered the "top" +directory where the program clones the dependent repositories. The +JSON file configures the build and install working directories to be +within this "top" directory. + +Note that the "dir" option can also specify an absolute path: + +$ cd My-Repo +$ scripts/update_deps.py --dir=/tmp/deps + +The "top" dir is then /tmp/deps (Linux filesystem example) and is +where this program will clone and build the dependent repositories. + +Helper CMake Config File +------------------------ + +When the program finishes building the dependencies, it writes a file +named "helper.cmake" to the "top" directory that contains CMake commands +for setting CMake variables for locating the dependent repositories. +This helper file can be used to set up the CMake build files for this +"home" repository. + +A complete sequence might look like: + +$ git clone git@github.com:My-Group/My-Repo.git +$ cd My-Repo +$ mkdir build +$ cd build +$ ../scripts/update_deps.py +$ cmake -C helper.cmake .. +$ cmake --build . + +JSON File Schema +---------------- + +There's no formal schema for the "known-good" JSON file, but here is +a description of its elements. All elements are required except those +marked as optional. Please see the "known_good.json" file for +examples of all of these elements. + +- name + +The name of the dependent repository. This field can be referenced +by the "deps.repo_name" structure to record a dependency. + +- url + +Specifies the URL of the repository. +Example: https://github.com/KhronosGroup/Vulkan-Loader.git + +- sub_dir + +The directory where the program clones the repository, relative to +the "top" directory. + +- build_dir + +The directory used to build the repository, relative to the "top" +directory. + +- install_dir + +The directory used to store the installed build artifacts, relative +to the "top" directory. + +- commit + +The commit used to checkout the repository. This can be a SHA-1 +object name or a refname used with the remote name "origin". +For example, this field can be set to "origin/sdk-1.1.77" to +select the end of the sdk-1.1.77 branch. + +- deps (optional) + +An array of pairs consisting of a CMake variable name and a +repository name to specify a dependent repo and a "link" to +that repo's install artifacts. For example: + +"deps" : [ + { + "var_name" : "VULKAN_HEADERS_INSTALL_DIR", + "repo_name" : "Vulkan-Headers" + } +] + +which represents that this repository depends on the Vulkan-Headers +repository and uses the VULKAN_HEADERS_INSTALL_DIR CMake variable to +specify the location where it expects to find the Vulkan-Headers install +directory. +Note that the "repo_name" element must match the "name" element of some +other repository in the JSON file. + +- prebuild (optional) +- prebuild_linux (optional) (For Linux and MacOS) +- prebuild_windows (optional) + +A list of commands to execute before building a dependent repository. +This is useful for repositories that require the execution of some +sort of "update" script or need to clone an auxillary repository like +googletest. + +The commands listed in "prebuild" are executed first, and then the +commands for the specific platform are executed. + +- cmake_options (optional) + +A list of options to pass to CMake during the generation phase. + +- ci_only (optional) + +A list of environment variables where one must be set to "true" +(case-insensitive) in order for this repo to be fetched and built. +This list can be used to specify repos that should be built only in CI. +Typically, this list might contain "TRAVIS" and/or "APPVEYOR" because +each of these CI systems sets an environment variable with its own +name to "true". Note that this could also be (ab)used to control +the processing of the repo with any environment variable. The default +is an empty list, which means that the repo is always processed. + +Note +---- + +The "sub_dir", "build_dir", and "install_dir" elements are all relative +to the effective "top" directory. Specifying absolute paths is not +supported. However, the "top" directory specified with the "--dir" +option can be a relative or absolute path. + +""" + +from __future__ import print_function + +import argparse +import json +import distutils.dir_util +import os.path +import subprocess +import sys +import platform +import multiprocessing +import shutil + +KNOWN_GOOD_FILE_NAME = 'known_good.json' + +CONFIG_MAP = { + 'debug': 'Debug', + 'release': 'Release', + 'relwithdebinfo': 'RelWithDebInfo', + 'minsizerel': 'MinSizeRel' +} + +VERBOSE = False + +DEVNULL = open(os.devnull, 'wb') + + +def command_output(cmd, directory, fail_ok=False): + """Runs a command in a directory and returns its standard output stream. + + Captures the standard error stream and prints it if error. + + Raises a RuntimeError if the command fails to launch or otherwise fails. + """ + if VERBOSE: + print('In {d}: {cmd}'.format(d=directory, cmd=cmd)) + p = subprocess.Popen( + cmd, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + if p.returncode != 0: + print('*** Error ***\nstderr contents:\n{}'.format(stderr)) + if not fail_ok: + raise RuntimeError('Failed to run {} in {}'.format(cmd, directory)) + if VERBOSE: + print(stdout) + return stdout + + +class GoodRepo(object): + """Represents a repository at a known-good commit.""" + + def __init__(self, json, args): + """Initializes this good repo object. + + Args: + 'json': A fully populated JSON object describing the repo. + 'args': Results from ArgumentParser + """ + self._json = json + self._args = args + # Required JSON elements + self.name = json['name'] + self.url = json['url'] + self.sub_dir = json['sub_dir'] + self.commit = json['commit'] + # Optional JSON elements + self.build_dir = None + self.install_dir = None + if json.get('build_dir'): + self.build_dir = json['build_dir'] + if json.get('install_dir'): + self.install_dir = json['install_dir'] + self.deps = json['deps'] if ('deps' in json) else [] + self.prebuild = json['prebuild'] if ('prebuild' in json) else [] + self.prebuild_linux = json['prebuild_linux'] if ( + 'prebuild_linux' in json) else [] + self.prebuild_windows = json['prebuild_windows'] if ( + 'prebuild_windows' in json) else [] + self.cmake_options = json['cmake_options'] if ( + 'cmake_options' in json) else [] + self.ci_only = json['ci_only'] if ('ci_only' in json) else [] + # Absolute paths for a repo's directories + dir_top = os.path.abspath(args.dir) + self.repo_dir = os.path.join(dir_top, self.sub_dir) + if self.build_dir: + self.build_dir = os.path.join(dir_top, self.build_dir) + if self.install_dir: + self.install_dir = os.path.join(dir_top, self.install_dir) + + def Clone(self): + distutils.dir_util.mkpath(self.repo_dir) + command_output(['git', 'clone', self.url, '.'], self.repo_dir) + + def Fetch(self): + command_output(['git', 'fetch', 'origin'], self.repo_dir) + + def Checkout(self): + print('Checking out {n} in {d}'.format(n=self.name, d=self.repo_dir)) + if self._args.do_clean_repo: + shutil.rmtree(self.repo_dir) + if not os.path.exists(os.path.join(self.repo_dir, '.git')): + self.Clone() + self.Fetch() + if len(self._args.ref): + command_output(['git', 'checkout', self._args.ref], self.repo_dir) + else: + command_output(['git', 'checkout', self.commit], self.repo_dir) + print(command_output(['git', 'status'], self.repo_dir)) + + def PreBuild(self): + """Execute any prebuild steps from the repo root""" + for p in self.prebuild: + command_output(p.split(), self.repo_dir) + if platform.system() == 'Linux' or platform.system() == 'Darwin': + for p in self.prebuild_linux: + command_output(p.split(), self.repo_dir) + if platform.system() == 'Windows': + for p in self.prebuild_windows: + command_output(p.split(), self.repo_dir) + + def CMakeConfig(self, repos): + """Build CMake command for the configuration phase and execute it""" + if self._args.do_clean_build: + shutil.rmtree(self.build_dir) + if self._args.do_clean_install: + shutil.rmtree(self.install_dir) + + # Create and change to build directory + distutils.dir_util.mkpath(self.build_dir) + os.chdir(self.build_dir) + + cmake_cmd = [ + 'cmake', self.repo_dir, + '-DCMAKE_INSTALL_PREFIX=' + self.install_dir + ] + + # For each repo this repo depends on, generate a CMake variable + # definitions for "...INSTALL_DIR" that points to that dependent + # repo's install dir. + for d in self.deps: + dep_commit = [r for r in repos if r.name == d['repo_name']] + if len(dep_commit): + cmake_cmd.append('-D{var_name}={install_dir}'.format( + var_name=d['var_name'], + install_dir=dep_commit[0].install_dir)) + + # Add any CMake options + for option in self.cmake_options: + cmake_cmd.append(option) + + # Set build config for single-configuration generators + if platform.system() == 'Linux' or platform.system() == 'Darwin': + cmake_cmd.append('-DCMAKE_BUILD_TYPE={config}'.format( + config=CONFIG_MAP[self._args.config])) + + # Use the CMake -A option to select the platform architecture + # without needing a Visual Studio generator. + if platform.system() == 'Windows': + if self._args.arch == '64' or self._args.arch == 'x64' or self._args.arch == 'win64': + cmake_cmd.append('-A') + cmake_cmd.append('x64') + + if VERBOSE: + print("CMake command: " + " ".join(cmake_cmd)) + + ret_code = subprocess.call(cmake_cmd) + if ret_code != 0: + sys.exit(ret_code) + + def CMakeBuild(self): + """Build CMake command for the build phase and execute it""" + cmake_cmd = ['cmake', '--build', self.build_dir, '--target', 'install'] + if self._args.do_clean: + cmake_cmd.append('--clean-first') + + if platform.system() == 'Windows': + cmake_cmd.append('--config') + cmake_cmd.append(CONFIG_MAP[self._args.config]) + + # Speed up the build. + if platform.system() == 'Linux' or platform.system() == 'Darwin': + cmake_cmd.append('--') + cmake_cmd.append('-j{ncpu}' + .format(ncpu=multiprocessing.cpu_count())) + if platform.system() == 'Windows': + cmake_cmd.append('--') + cmake_cmd.append('/maxcpucount') + + if VERBOSE: + print("CMake command: " + " ".join(cmake_cmd)) + + ret_code = subprocess.call(cmake_cmd) + if ret_code != 0: + sys.exit(ret_code) + + def Build(self, repos): + """Build the dependent repo""" + print('Building {n} in {d}'.format(n=self.name, d=self.repo_dir)) + print('Build dir = {b}'.format(b=self.build_dir)) + print('Install dir = {i}\n'.format(i=self.install_dir)) + + # Run any prebuild commands + self.PreBuild() + + # Build and execute CMake command for creating build files + self.CMakeConfig(repos) + + # Build and execute CMake command for the build + self.CMakeBuild() + + +def GetGoodRepos(args): + """Returns the latest list of GoodRepo objects. + + The known-good file is expected to be in the same + directory as this script unless overridden by the 'known_good_dir' + parameter. + """ + if args.known_good_dir: + known_good_file = os.path.join( os.path.abspath(args.known_good_dir), + KNOWN_GOOD_FILE_NAME) + else: + known_good_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), KNOWN_GOOD_FILE_NAME) + with open(known_good_file) as known_good: + return [ + GoodRepo(repo, args) + for repo in json.loads(known_good.read())['repos'] + ] + + +def GetInstallNames(args): + """Returns the install names list. + + The known-good file is expected to be in the same + directory as this script unless overridden by the 'known_good_dir' + parameter. + """ + if args.known_good_dir: + known_good_file = os.path.join(os.path.abspath(args.known_good_dir), + KNOWN_GOOD_FILE_NAME) + else: + known_good_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), KNOWN_GOOD_FILE_NAME) + with open(known_good_file) as known_good: + install_info = json.loads(known_good.read()) + if install_info.get('install_names'): + return install_info['install_names'] + else: + return None + + +def CreateHelper(args, repos, filename): + """Create a CMake config helper file. + + The helper file is intended to be used with 'cmake -C ' + to build this home repo using the dependencies built by this script. + + The install_names dictionary represents the CMake variables used by the + home repo to locate the install dirs of the dependent repos. + This information is baked into the CMake files of the home repo and so + this dictionary is kept with the repo via the json file. + """ + install_names = GetInstallNames(args) + with open(filename, 'w') as helper_file: + for repo in repos: + if install_names and repo.name in install_names: + helper_file.write('set({var} "{dir}" CACHE STRING "" FORCE)\n' + .format( + var=install_names[repo.name], + dir=repo.install_dir)) + + +def main(): + parser = argparse.ArgumentParser( + description='Get and build dependent repos at known-good commits') + parser.add_argument( + '--known_good_dir', + dest='known_good_dir', + help="Specify directory for known_good.json file.") + parser.add_argument( + '--dir', + dest='dir', + default='.', + help="Set target directory for repository roots. Default is \'.\'.") + parser.add_argument( + '--ref', + dest='ref', + default='', + help="Override 'commit' with git reference. E.g., 'origin/master'") + parser.add_argument( + '--no-build', + dest='do_build', + action='store_false', + help= + "Clone/update repositories and generate build files without performing compilation", + default=True) + parser.add_argument( + '--clean', + dest='do_clean', + action='store_true', + help="Clean files generated by compiler and linker before building", + default=False) + parser.add_argument( + '--clean-repo', + dest='do_clean_repo', + action='store_true', + help="Delete repository directory before building", + default=False) + parser.add_argument( + '--clean-build', + dest='do_clean_build', + action='store_true', + help="Delete build directory before building", + default=False) + parser.add_argument( + '--clean-install', + dest='do_clean_install', + action='store_true', + help="Delete install directory before building", + default=False) + parser.add_argument( + '--arch', + dest='arch', + choices=['32', '64', 'x86', 'x64', 'win32', 'win64'], + type=str.lower, + help="Set build files architecture (Windows)", + default='64') + parser.add_argument( + '--config', + dest='config', + choices=['debug', 'release', 'relwithdebinfo', 'minsizerel'], + type=str.lower, + help="Set build files configuration", + default='debug') + + args = parser.parse_args() + save_cwd = os.getcwd() + + # Create working "top" directory if needed + distutils.dir_util.mkpath(args.dir) + abs_top_dir = os.path.abspath(args.dir) + + repos = GetGoodRepos(args) + + print('Starting builds in {d}'.format(d=abs_top_dir)) + for repo in repos: + # If the repo has a CI whitelist, skip the repo unless + # one of the CI's environment variable is set to true. + if len(repo.ci_only): + do_build = False + for env in repo.ci_only: + if not env in os.environ: + continue + if os.environ[env].lower() == 'true': + do_build = True + break + if not do_build: + continue + + # Clone/update the repository + repo.Checkout() + + # Build the repository + if args.do_build: + repo.Build(repos) + + # Need to restore original cwd in order for CreateHelper to find json file + os.chdir(save_cwd) + CreateHelper(args, repos, os.path.join(abs_top_dir, 'helper.cmake')) + + sys.exit(0) + + +if __name__ == '__main__': + main() -- 2.7.4