scripts: Update update_deps.py
authorJuan Ramos <juan@lunarg.com>
Mon, 23 Oct 2023 23:41:48 +0000 (17:41 -0600)
committerJuan Ramos <114601453+juan-lunarg@users.noreply.github.com>
Tue, 24 Oct 2023 18:14:56 +0000 (12:14 -0600)
.github/workflows/build.yml
scripts/CMakeLists.txt
scripts/update_deps.py

index 0bb028f5952f21d613f93067965a24745d23d51a..fe5d99af82e9da568f6e0768ab6c933e2b55d19d 100644 (file)
@@ -83,8 +83,12 @@ jobs:
               working-directory: ./build
               run: ctest --output-on-failure
 
-            - name: Verify generated source files
-              run: python scripts/generate_source.py --verify external/${{matrix.config}}/Vulkan-Headers/registry
+    codegen:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v4
+        - run: scripts/update_deps.py --dir ext --no-build
+        - run: scripts/generate_source.py --verify ext/Vulkan-Headers/registry/
 
     linux-no-asm:
         runs-on: ubuntu-22.04
index 4fce83e1b4ef2940901d48df2d7b1f5770e097dc..5943cdb42174882eb109333aae96efb488e0fed2 100644 (file)
@@ -42,9 +42,12 @@ if (UPDATE_DEPS)
     list(APPEND update_dep_command "${_build_type}")
 
     set(UPDATE_DEPS_DIR_SUFFIX "${_build_type}")
-    if (ANDROID)
-        set(UPDATE_DEPS_DIR_SUFFIX "android/${UPDATE_DEPS_DIR_SUFFIX}")
-    endif()
+    if (CMAKE_CROSSCOMPILING)
+        set(UPDATE_DEPS_DIR_SUFFIX "${CMAKE_SYSTEM_NAME}/${UPDATE_DEPS_DIR_SUFFIX}/${CMAKE_SYSTEM_PROCESSOR}")
+    else()
+        math(EXPR bitness "8 * ${CMAKE_SIZEOF_VOID_P}")
+        set(UPDATE_DEPS_DIR_SUFFIX "${UPDATE_DEPS_DIR_SUFFIX}/${bitness}")
+    endif()    
     set(UPDATE_DEPS_DIR "${PROJECT_SOURCE_DIR}/external/${UPDATE_DEPS_DIR_SUFFIX}" CACHE PATH "Location where update_deps.py installs packages")
     list(APPEND update_dep_command "--dir" )
     list(APPEND update_dep_command "${UPDATE_DEPS_DIR}")
@@ -58,6 +61,18 @@ if (UPDATE_DEPS)
     endif()
 
     list(APPEND cmake_vars "CMAKE_TOOLCHAIN_FILE")
+
+    # Avoids manually setting CMAKE_SYSTEM_NAME unless it's needed:
+    # https://cmake.org/cmake/help/latest/variable/CMAKE_CROSSCOMPILING.html
+    if (NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "${CMAKE_HOST_SYSTEM_NAME}")
+        list(APPEND cmake_vars "CMAKE_SYSTEM_NAME")
+    endif()
+    if (APPLE)
+        list(APPEND cmake_vars "CMAKE_OSX_ARCHITECTURES" "CMAKE_OSX_DEPLOYMENT_TARGET")
+    endif()
+    if (NOT MSVC_IDE)
+        list(APPEND cmake_vars "CMAKE_CXX_COMPILER" "CMAKE_C_COMPILER" "CMAKE_ASM_COMPILER")
+    endif()
     if (ANDROID)
         list(APPEND cmake_vars "ANDROID_PLATFORM" "CMAKE_ANDROID_ARCH_ABI" "CMAKE_ANDROID_STL_TYPE" "CMAKE_ANDROID_RTTI" "CMAKE_ANDROID_EXCEPTIONS" "ANDROID_USE_LEGACY_TOOLCHAIN_FILE")
     endif()
index 937c790281364c66e93f242615672fda018965ec..a4935c1637d87d4f237243cdd6ce8c02611fd5ce 100755 (executable)
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2017 The Glslang Authors. All rights reserved.
 # Copyright (c) 2018-2023 Valve Corporation
@@ -141,8 +141,6 @@ to the "top" directory.
 
 The commit used to checkout the repository.  This can be a SHA-1
 object name or a refname used with the remote name "origin".
-For example, this field can be set to "origin/sdk-1.1.77" to
-select the end of the sdk-1.1.77 branch.
 
 - deps (optional)
 
@@ -220,6 +218,7 @@ Legal options include:
 "windows"
 "linux"
 "darwin"
+"android"
 
 Builds on all platforms by default.
 
@@ -235,6 +234,7 @@ option can be a relative or absolute path.
 
 import argparse
 import json
+import os
 import os.path
 import subprocess
 import sys
@@ -254,7 +254,8 @@ CONFIG_MAP = {
     'minsizerel': 'MinSizeRel'
 }
 
-VERBOSE = False
+# NOTE: CMake also uses the VERBOSE environment variable. This is intentional.
+VERBOSE = os.getenv("VERBOSE")
 
 DEVNULL = open(os.devnull, 'wb')
 
@@ -273,25 +274,40 @@ def make_or_exist_dirs(path):
     if not os.path.isdir(path):
         os.makedirs(path)
 
-def command_output(cmd, directory, fail_ok=False):
-    """Runs a command in a directory and returns its standard output stream.
+def command_output(cmd, directory):
+    # Runs a command in a directory and returns its standard output stream.
+    # Captures the standard error stream and prints it an error occurs.
+    # Raises a RuntimeError if the command fails to launch or otherwise fails.
+    if VERBOSE:
+        print('In {d}: {cmd}'.format(d=directory, cmd=cmd))
 
-    Captures the standard error stream and prints it if error.
+    result = subprocess.run(cmd, cwd=directory, capture_output=True, text=True)
+
+    if result.returncode != 0:
+        print(f'{result.stderr}', file=sys.stderr)
+        raise RuntimeError(f'Failed to run {cmd} in {directory}')
 
-    Raises a RuntimeError if the command fails to launch or otherwise fails.
-    """
     if VERBOSE:
-        print('In {d}: {cmd}'.format(d=directory, cmd=cmd))
-    p = subprocess.Popen(
-        cmd, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    (stdout, stderr) = p.communicate()
-    if p.returncode != 0:
-        print('*** Error ***\nstderr contents:\n{}'.format(stderr))
-        if not fail_ok:
-            raise RuntimeError('Failed to run {} in {}'.format(cmd, directory))
+        print(result.stdout)
+    return result.stdout
+
+def run_cmake_command(cmake_cmd):
+    # NOTE: Because CMake is an exectuable that runs executables
+    # stdout/stderr are mixed together. So this combines the outputs
+    # and prints them properly in case there is a non-zero exit code.
+    result = subprocess.run(cmake_cmd, 
+        stdout = subprocess.PIPE,
+        stderr = subprocess.STDOUT,
+        text = True
+    )
+
     if VERBOSE:
-        print(stdout)
-    return stdout
+        print(result.stdout)
+        print(f"CMake command: {cmake_cmd} ", flush=True)
+
+    if result.returncode != 0:
+        print(result.stdout, file=sys.stderr)
+        sys.exit(result.returncode)
 
 def escape(path):
     return path.replace('\\', '/')
@@ -341,13 +357,21 @@ class GoodRepo(object):
             self.build_dir = os.path.join(dir_top, self.build_dir)
         if self.install_dir:
             self.install_dir = os.path.join(dir_top, self.install_dir)
-           # Check if platform is one to build on
+
+        # By default the target platform is the host platform.
+        target_platform = platform.system().lower()
+        # However, we need to account for cross-compiling.
+        for cmake_var in self._args.cmake_var:
+            if "android.toolchain.cmake" in cmake_var:
+                target_platform = 'android'
+
         self.on_build_platform = False
-        if self.build_platforms == [] or platform.system().lower() in self.build_platforms:
+        if self.build_platforms == [] or target_platform in self.build_platforms:
             self.on_build_platform = True
 
     def Clone(self, retries=10, retry_seconds=60):
-        print('Cloning {n} into {d}'.format(n=self.name, d=self.repo_dir))
+        if VERBOSE:
+            print('Cloning {n} into {d}'.format(n=self.name, d=self.repo_dir))
         for retry in range(retries):
             make_or_exist_dirs(self.repo_dir)
             try:
@@ -388,7 +412,9 @@ class GoodRepo(object):
                 raise e
 
     def Checkout(self):
-        print('Checking out {n} in {d}'.format(n=self.name, d=self.repo_dir))
+        if VERBOSE:
+            print('Checking out {n} in {d}'.format(n=self.name, d=self.repo_dir))
+
         if self._args.do_clean_repo:
             if os.path.isdir(self.repo_dir):
                 shutil.rmtree(self.repo_dir, onerror = on_rm_error)
@@ -399,7 +425,9 @@ class GoodRepo(object):
             command_output(['git', 'checkout', self._args.ref], self.repo_dir)
         else:
             command_output(['git', 'checkout', self.commit], self.repo_dir)
-        print(command_output(['git', 'status'], self.repo_dir))
+
+        if VERBOSE:
+            print(command_output(['git', 'status'], self.repo_dir))
 
     def CustomPreProcess(self, cmd_str, repo_dict):
         return cmd_str.format(repo_dict, self._args, CONFIG_MAP[self._args.config])
@@ -477,12 +505,12 @@ class GoodRepo(object):
         if self._args.generator is not None:
             cmake_cmd.extend(['-G', self._args.generator])
 
-        if VERBOSE:
-            print("CMake command: " + " ".join(cmake_cmd))
+        # Removes warnings related to unused CLI
+        # EX: Setting CMAKE_CXX_COMPILER for a C project
+        if not VERBOSE:
+            cmake_cmd.append("--no-warn-unused-cli")
 
-        ret_code = subprocess.call(cmake_cmd)
-        if ret_code != 0:
-            sys.exit(ret_code)
+        run_cmake_command(cmake_cmd)
 
     def CMakeBuild(self):
         """Build CMake command for the build phase and execute it"""
@@ -490,23 +518,21 @@ class GoodRepo(object):
         if self._args.do_clean:
             cmake_cmd.append('--clean-first')
 
-        # Ninja is parallel by default
-        if self._args.generator != "Ninja":
+        # Xcode / Ninja are parallel by default.
+        if self._args.generator != "Ninja" or self._args.generator != "Xcode":
             cmake_cmd.append('--parallel')
             cmake_cmd.append(format(multiprocessing.cpu_count()))
 
-        if VERBOSE:
-            print("CMake command: " + " ".join(cmake_cmd))
-
-        ret_code = subprocess.call(cmake_cmd)
-        if ret_code != 0:
-            sys.exit(ret_code)
+        run_cmake_command(cmake_cmd)
 
     def Build(self, repos, repo_dict):
-        """Build the dependent repo"""
-        print('Building {n} in {d}'.format(n=self.name, d=self.repo_dir))
-        print('Build dir = {b}'.format(b=self.build_dir))
-        print('Install dir = {i}\n'.format(i=self.install_dir))
+        """Build the dependent repo and time how long it took"""
+        if VERBOSE:
+            print('Building {n} in {d}'.format(n=self.name, d=self.repo_dir))
+            print('Build dir = {b}'.format(b=self.build_dir))
+            print('Install dir = {i}\n'.format(i=self.install_dir))
+
+        start = time.time()
 
         # Run any prebuild commands
         self.PreBuild()
@@ -521,9 +547,12 @@ class GoodRepo(object):
         # Build and execute CMake command for the build
         self.CMakeBuild()
 
+        total_time = time.time() - start
+
+        print(f"Installed {self.name} ({self.commit}) in {total_time} seconds", flush=True)
+
     def IsOptional(self, opts):
-        if len(self.optional.intersection(opts)) > 0: return True
-        else: return False
+        return len(self.optional.intersection(opts)) > 0
 
 def GetGoodRepos(args):
     """Returns the latest list of GoodRepo objects.
@@ -741,7 +770,7 @@ def main():
         if len(repo.ci_only):
             do_build = False
             for env in repo.ci_only:
-                if not env in os.environ:
+                if env not in os.environ:
                     continue
                 if os.environ[env].lower() == 'true':
                     do_build = True
@@ -765,3 +794,4 @@ def main():
 
 if __name__ == '__main__':
     main()
+