ci/gitlab/build_manifest.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import os
   5 import sys
   6 import subprocess
   7 import urllib.error
   8 import urllib.parse
   9 import urllib.request
  10 import json
  11
  12 from typing import Dict, Tuple, List
  13 # from pprint import pprint
  14
  15 if sys.version_info < (3, 6):
  16     raise SystemExit('Need Python 3.6 or newer')
  17
  18 GSTREAMER_MODULES: List[str] = [
  19     'orc',
  20     'cerbero',
  21     'gst-build',
  22     'gstreamer',
  23     'gst-plugins-base',
  24     'gst-plugins-good',
  25     'gst-plugins-bad',
  26     'gst-plugins-ugly',
  27     'gst-libav',
  28     'gst-devtools',
  29     'gst-docs',
  30     'gst-editing-services',
  31     'gst-omx',
  32     'gst-python',
  33     'gst-rtsp-server',
  34     'gstreamer-sharp',
  35     'gstreamer-vaapi',
  36     'gst-integration-testsuites',
  37     'gst-examples',
  38 ]
  39
  40 MANIFEST_TEMPLATE: str = """<?xml version="1.0" encoding="UTF-8"?>
  41 <manifest>
  42   <remote fetch="{}" name="user"/>
  43   <remote fetch="https://gitlab.freedesktop.org/gstreamer/" name="origin"/>
  44 {}
  45 </manifest>"""
  46
  47
  48 CERBERO_DEPS_LOGS_TARGETS = (
  49     ('cross-ios', 'universal'),
  50     ('cross-windows-mingw', 'x86'),
  51     ('cross-windows-mingw', 'x86_64'),
  52     ('cross-android', 'universal'),
  53     ('fedora', 'x86_64'),
  54     ('macos', 'x86_64'),
  55     ('windows-msvc', 'x86_64'),
  56 )
  57
  58 # Disallow git prompting for a username/password
  59 os.environ['GIT_TERMINAL_PROMPT'] = '0'
  60 def git(*args, repository_path='.'):
  61     return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
  62
  63 def get_cerbero_last_build_info (branch : str):
  64     # Fetch the deps log for all (distro, arch) targets
  65     all_commits = {}
  66     for distro, arch in CERBERO_DEPS_LOGS_TARGETS:
  67         url = f'https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/{distro}/{arch}/cerbero-deps.log'
  68         print(f'Fetching {url}')
  69         try:
  70             req = urllib.request.Request(url)
  71             resp = urllib.request.urlopen(req);
  72             deps = json.loads(resp.read())
  73         except urllib.error.URLError as e:
  74             print(f'WARNING: Failed to GET {url}: {e!s}')
  75             continue
  76
  77         for dep in deps:
  78             commit = dep['commit']
  79             if commit not in all_commits:
  80                 all_commits[commit] = []
  81             all_commits[commit].append((distro, arch))
  82
  83     # Fetch the cerbero commit that has the most number of caches
  84     best_commit = None
  85     newest_commit = None
  86     max_caches = 0
  87     total_caches = len(CERBERO_DEPS_LOGS_TARGETS)
  88     for commit, targets in all_commits.items():
  89         if newest_commit is None:
  90             newest_commit = commit
  91         have_caches = len(targets)
  92         # If this commit has caches for all targets, just use it
  93         if have_caches == total_caches:
  94             best_commit = commit
  95             break
  96         # Else, try to find the commit with the most caches
  97         if have_caches > max_caches:
  98             max_caches = have_caches
  99             best_commit = commit
 100     if newest_commit is None:
 101         print('WARNING: No deps logs were found, will build from scratch')
 102     if best_commit != newest_commit:
 103         print(f'WARNING: Cache is not up-to-date for commit {newest_commit}, using commit {best_commit} instead')
 104     return best_commit
 105
 106
 107 def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
 108     try:
 109         res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
 110     except subprocess.CalledProcessError:
 111         return None, None
 112
 113     if not res:
 114         return None, None
 115
 116     # Special case cerbero to avoid cache misses
 117     if module == 'cerbero':
 118         sha = get_cerbero_last_build_info(branch)
 119         if sha is not None:
 120             return sha, sha
 121
 122     lines = res.split('\n')
 123     for line in lines:
 124         if line.endswith('/' + branch):
 125             try:
 126                 sha, refname = line.split('\t')
 127             except ValueError:
 128                 continue
 129             return refname.strip(), sha
 130
 131     return None, None
 132
 133
 134 def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
 135     namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
 136     ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
 137
 138     if module == "orc":
 139         ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
 140
 141     if module == os.environ['CI_PROJECT_NAME']:
 142         return 'user', branchname, os.environ['CI_COMMIT_SHA']
 143
 144     if branchname != ups_branch:
 145         remote_refname, sha = get_branch_info(module, namespace, branchname)
 146         if sha is not None:
 147             return 'user', remote_refname, sha
 148
 149     # Check upstream project for a branch
 150     remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
 151     if sha is not None:
 152         return 'origin', remote_refname, sha
 153
 154     # This should never occur given the upstream fallback above
 155     print(f"Could not find anything for {module}:{branchname}")
 156     print("If something reaches that point, please file a bug")
 157     print("https://gitlab.freedesktop.org/gstreamer/gst-ci/issues")
 158     assert False
 159
 160
 161 # --- Unit tests --- #
 162 # Basically, pytest will happily let a test mutate a variable, and then run
 163 # the next tests one the same environment without reset the vars.
 164 def preserve_ci_vars(func):
 165     """Preserve the original CI Variable values"""
 166     def wrapper():
 167         try:
 168             url = os.environ["CI_PROJECT_URL"]
 169             user = os.environ["CI_PROJECT_NAMESPACE"]
 170         except KeyError:
 171             url = "invalid"
 172             user = ""
 173
 174         private = os.getenv("READ_PROJECTS_TOKEN", default=None)
 175         if not private:
 176             os.environ["READ_PROJECTS_TOKEN"] = "FOO"
 177
 178         func()
 179
 180         os.environ["CI_PROJECT_URL"] = url
 181         os.environ["CI_PROJECT_NAMESPACE"] = user
 182
 183         if private:
 184             os.environ["READ_PROJECTS_TOKEN"] = private
 185         # if it was set after
 186         elif os.getenv("READ_PROJECTS_TOKEN", default=None):
 187             del os.environ["READ_PROJECTS_TOKEN"]
 188
 189     return wrapper
 190
 191 @preserve_ci_vars
 192 def test_find_repository_sha():
 193     os.environ["CI_PROJECT_NAME"] = "some-random-project"
 194     os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
 195     os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
 196     os.environ["GST_UPSTREAM_BRANCH"] = "master"
 197     del os.environ["READ_PROJECTS_TOKEN"]
 198
 199     # This should find the repository in the user namespace
 200     remote, refname, git_ref = find_repository_sha("gst-plugins-good", "1.2")
 201     assert remote == "user"
 202     assert git_ref == "08ab260b8a39791e7e62c95f4b64fd5b69959325"
 203     assert refname == "refs/heads/1.2"
 204
 205     # This should fallback to upstream master branch since no matching branch was found
 206     remote, refname, git_ref = find_repository_sha("gst-plugins-good", "totally-valid-branch-name")
 207     assert remote == "origin"
 208     assert refname == "refs/heads/master"
 209
 210     os.environ["CI_PROJECT_NAME"] = "the_project"
 211     os.environ["CI_COMMIT_SHA"] = "MySha"
 212
 213     remote, refname, git_ref = find_repository_sha("the_project", "whatever")
 214     assert remote == "user"
 215     assert git_ref == "MySha"
 216     assert refname == "whatever"
 217
 218
 219 @preserve_ci_vars
 220 def test_get_project_branch():
 221     os.environ["CI_PROJECT_NAME"] = "some-random-project"
 222     os.environ["CI_COMMIT_SHA"] = "dwbuiw"
 223     os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
 224     os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
 225     del os.environ["READ_PROJECTS_TOKEN"]
 226
 227     os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
 228     remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
 229     assert twelve is not None
 230     assert remote == 'origin'
 231     assert refname == "refs/heads/1.12"
 232
 233     os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
 234     remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
 235     assert fourteen is not None
 236     assert remote == 'origin'
 237     assert refname == "refs/heads/1.14"
 238
 239
 240 if __name__ == "__main__":
 241     parser = argparse.ArgumentParser()
 242     parser.add_argument("--self-update", action="store_true", default=False)
 243     parser.add_argument(dest="output", default='manifest.xml', nargs='?')
 244     options = parser.parse_args()
 245
 246     current_branch: str = os.environ['CI_COMMIT_REF_NAME']
 247     user_remote_url: str = os.path.dirname(os.environ['CI_PROJECT_URL'])
 248     if not user_remote_url.endswith('/'):
 249         user_remote_url += '/'
 250
 251     if options.self_update:
 252         remote, remote_refname, sha = find_repository_sha("gst-ci", current_branch)
 253         if remote == 'user':
 254             remote = user_remote_url + 'gst-ci'
 255         else:
 256             remote = "https://gitlab.freedesktop.org/gstreamer/gst-ci"
 257
 258         git('fetch', remote, remote_refname)
 259         git('checkout', '--detach', sha)
 260         sys.exit(0)
 261
 262     projects: str = ''
 263     for module in GSTREAMER_MODULES:
 264         print(f"Checking {module}:", end=' ')
 265         remote, refname, revision = find_repository_sha(module, current_branch)
 266         print(f"remote '{remote}', refname: '{refname}', revision: '{revision}'")
 267         projects += f"  <project path=\"{module}\" name=\"{module}.git\" remote=\"{remote}\" revision=\"{revision}\" refname=\"{refname}\" />\n"
 268
 269     with open(options.output, mode='w') as manifest:
 270         print(MANIFEST_TEMPLATE.format(user_remote_url, projects), file=manifest)