third_party/pigweed/repo/pw_env_setup/py/pw_env_setup/cipd_setup/update.py

   1 #!/usr/bin/env python
   2 # Copyright 2020 The Pigweed Authors
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5 # use this file except in compliance with the License. You may obtain a copy of
   6 # the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 # License for the specific language governing permissions and limitations under
  14 # the License.
  15 """Installs or updates prebuilt tools.
  16
  17 Must be tested with Python 2 and Python 3.
  18
  19 The stdout of this script is meant to be executed by the invoking shell.
  20 """
  21
  22 from __future__ import print_function
  23
  24 import argparse
  25 import json
  26 import os
  27 import re
  28 import shutil
  29 import subprocess
  30 import sys
  31 import tempfile
  32
  33
  34 def parse(argv=None):
  35     """Parse arguments."""
  36
  37     script_root = os.path.join(os.environ['PW_ROOT'], 'pw_env_setup', 'py',
  38                                'pw_env_setup', 'cipd_setup')
  39     git_root = subprocess.check_output(
  40         ('git', 'rev-parse', '--show-toplevel'),
  41         cwd=script_root,
  42     ).decode('utf-8').strip()
  43
  44     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
  45     parser.add_argument(
  46         '--install-dir',
  47         dest='root_install_dir',
  48         default=os.path.join(git_root, '.cipd'),
  49     )
  50     parser.add_argument('--package-file',
  51                         dest='package_files',
  52                         metavar='PACKAGE_FILE',
  53                         action='append')
  54     parser.add_argument('--cipd',
  55                         default=os.path.join(script_root, 'wrapper.py'))
  56     parser.add_argument('--cache-dir',
  57                         default=os.environ.get(
  58                             'CIPD_CACHE_DIR',
  59                             os.path.expanduser('~/.cipd-cache-dir')))
  60
  61     return parser.parse_args(argv)
  62
  63
  64 def check_auth(cipd, package_files):
  65     """Check have access to CIPD pigweed directory."""
  66
  67     paths = []
  68     for package_file in package_files:
  69         with open(package_file, 'r') as ins:
  70             # This is an expensive RPC, so only check the first few entries
  71             # in each file.
  72             for i, entry in enumerate(json.load(ins)):
  73                 if i >= 3:
  74                     break
  75                 parts = entry['path'].split('/')
  76                 while '${' in parts[-1]:
  77                     parts.pop(-1)
  78                 paths.append('/'.join(parts))
  79
  80     try:
  81         output = subprocess.check_output([cipd, 'auth-info'],
  82                                          stderr=subprocess.STDOUT).decode()
  83         logged_in = True
  84
  85         username = None
  86         match = re.search(r'Logged in as (\S*)\.', output)
  87         if match:
  88             username = match.group(1)
  89
  90     except subprocess.CalledProcessError:
  91         logged_in = False
  92
  93     for path in paths:
  94         # Not catching CalledProcessError because 'cipd ls' seems to never
  95         # return an error code unless it can't reach the CIPD server.
  96         output = subprocess.check_output([cipd, 'ls', path],
  97                                          stderr=subprocess.STDOUT).decode()
  98         if 'No matching packages' not in output:
  99             continue
 100
 101         # 'cipd ls' only lists sub-packages but ignores any packages at the
 102         # given path. 'cipd instances' will give versions of that package.
 103         # 'cipd instances' does use an error code if there's no such package or
 104         # that package is inaccessible.
 105         try:
 106             subprocess.check_output([cipd, 'instances', path],
 107                                     stderr=subprocess.STDOUT)
 108         except subprocess.CalledProcessError:
 109             stderr = lambda *args: print(*args, file=sys.stderr)
 110             stderr()
 111             stderr('=' * 60)
 112             stderr('ERROR: no access to CIPD path "{}"'.format(path))
 113             if logged_in:
 114                 username_part = ''
 115                 if username:
 116                     username_part = '({}) '.format(username)
 117                 stderr('Your account {}does not have access to this '
 118                        'path'.format(username_part))
 119             else:
 120                 stderr('Try logging in with this command:')
 121                 stderr()
 122                 stderr('    {} auth-login'.format(cipd))
 123             stderr('=' * 60)
 124             return False
 125
 126     return True
 127
 128
 129 def write_ensure_file(package_file, ensure_file):
 130     with open(package_file, 'r') as ins:
 131         data = json.load(ins)
 132
 133     # TODO(pwbug/103) Remove 30 days after bug fixed.
 134     if os.path.isdir(ensure_file):
 135         shutil.rmtree(ensure_file)
 136
 137     with open(ensure_file, 'w') as outs:
 138         outs.write('$VerifiedPlatform linux-amd64\n'
 139                    '$VerifiedPlatform mac-amd64\n'
 140                    '$ParanoidMode CheckPresence\n')
 141
 142         for entry in data:
 143             outs.write('@Subdir {}\n'.format(entry.get('subdir', '')))
 144             outs.write('{} {}\n'.format(entry['path'],
 145                                         ' '.join(entry['tags'])))
 146
 147
 148 def update(
 149     cipd,
 150     package_files,
 151     root_install_dir,
 152     cache_dir,
 153     env_vars=None,
 154 ):
 155     """Grab the tools listed in ensure_files."""
 156
 157     if not check_auth(cipd, package_files):
 158         return False
 159
 160     # TODO(mohrr) use os.makedirs(..., exist_ok=True).
 161     if not os.path.isdir(root_install_dir):
 162         os.makedirs(root_install_dir)
 163
 164     if env_vars:
 165         env_vars.prepend('PATH', root_install_dir)
 166         env_vars.set('PW_CIPD_INSTALL_DIR', root_install_dir)
 167         env_vars.set('CIPD_CACHE_DIR', cache_dir)
 168
 169     pw_root = None
 170     if env_vars:
 171         pw_root = env_vars.get('PW_ROOT', None)
 172     if not pw_root:
 173         pw_root = os.environ['PW_ROOT']
 174
 175     # Run cipd for each json file.
 176     for package_file in package_files:
 177         if os.path.splitext(package_file)[1] == '.ensure':
 178             ensure_file = package_file
 179         else:
 180             ensure_file = os.path.join(
 181                 root_install_dir,
 182                 os.path.basename(
 183                     os.path.splitext(package_file)[0] + '.ensure'))
 184             write_ensure_file(package_file, ensure_file)
 185
 186         install_dir = os.path.join(
 187             root_install_dir,
 188             os.path.basename(os.path.splitext(package_file)[0]))
 189
 190         cmd = [
 191             cipd,
 192             'ensure',
 193             '-ensure-file', ensure_file,
 194             '-root', install_dir,
 195             '-log-level', 'warning',
 196             '-cache-dir', cache_dir,
 197             '-max-threads', '0',  # 0 means use CPU count.
 198         ]  # yapf: disable
 199
 200         # TODO(pwbug/135) Use function from common utility module.
 201         with tempfile.TemporaryFile(mode='w+') as temp:
 202             print(*cmd, file=temp)
 203             try:
 204                 subprocess.check_call(cmd,
 205                                       stdout=temp,
 206                                       stderr=subprocess.STDOUT)
 207             except subprocess.CalledProcessError:
 208                 temp.seek(0)
 209                 sys.stderr.write(temp.read())
 210                 raise
 211
 212         # Set environment variables so tools can later find things under, for
 213         # example, 'share'.
 214         name = os.path.basename(install_dir)
 215
 216         if env_vars:
 217             # Some executables get installed at top-level and some get
 218             # installed under 'bin'.
 219             env_vars.prepend('PATH', install_dir)
 220             env_vars.prepend('PATH', os.path.join(install_dir, 'bin'))
 221             env_vars.set('PW_{}_CIPD_INSTALL_DIR'.format(name.upper()),
 222                          install_dir)
 223
 224             # Windows has its own special toolchain.
 225             if os.name == 'nt':
 226                 env_vars.prepend('PATH',
 227                                  os.path.join(install_dir, 'mingw64', 'bin'))
 228
 229     return True
 230
 231
 232 if __name__ == '__main__':
 233     update(**vars(parse()))
 234     sys.exit(0)