tools/run_tests/artifacts/artifact_targets.py

   1 #!/usr/bin/env python
   2 # Copyright 2016 gRPC authors.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Definition of targets to build artifacts."""
  16
  17 import os.path
  18 import random
  19 import string
  20 import sys
  21
  22 sys.path.insert(0, os.path.abspath('..'))
  23 import python_utils.jobset as jobset
  24
  25
  26 def create_docker_jobspec(name,
  27                           dockerfile_dir,
  28                           shell_command,
  29                           environ={},
  30                           flake_retries=0,
  31                           timeout_retries=0,
  32                           timeout_seconds=30 * 60,
  33                           extra_docker_args=None,
  34                           verbose_success=False):
  35     """Creates jobspec for a task running under docker."""
  36     environ = environ.copy()
  37     environ['RUN_COMMAND'] = shell_command
  38     environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
  39
  40     docker_args = []
  41     for k, v in environ.items():
  42         docker_args += ['-e', '%s=%s' % (k, v)]
  43     docker_env = {
  44         'DOCKERFILE_DIR': dockerfile_dir,
  45         'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
  46         'OUTPUT_DIR': 'artifacts'
  47     }
  48     if extra_docker_args is not None:
  49         docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
  50     jobspec = jobset.JobSpec(
  51         cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
  52         docker_args,
  53         environ=docker_env,
  54         shortname='build_artifact.%s' % (name),
  55         timeout_seconds=timeout_seconds,
  56         flake_retries=flake_retries,
  57         timeout_retries=timeout_retries,
  58         verbose_success=verbose_success)
  59     return jobspec
  60
  61
  62 def create_jobspec(name,
  63                    cmdline,
  64                    environ={},
  65                    shell=False,
  66                    flake_retries=0,
  67                    timeout_retries=0,
  68                    timeout_seconds=30 * 60,
  69                    use_workspace=False,
  70                    cpu_cost=1.0,
  71                    verbose_success=False):
  72     """Creates jobspec."""
  73     environ = environ.copy()
  74     if use_workspace:
  75         environ['WORKSPACE_NAME'] = 'workspace_%s' % name
  76         environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
  77         cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
  78                   ] + cmdline
  79     else:
  80         environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
  81
  82     jobspec = jobset.JobSpec(cmdline=cmdline,
  83                              environ=environ,
  84                              shortname='build_artifact.%s' % (name),
  85                              timeout_seconds=timeout_seconds,
  86                              flake_retries=flake_retries,
  87                              timeout_retries=timeout_retries,
  88                              shell=shell,
  89                              cpu_cost=cpu_cost,
  90                              verbose_success=verbose_success)
  91     return jobspec
  92
  93
  94 _MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10'
  95
  96 _ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}
  97
  98
  99 class PythonArtifact:
 100     """Builds Python artifacts."""
 101
 102     def __init__(self, platform, arch, py_version):
 103         self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
 104         self.platform = platform
 105         self.arch = arch
 106         self.labels = ['artifact', 'python', platform, arch, py_version]
 107         self.py_version = py_version
 108         if 'manylinux' in platform:
 109             self.labels.append('linux')
 110
 111     def pre_build_jobspecs(self):
 112         return []
 113
 114     def build_jobspec(self):
 115         environ = {}
 116         if self.platform == 'linux_extra':
 117             # Crosscompilation build for armv7 (e.g. Raspberry Pi)
 118             environ['PYTHON'] = '/opt/python/{}/bin/python3'.format(
 119                 self.py_version)
 120             environ['PIP'] = '/opt/python/{}/bin/pip3'.format(self.py_version)
 121             environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
 122             environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
 123             # when crosscompiling, we need to force statically linking libstdc++
 124             # otherwise libstdc++ symbols would be too new and the resulting
 125             # wheel wouldn't pass the auditwheel check.
 126             # This is needed because C core won't build with GCC 4.8 that's
 127             # included in the default dockcross toolchain and we needed
 128             # to opt into using a slighly newer version of GCC.
 129             environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE'
 130
 131             return create_docker_jobspec(
 132                 self.name,
 133                 'tools/dockerfile/grpc_artifact_python_linux_{}'.format(
 134                     self.arch),
 135                 'tools/run_tests/artifacts/build_artifact_python.sh',
 136                 environ=environ,
 137                 timeout_seconds=60 * 60)
 138         elif 'manylinux' in self.platform:
 139             if self.arch == 'x86':
 140                 environ['SETARCH_CMD'] = 'linux32'
 141             # Inside the manylinux container, the python installations are located in
 142             # special places...
 143             environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
 144                 self.py_version)
 145             environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
 146             environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
 147             if self.arch == 'aarch64':
 148                 environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
 149                 # when crosscompiling, we need to force statically linking libstdc++
 150                 # otherwise libstdc++ symbols would be too new and the resulting
 151                 # wheel wouldn't pass the auditwheel check.
 152                 # This is needed because C core won't build with GCC 4.8 that's
 153                 # included in the default dockcross toolchain and we needed
 154                 # to opt into using a slighly newer version of GCC.
 155                 environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE'
 156
 157             else:
 158                 # only run auditwheel if we're not crosscompiling
 159                 environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE'
 160                 # only build the packages that depend on grpcio-tools
 161                 # if we're not crosscompiling.
 162                 # - they require protoc to run on current architecture
 163                 # - they only have sdist packages anyway, so it's useless to build them again
 164                 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
 165             return create_docker_jobspec(
 166                 self.name,
 167                 'tools/dockerfile/grpc_artifact_python_%s_%s' %
 168                 (self.platform, self.arch),
 169                 'tools/run_tests/artifacts/build_artifact_python.sh',
 170                 environ=environ,
 171                 timeout_seconds=60 * 60 * 2)
 172         elif self.platform == 'windows':
 173             if 'Python27' in self.py_version:
 174                 environ['EXT_COMPILER'] = 'mingw32'
 175             else:
 176                 environ['EXT_COMPILER'] = 'msvc'
 177             # For some reason, the batch script %random% always runs with the same
 178             # seed.  We create a random temp-dir here
 179             dir = ''.join(
 180                 random.choice(string.ascii_uppercase) for _ in range(10))
 181             return create_jobspec(self.name, [
 182                 'tools\\run_tests\\artifacts\\build_artifact_python.bat',
 183                 self.py_version, '32' if self.arch == 'x86' else '64'
 184             ],
 185                                   environ=environ,
 186                                   timeout_seconds=45 * 60,
 187                                   use_workspace=True)
 188         else:
 189             environ['PYTHON'] = self.py_version
 190             environ['SKIP_PIP_INSTALL'] = 'TRUE'
 191             return create_jobspec(
 192                 self.name,
 193                 ['tools/run_tests/artifacts/build_artifact_python.sh'],
 194                 environ=environ,
 195                 timeout_seconds=60 * 60 * 2,
 196                 use_workspace=True)
 197
 198     def __str__(self):
 199         return self.name
 200
 201
 202 class RubyArtifact:
 203     """Builds ruby native gem."""
 204
 205     def __init__(self, platform, arch):
 206         self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
 207         self.platform = platform
 208         self.arch = arch
 209         self.labels = ['artifact', 'ruby', platform, arch]
 210
 211     def pre_build_jobspecs(self):
 212         return []
 213
 214     def build_jobspec(self):
 215         # Ruby build uses docker internally and docker cannot be nested.
 216         # We are using a custom workspace instead.
 217         return create_jobspec(
 218             self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
 219             use_workspace=True,
 220             timeout_seconds=60 * 60)
 221
 222
 223 class CSharpExtArtifact:
 224     """Builds C# native extension library"""
 225
 226     def __init__(self, platform, arch, arch_abi=None):
 227         self.name = 'csharp_ext_%s_%s' % (platform, arch)
 228         self.platform = platform
 229         self.arch = arch
 230         self.arch_abi = arch_abi
 231         self.labels = ['artifact', 'csharp', platform, arch]
 232         if arch_abi:
 233             self.name += '_%s' % arch_abi
 234             self.labels.append(arch_abi)
 235
 236     def pre_build_jobspecs(self):
 237         return []
 238
 239     def build_jobspec(self):
 240         if self.arch == 'android':
 241             return create_docker_jobspec(
 242                 self.name,
 243                 'tools/dockerfile/grpc_artifact_android_ndk',
 244                 'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
 245                 environ={'ANDROID_ABI': self.arch_abi})
 246         elif self.arch == 'ios':
 247             return create_jobspec(
 248                 self.name,
 249                 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
 250                 timeout_seconds=60 * 60,
 251                 use_workspace=True)
 252         elif self.platform == 'windows':
 253             return create_jobspec(self.name, [
 254                 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
 255                 self.arch
 256             ],
 257                                   use_workspace=True)
 258         else:
 259             if self.platform == 'linux':
 260                 dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
 261                     self.arch)
 262                 if self.arch == 'aarch64':
 263                     # for aarch64, use a dockcross manylinux image that will
 264                     # give us both ready to use crosscompiler and sufficient backward compatibility
 265                     dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64'
 266                 return create_docker_jobspec(
 267                     self.name, dockerfile_dir,
 268                     'tools/run_tests/artifacts/build_artifact_csharp.sh')
 269             else:
 270                 return create_jobspec(
 271                     self.name,
 272                     ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
 273                     timeout_seconds=45 * 60,
 274                     use_workspace=True)
 275
 276     def __str__(self):
 277         return self.name
 278
 279
 280 class PHPArtifact:
 281     """Builds PHP PECL package"""
 282
 283     def __init__(self, platform, arch):
 284         self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
 285         self.platform = platform
 286         self.arch = arch
 287         self.labels = ['artifact', 'php', platform, arch]
 288
 289     def pre_build_jobspecs(self):
 290         return []
 291
 292     def build_jobspec(self):
 293         return create_docker_jobspec(
 294             self.name,
 295             'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch),
 296             'tools/run_tests/artifacts/build_artifact_php.sh')
 297
 298
 299 class ProtocArtifact:
 300     """Builds protoc and protoc-plugin artifacts"""
 301
 302     def __init__(self, platform, arch):
 303         self.name = 'protoc_%s_%s' % (platform, arch)
 304         self.platform = platform
 305         self.arch = arch
 306         self.labels = ['artifact', 'protoc', platform, arch]
 307
 308     def pre_build_jobspecs(self):
 309         return []
 310
 311     def build_jobspec(self):
 312         if self.platform != 'windows':
 313             environ = {'CXXFLAGS': '', 'LDFLAGS': ''}
 314             if self.platform == 'linux':
 315                 dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
 316                     self.arch)
 317                 if self.arch == 'aarch64':
 318                     # for aarch64, use a dockcross manylinux image that will
 319                     # give us both ready to use crosscompiler and sufficient backward compatibility
 320                     dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64'
 321                 environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s'
 322                 return create_docker_jobspec(
 323                     self.name,
 324                     dockerfile_dir,
 325                     'tools/run_tests/artifacts/build_artifact_protoc.sh',
 326                     environ=environ)
 327             else:
 328                 environ[
 329                     'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
 330                 return create_jobspec(
 331                     self.name,
 332                     ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
 333                     environ=environ,
 334                     timeout_seconds=60 * 60,
 335                     use_workspace=True)
 336         else:
 337             generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015'
 338             return create_jobspec(
 339                 self.name,
 340                 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
 341                 environ={'generator': generator},
 342                 use_workspace=True)
 343
 344     def __str__(self):
 345         return self.name
 346
 347
 348 def targets():
 349     """Gets list of supported targets"""
 350     return [
 351         ProtocArtifact('linux', 'x64'),
 352         ProtocArtifact('linux', 'x86'),
 353         ProtocArtifact('linux', 'aarch64'),
 354         ProtocArtifact('macos', 'x64'),
 355         ProtocArtifact('windows', 'x64'),
 356         ProtocArtifact('windows', 'x86'),
 357         CSharpExtArtifact('linux', 'x64'),
 358         CSharpExtArtifact('linux', 'aarch64'),
 359         CSharpExtArtifact('macos', 'x64'),
 360         CSharpExtArtifact('windows', 'x64'),
 361         CSharpExtArtifact('windows', 'x86'),
 362         CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'),
 363         CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'),
 364         CSharpExtArtifact('linux', 'android', arch_abi='x86'),
 365         CSharpExtArtifact('macos', 'ios'),
 366         PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'),
 367         PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'),
 368         PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'),
 369         PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'),
 370         PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),
 371         PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'),
 372         PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'),
 373         PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'),
 374         PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'),
 375         PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'),
 376         PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'),
 377         PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'),
 378         PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'),
 379         PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'),
 380         PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'),
 381         PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'),
 382         PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'),
 383         PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'),
 384         PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'),
 385         PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'),
 386         PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'),
 387         PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'),
 388         PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'),
 389         PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'),
 390         PythonArtifact('manylinux2014', 'aarch64', 'cp36-cp36m'),
 391         PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m'),
 392         PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38'),
 393         PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'),
 394         PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m'),
 395         PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m'),
 396         PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'),
 397         PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'),
 398         PythonArtifact('macos', 'x64', 'python2.7'),
 399         PythonArtifact('macos', 'x64', 'python3.5'),
 400         PythonArtifact('macos', 'x64', 'python3.6'),
 401         PythonArtifact('macos', 'x64', 'python3.7'),
 402         PythonArtifact('macos', 'x64', 'python3.8'),
 403         PythonArtifact('macos', 'x64', 'python3.9'),
 404         PythonArtifact('windows', 'x86', 'Python27_32bit'),
 405         PythonArtifact('windows', 'x86', 'Python35_32bit'),
 406         PythonArtifact('windows', 'x86', 'Python36_32bit'),
 407         PythonArtifact('windows', 'x86', 'Python37_32bit'),
 408         PythonArtifact('windows', 'x86', 'Python38_32bit'),
 409         PythonArtifact('windows', 'x86', 'Python39_32bit'),
 410         PythonArtifact('windows', 'x64', 'Python27'),
 411         PythonArtifact('windows', 'x64', 'Python35'),
 412         PythonArtifact('windows', 'x64', 'Python36'),
 413         PythonArtifact('windows', 'x64', 'Python37'),
 414         PythonArtifact('windows', 'x64', 'Python38'),
 415         PythonArtifact('windows', 'x64', 'Python39'),
 416         RubyArtifact('linux', 'x64'),
 417         RubyArtifact('macos', 'x64'),
 418         PHPArtifact('linux', 'x64')
 419     ]