from recipe_engine import recipe_api
-import os
BOTO_CHROMIUM_SKIA_GM = 'chromium-skia-gm.boto'
CONFIG_RELEASE = 'Release'
-def device_cfg(builder_dict):
- # Android.
- if 'Android' in builder_dict.get('extra_config', ''):
- if 'NoNeon' in builder_dict['extra_config']:
- return 'arm_v7'
- return {
- 'Arm64': 'arm64',
- 'x86': 'x86',
- 'x86_64': 'x86_64',
- 'Mips': 'mips',
- 'Mips64': 'mips64',
- 'MipsDSP2': 'mips_dsp2',
- }.get(builder_dict['target_arch'], 'arm_v7_neon')
- elif builder_dict.get('os') == 'Android':
- return {
- 'AndroidOne': 'arm_v7_neon',
- 'GalaxyS3': 'arm_v7_neon',
- 'GalaxyS4': 'arm_v7_neon',
- 'NVIDIA_Shield': 'arm64',
- 'Nexus10': 'arm_v7_neon',
- 'Nexus5': 'arm_v7_neon',
- 'Nexus6': 'arm_v7_neon',
- 'Nexus7': 'arm_v7_neon',
- 'Nexus7v2': 'arm_v7_neon',
- 'Nexus9': 'arm64',
- 'NexusPlayer': 'x86',
- }[builder_dict['model']]
-
- # iOS.
- if 'iOS' in builder_dict.get('os', ''):
- return {
- 'iPad4': 'iPad4,1',
- }[builder_dict['model']]
-
- return None
-
-
-def product_board(builder_dict):
- if 'Android' in builder_dict.get('os', ''):
- return {
- 'AndroidOne': 'sprout',
- 'GalaxyS3': 'm0', #'smdk4x12', Detected incorrectly by swarming?
- 'GalaxyS4': None, # TODO(borenet,kjlubick)
- 'NVIDIA_Shield': 'foster',
- 'Nexus10': 'manta',
- 'Nexus5': 'hammerhead',
- 'Nexus6': 'shamu',
- 'Nexus7': 'grouper',
- 'Nexus7v2': 'flo',
- 'Nexus9': 'flounder',
- 'NexusPlayer': 'fugu',
- }[builder_dict['model']]
- return None
-
-
-def get_builder_spec(api, builder_name):
- builder_dict = api.builder_name_schema.DictForBuilderName(builder_name)
- rv = {
- 'builder_cfg': builder_dict,
- }
- device = device_cfg(builder_dict)
- if device:
- rv['device_cfg'] = device
- board = product_board(builder_dict)
- if board:
- rv['product.board'] = board
-
- role = builder_dict['role']
- if role == api.builder_name_schema.BUILDER_ROLE_HOUSEKEEPER:
- configuration = CONFIG_RELEASE
- else:
- configuration = builder_dict.get('configuration', CONFIG_DEBUG)
- arch = (builder_dict.get('arch') or builder_dict.get('target_arch'))
- if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'):
- configuration += '_x64'
- rv['configuration'] = configuration
- if configuration == 'Coverage':
- rv['do_compile_steps'] = False
- rv['do_test_steps'] = role == api.builder_name_schema.BUILDER_ROLE_TEST
- rv['do_perf_steps'] = role == api.builder_name_schema.BUILDER_ROLE_PERF
-
- # Do we upload perf results?
- upload_perf_results = False
- if (role == api.builder_name_schema.BUILDER_ROLE_PERF and
- CONFIG_RELEASE in configuration):
- upload_perf_results = True
- rv['upload_perf_results'] = upload_perf_results
-
- # Do we upload correctness results?
- skip_upload_bots = [
- 'ASAN',
- 'Coverage',
- 'MSAN',
- 'TSAN',
- 'UBSAN',
- 'Valgrind',
- ]
- upload_dm_results = True
- for s in skip_upload_bots:
- if s in builder_name:
- upload_dm_results = False
- break
- rv['upload_dm_results'] = upload_dm_results
-
- return rv
-
-
class SkiaVarsApi(recipe_api.RecipeApi):
def make_path(self, *path):
self.m.path.c.base_paths[key] = tuple(path)
return self.m.path[key]
- def gsutil_env(self, boto_file):
- """Environment variables for gsutil."""
- boto_path = None
- if boto_file:
- boto_path = self.m.path.join(self.home_dir, boto_file)
- return {'AWS_CREDENTIAL_FILE': boto_path,
- 'BOTO_CONFIG': boto_path}
-
- @property
- def home_dir(self):
- """Find the home directory."""
- home_dir = os.path.expanduser('~')
- if self._test_data.enabled:
- home_dir = '[HOME]'
- return home_dir
-
- def get_builder_spec(self, builder_name):
- """Return the builder_spec for the given builder name."""
- return get_builder_spec(self.m, builder_name)
-
def setup(self):
"""Prepare the variables."""
# Setup
# Some bots also require a checkout of PDFium.
self.need_pdfium_checkout = 'PDFium' in self.builder_name
- # Obtain the spec for this builder. Use it to set more properties.
- self.builder_spec = get_builder_spec(self.m, self.builder_name)
-
- self.builder_cfg = self.builder_spec['builder_cfg']
+ self.builder_cfg = self.m.builder_name_schema.DictForBuilderName(
+ self.builder_name)
self.role = self.builder_cfg['role']
+ if self.role == self.m.builder_name_schema.BUILDER_ROLE_HOUSEKEEPER:
+ self.configuration = CONFIG_RELEASE
+ else:
+ self.configuration = self.builder_cfg.get('configuration', CONFIG_DEBUG)
+ arch = (self.builder_cfg.get('arch') or self.builder_cfg.get('target_arch'))
+ if ('Win' in self.builder_cfg.get('os', '') and arch == 'x86_64'):
+ self.configuration += '_x64'
- self.configuration = self.builder_spec['configuration']
self.default_env.update({'SKIA_OUT': self.skia_out,
'BUILDTYPE': self.configuration})
- self.do_compile_steps = self.builder_spec.get('do_compile_steps', True)
- self.do_test_steps = self.builder_spec['do_test_steps']
- self.do_perf_steps = self.builder_spec['do_perf_steps']
self.is_trybot = self.builder_cfg['is_trybot']
self.issue = None
self.patchset = None
self.issue = self.m.properties['issue']
self.patchset = self.m.properties['patchset']
self.rietveld = self.m.properties['rietveld']
- self.upload_dm_results = self.builder_spec['upload_dm_results']
- self.upload_perf_results = self.builder_spec['upload_perf_results']
self.dm_dir = self.m.path.join(
self.swarming_out_dir, 'dm')
self.perf_data_dir = self.m.path.join(self.swarming_out_dir,
'perfdata', self.builder_name, 'data')
+
+ @property
+ def upload_dm_results(self):
+ # TODO(borenet): Move this into the swarm_test recipe.
+ skip_upload_bots = [
+ 'ASAN',
+ 'Coverage',
+ 'MSAN',
+ 'TSAN',
+ 'UBSAN',
+ 'Valgrind',
+ ]
+ upload_dm_results = True
+ for s in skip_upload_bots:
+ if s in self.m.properties['buildername']:
+ upload_dm_results = False
+ break
+ return upload_dm_results
+
+ @property
+ def upload_perf_results(self):
+ # TODO(borenet): Move this into the swarm_perf recipe.
+ return ('Release' in self.m.properties['buildername'])
# Recipe module for Skia Swarming trigger.
+import os
import json
if builder_cfg['role'] == 'Housekeeper':
return 'Build-Ubuntu-GCC-x86_64-Release-Shared'
if builder_cfg['role'] in ('Test', 'Perf'):
- os = builder_cfg['os']
+ task_os = builder_cfg['os']
extra_config = builder_cfg.get('extra_config')
- if os == 'Android':
+ if task_os == 'Android':
if extra_config == 'Vulkan':
- extra_config = '%s_%s' % (os, 'Vulkan')
+ extra_config = '%s_%s' % (task_os, 'Vulkan')
else:
- extra_config = os
- os = 'Ubuntu'
- elif os == 'iOS':
- extra_config = os
- os = 'Mac'
- elif 'Win' in os:
- os = 'Win'
+ extra_config = task_os
+ task_os = 'Ubuntu'
+ elif task_os == 'iOS':
+ extra_config = task_os
+ task_os = 'Mac'
+ elif 'Win' in task_os:
+ task_os = 'Win'
return api.builder_name_schema.MakeBuilderName(
role=api.builder_name_schema.BUILDER_ROLE_BUILD,
- os=os,
+ os=task_os,
compiler=builder_cfg['compiler'],
target_arch=builder_cfg['arch'],
configuration=builder_cfg['configuration'],
return builder_name
-def swarm_dimensions(builder_spec):
+def swarm_dimensions(builder_cfg):
"""Return a dict of keys and values to be used as Swarming bot dimensions."""
dimensions = {
'pool': 'Skia',
}
- builder_cfg = builder_spec['builder_cfg']
dimensions['os'] = builder_cfg.get('os', 'Ubuntu')
if 'Win' in builder_cfg.get('os', ''):
dimensions['os'] = 'Windows'
if builder_cfg['role'] in ('Test', 'Perf'):
if 'Android' in builder_cfg['os']:
# For Android, the device type is a better dimension than CPU or GPU.
- dimensions['device_type'] = builder_spec['product.board']
+ dimensions['device_type'] = {
+ 'AndroidOne': 'sprout',
+ 'GalaxyS3': 'm0', #'smdk4x12', Detected incorrectly by swarming?
+ 'GalaxyS4': None, # TODO(borenet,kjlubick)
+ 'NVIDIA_Shield': 'foster',
+ 'Nexus10': 'manta',
+ 'Nexus5': 'hammerhead',
+ 'Nexus6': 'shamu',
+ 'Nexus7': 'grouper',
+ 'Nexus7v2': 'flo',
+ 'Nexus9': 'flounder',
+ 'NexusPlayer': 'fugu',
+ }[builder_cfg['model']]
elif 'iOS' in builder_cfg['os']:
# For iOS, the device type is a better dimension than CPU or GPU.
- dimensions['device'] = builder_spec['device_cfg']
+ dimensions['device'] = {
+ 'iPad4': 'iPad4,1',
+ }[builder_cfg['model']]
# TODO(borenet): Replace this hack with something better.
dimensions['os'] = 'iOS-9.2'
elif builder_cfg['cpu_or_gpu'] == 'CPU':
def trigger_task(api, task_name, builder, master, slave, buildnumber,
- builder_spec, got_revision, infrabots_dir, idempotent=False,
+ builder_cfg, got_revision, infrabots_dir, idempotent=False,
store_output=True, extra_isolate_hashes=None, expiration=None,
hard_timeout=None, io_timeout=None, cipd_packages=None):
"""Trigger the given bot to run as a Swarming task."""
'slavename': slave,
'swarm_out_dir': '${ISOLATED_OUTDIR}',
}
- builder_cfg = builder_spec['builder_cfg']
if builder_cfg['is_trybot']:
properties['issue'] = str(api.properties['issue'])
properties['patchset'] = str(api.properties['patchset'])
extra_args.append('%s=%s' % (k, v))
isolate_base_dir = api.path['slave_build']
- dimensions = swarm_dimensions(builder_spec)
+ dimensions = swarm_dimensions(builder_cfg)
isolate_blacklist = ['.git', 'out', '*.pyc', '.recipe_deps']
isolate_vars = {
'WORKDIR': api.path['slave_build'],
return got_revision
-def housekeeper_swarm(api, builder_spec, got_revision, infrabots_dir,
+def housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
extra_isolate_hashes):
task = trigger_task(
api,
api.properties['mastername'],
api.properties['slavename'],
api.properties['buildnumber'],
- builder_spec,
+ builder_cfg,
got_revision,
infrabots_dir,
idempotent=False,
return api.swarming.collect_swarming_task(task)
-def recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
+def recreate_skps_swarm(api, builder_cfg, got_revision, infrabots_dir,
extra_isolate_hashes):
task = trigger_task(
api,
api.properties['mastername'],
api.properties['slavename'],
api.properties['buildnumber'],
- builder_spec,
+ builder_cfg,
got_revision,
infrabots_dir,
idempotent=False,
def infra_swarm(api, got_revision, infrabots_dir, extra_isolate_hashes):
- # Fake the builder spec.
- builder_spec = {
- 'builder_cfg': {
- 'role': 'Infra',
- 'is_trybot': api.builder_name_schema.IsTrybot(
- api.properties['buildername'])
- }
+ # Fake the builder cfg.
+ builder_cfg = {
+ 'role': 'Infra',
+ 'is_trybot': api.builder_name_schema.IsTrybot(
+ api.properties['buildername'])
}
task = trigger_task(
api,
api.properties['mastername'],
api.properties['slavename'],
api.properties['buildnumber'],
- builder_spec,
+ builder_cfg,
got_revision,
infrabots_dir,
idempotent=False,
return api.swarming.collect_swarming_task(task)
-def compile_steps_swarm(api, builder_spec, got_revision, infrabots_dir,
- extra_isolate_hashes, cipd_packages):
+def compile_steps_swarm(api, builder_cfg, got_revision, infrabots_dir):
builder_name = derive_compile_bot_name(api)
- compile_builder_spec = builder_spec
- if builder_name != api.properties['buildername']:
- compile_builder_spec = api.vars.get_builder_spec(builder_name)
+ compile_builder_cfg = api.builder_name_schema.DictForBuilderName(builder_name)
- extra_hashes = extra_isolate_hashes[:]
+ cipd_packages = []
+
+ # Android bots require a toolchain.
+ if 'Android' in api.properties['buildername']:
+ cipd_packages.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
# Windows bots require a toolchain.
if 'Win' in builder_name:
master,
slave,
buildnumber,
- compile_builder_spec,
+ compile_builder_cfg,
got_revision,
infrabots_dir,
idempotent=True,
store_output=False,
- extra_isolate_hashes=extra_hashes,
cipd_packages=cipd_packages)
# Wait for compile to finish, record the results hash.
return expiration, hard_timeout, io_timeout
-def perf_steps_trigger(api, builder_spec, got_revision, infrabots_dir,
+def gsutil_env(api, boto_file):
+ """Environment variables for gsutil."""
+ home_dir = os.path.expanduser('~')
+ if api.path._test_data.enabled:
+ home_dir = '[HOME]'
+
+ boto_path = None
+ if boto_file:
+ boto_path = api.path.join(home_dir, boto_file)
+ return {'AWS_CREDENTIAL_FILE': boto_path,
+ 'BOTO_CONFIG': boto_path}
+
+
+def perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
extra_hashes, cipd_packages):
"""Trigger perf tests via Swarming."""
- expiration, hard_timeout, io_timeout = get_timeouts(
- builder_spec['builder_cfg'])
+ expiration, hard_timeout, io_timeout = get_timeouts(builder_cfg)
return trigger_task(
api,
'perf',
api.properties['mastername'],
api.properties['slavename'],
api.properties['buildnumber'],
- builder_spec,
+ builder_cfg,
got_revision,
infrabots_dir,
extra_isolate_hashes=extra_hashes,
cipd_packages=cipd_packages)
-def perf_steps_collect(api, task, upload_perf_results, got_revision,
- is_trybot):
+def perf_steps_collect(api, task, got_revision, is_trybot):
"""Wait for perf steps to finish and upload results."""
# Wait for nanobench to finish, download the results.
api.run.rmtree(task.task_output_dir)
api.swarming.collect_swarming_task(task)
# Upload the results.
- if upload_perf_results:
+ if api.vars.upload_perf_results:
perf_data_dir = api.path['slave_build'].join(
'perfdata', api.properties['buildername'], 'data')
git_timestamp = api.git.get_timestamp(test_data='1408633190',
if is_trybot:
upload_args.append(api.properties['issue'])
api.python(
- 'Upload perf results',
- script=api.core.resource('upload_bench_results.py'),
- args=upload_args,
- cwd=api.path['checkout'],
- infra_step=True)
+ 'Upload perf results',
+ script=api.core.resource('upload_bench_results.py'),
+ args=upload_args,
+ cwd=api.path['checkout'],
+ infra_step=True)
-def test_steps_trigger(api, builder_spec, got_revision, infrabots_dir,
+def test_steps_trigger(api, builder_cfg, got_revision, infrabots_dir,
extra_hashes, cipd_packages):
"""Trigger DM via Swarming."""
- expiration, hard_timeout, io_timeout = get_timeouts(
- builder_spec['builder_cfg'])
+ expiration, hard_timeout, io_timeout = get_timeouts(builder_cfg)
return trigger_task(
api,
'test',
api.properties['mastername'],
api.properties['slavename'],
api.properties['buildnumber'],
- builder_spec,
+ builder_cfg,
got_revision,
infrabots_dir,
extra_isolate_hashes=extra_hashes,
cipd_packages=cipd_packages)
-def test_steps_collect(api, task, upload_dm_results, got_revision, is_trybot,
- builder_cfg):
+def test_steps_collect(api, task, got_revision, is_trybot, builder_cfg):
"""Collect the test results from Swarming."""
# Wait for tests to finish, download the results.
api.run.rmtree(task.task_output_dir)
api.swarming.collect_swarming_task(task)
# Upload the results.
- if upload_dm_results:
+ if api.vars.upload_dm_results:
dm_dir = api.path['slave_build'].join('dm')
dm_src = task.task_output_dir.join('0', 'dm')
api.run.rmtree(dm_dir)
api.path['slave_build'].join('skia', 'common', 'py', 'utils'),
],
cwd=api.path['checkout'],
- env=api.vars.gsutil_env('chromium-skia-gm.boto'),
+ env=gsutil_env(api, 'chromium-skia-gm.boto'),
infra_step=True)
if builder_cfg['configuration'] == 'Coverage':
script=api.core.resource('upload_bench_results.py'),
args=upload_args,
cwd=api.path['checkout'],
- env=api.vars.gsutil_env('chromium-skia-gm.boto'),
+ env=gsutil_env(api, 'chromium-skia-gm.boto'),
infra_step=True)
# Transform the coverage_by_line_${git_hash}.json file received from
extra_hashes = []
# Get ready to compile.
- compile_cipd_deps = []
- extra_compile_hashes = []
-
infrabots_dir = api.path['checkout'].join('infra', 'bots')
if 'Infra' in api.properties['buildername']:
return infra_swarm(api, got_revision, infrabots_dir, extra_hashes)
- builder_spec = api.vars.get_builder_spec(api.properties['buildername'])
- builder_cfg = builder_spec['builder_cfg']
+ builder_cfg = api.builder_name_schema.DictForBuilderName(
+ api.properties['buildername'])
if 'RecreateSKPs' in api.properties['buildername']:
- recreate_skps_swarm(api, builder_spec, got_revision, infrabots_dir,
+ recreate_skps_swarm(api, builder_cfg, got_revision, infrabots_dir,
extra_hashes)
return
- if 'Android' in api.properties['buildername']:
- compile_cipd_deps.append(cipd_pkg(api, infrabots_dir, 'android_sdk'))
# Compile.
- do_compile_steps = builder_spec.get('do_compile_steps', True)
+ do_compile_steps = True
+ if 'Coverage' in api.properties['buildername']:
+ do_compile_steps = False
if do_compile_steps:
extra_hashes.append(compile_steps_swarm(
- api, builder_spec, got_revision, infrabots_dir, extra_compile_hashes,
- cipd_packages=compile_cipd_deps))
+ api, builder_cfg, got_revision, infrabots_dir))
if builder_cfg['role'] == 'Housekeeper':
- housekeeper_swarm(api, builder_spec, got_revision, infrabots_dir,
+ housekeeper_swarm(api, builder_cfg, got_revision, infrabots_dir,
extra_hashes)
return
# CIPD packages needed by test/perf.
cipd_packages = []
- do_test_steps = builder_spec['do_test_steps']
- do_perf_steps = builder_spec['do_perf_steps']
+ do_test_steps = (
+ builder_cfg['role'] == api.builder_name_schema.BUILDER_ROLE_TEST)
+ do_perf_steps = (
+ builder_cfg['role'] == api.builder_name_schema.BUILDER_ROLE_PERF)
if not (do_test_steps or do_perf_steps):
return
test_task = None
perf_task = None
if do_test_steps:
- test_task = test_steps_trigger(api, builder_spec, got_revision,
+ test_task = test_steps_trigger(api, builder_cfg, got_revision,
infrabots_dir, extra_hashes, cipd_packages)
if do_perf_steps:
- perf_task = perf_steps_trigger(api, builder_spec, got_revision,
+ perf_task = perf_steps_trigger(api, builder_cfg, got_revision,
infrabots_dir, extra_hashes, cipd_packages)
is_trybot = builder_cfg['is_trybot']
+
+ # Wait for results, then upload them if necessary.
+
if test_task:
- test_steps_collect(api, test_task, builder_spec['upload_dm_results'],
+ test_steps_collect(api, test_task,
got_revision, is_trybot, builder_cfg)
+
if perf_task:
- perf_steps_collect(api, perf_task, builder_spec['upload_perf_results'],
+ perf_steps_collect(api, perf_task,
got_revision, is_trybot)
projects / platform / upstream / libSkiaSharp.git / commitdiff