+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import argparse
-import json
-
-
-"""Add the given hash to the includes section of the given isolated file."""
-
-
-def add_isolated_hash(isolated_file, hash_str):
- with open(isolated_file) as f:
- isolated = json.load(f)
- isolated['includes'].append(hash_str)
- with open(isolated_file, 'w') as f:
- json.dump(isolated, f, sort_keys=True)
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--isolated_file', required=True)
- parser.add_argument('--hash', required=True)
- args = parser.parse_args()
- add_isolated_hash(args.isolated_file, args.hash)
-
-
-if __name__ == '__main__':
- main()
# found in the LICENSE file.
-import contextlib
-import glob
-import math
import os
-import psutil
import shutil
-import socket
import subprocess
-import sys
-import time
-import urllib2
-from flavor import android_flavor
-from flavor import chromeos_flavor
-from flavor import cmake_flavor
-from flavor import coverage_flavor
-from flavor import default_flavor
-from flavor import ios_flavor
-from flavor import valgrind_flavor
-from flavor import xsan_flavor
-
-
-CONFIG_COVERAGE = 'Coverage'
-CONFIG_DEBUG = 'Debug'
-CONFIG_RELEASE = 'Release'
-VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE)
-
-BUILD_PRODUCTS_WHITELIST = [
- 'dm',
- 'dm.exe',
- 'nanobench',
- 'nanobench.exe',
- '*.so',
- '*.dll',
-]
-
-GM_ACTUAL_FILENAME = 'actual-results.json'
-GM_EXPECTATIONS_FILENAME = 'expected-results.json'
-GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt'
-
-GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes'
GS_GM_BUCKET = 'chromium-skia-gm'
-GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries'
GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s'
GS_SUBDIR_TMPL_SKP = 'playback_%s/skps'
-SKIA_REPO = 'https://skia.googlesource.com/skia.git'
-INFRA_REPO = 'https://skia.googlesource.com/buildbot.git'
-
-SERVICE_ACCOUNT_FILE = 'service-account-skia.json'
-SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json'
-
VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION'
VERSION_FILE_SKP = 'SKP_VERSION'
-def is_android(bot_cfg):
- """Determine whether the given bot is an Android bot."""
- return ('Android' in bot_cfg.get('extra_config', '') or
- bot_cfg.get('os') == 'Android')
-
-def is_chromeos(bot_cfg):
- return ('CrOS' in bot_cfg.get('extra_config', '') or
- bot_cfg.get('os') == 'ChromeOS')
-
-def is_cmake(bot_cfg):
- return 'CMake' in bot_cfg.get('extra_config', '')
-
-def is_ios(bot_cfg):
- return ('iOS' in bot_cfg.get('extra_config', '') or
- bot_cfg.get('os') == 'iOS')
-
-
-def is_valgrind(bot_cfg):
- return 'Valgrind' in bot_cfg.get('extra_config', '')
-
-
-def is_xsan(bot_cfg):
- return (bot_cfg.get('extra_config') == 'ASAN' or
- bot_cfg.get('extra_config') == 'MSAN' or
- bot_cfg.get('extra_config') == 'TSAN')
-
-
def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir):
# Ensure that the tmp_dir exists.
if not os.path.isdir(tmp_dir):
subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir])
with open(actual_version_file, 'w') as f:
f.write(expected_version)
-
-
-def get_uninteresting_hashes(hashes_file):
- retries = 5
- timeout = 60
- wait_base = 15
-
- socket.setdefaulttimeout(timeout)
- for retry in range(retries):
- try:
- with contextlib.closing(
- urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w:
- hashes = w.read()
- with open(hashes_file, 'w') as f:
- f.write(hashes)
- break
- except Exception as e:
- print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % (
- GOLD_UNINTERESTING_HASHES_URL, e)
- if retry == retries:
- raise
- waittime = wait_base * math.pow(2, retry)
- print 'Retry in %d seconds.' % waittime
- time.sleep(waittime)
-
-
-class BotInfo(object):
- def __init__(self, bot_name, swarm_out_dir):
- """Initialize the bot, given its name.
-
- Assumes that CWD is the directory containing this file.
- """
- self.name = bot_name
- self.skia_dir = os.path.abspath(os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- os.pardir, os.pardir))
- self.swarm_out_dir = swarm_out_dir
- os.chdir(self.skia_dir)
- self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir))
- self.infrabots_dir = os.path.join(self.skia_dir, 'infra', 'bots')
- self.home_dir = os.path.expanduser('~')
-
- self.spec = self.get_bot_spec(bot_name)
- self.bot_cfg = self.spec['builder_cfg']
- self.out_dir = os.path.join(os.pardir, 'out')
- self.configuration = self.spec['configuration']
- self.default_env = {
- 'CHROME_HEADLESS': '1',
- 'SKIA_OUT': self.out_dir,
- 'BUILDTYPE': self.configuration,
- 'PATH': os.environ['PATH'],
- }
- if 'Win' in self.bot_cfg['os']:
- self.default_env['SystemRoot'] = 'C:\\Windows'
- self.default_env['TEMP'] = os.path.join(
- self.home_dir, 'AppData', 'Local', 'Temp')
- self.default_env['TMP'] = self.default_env['TEMP']
- self.default_env.update(self.spec['env'])
- self.build_targets = [str(t) for t in self.spec['build_targets']]
- self.is_trybot = self.bot_cfg['is_trybot']
- self.upload_dm_results = self.spec['upload_dm_results']
- self.upload_perf_results = self.spec['upload_perf_results']
- self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata',
- self.name, 'data')
- self.resource_dir = os.path.join(self.skia_dir, 'resources')
- self.images_dir = os.path.join(self.build_dir, 'images')
- self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps')
- self.dm_flags = self.spec['dm_flags']
- self.nanobench_flags = self.spec['nanobench_flags']
- self._ccache = None
- self._checked_for_ccache = False
- self._already_ran = {}
- self.tmp_dir = os.path.join(self.build_dir, 'tmp')
- self.flavor = self.get_flavor(self.bot_cfg)
-
- # These get filled in during subsequent steps.
- self.device_dirs = None
- self.build_number = None
- self.got_revision = None
- self.master_name = None
- self.slave_name = None
-
- @property
- def ccache(self):
- if not self._checked_for_ccache:
- self._checked_for_ccache = True
- if sys.platform != 'win32':
- try:
- result = subprocess.check_output(['which', 'ccache'])
- self._ccache = result.rstrip()
- except subprocess.CalledProcessError:
- pass
-
- return self._ccache
-
- def get_bot_spec(self, bot_name):
- """Retrieve the bot spec for this bot."""
- sys.path.append(self.skia_dir)
- from tools import buildbot_spec
- return buildbot_spec.get_builder_spec(bot_name)
-
- def get_flavor(self, bot_cfg):
- """Return a flavor utils object specific to the given bot."""
- if is_android(bot_cfg):
- return android_flavor.AndroidFlavorUtils(self)
- elif is_chromeos(bot_cfg):
- return chromeos_flavor.ChromeOSFlavorUtils(self)
- elif is_cmake(bot_cfg):
- return cmake_flavor.CMakeFlavorUtils(self)
- elif is_ios(bot_cfg):
- return ios_flavor.iOSFlavorUtils(self)
- elif is_valgrind(bot_cfg):
- return valgrind_flavor.ValgrindFlavorUtils(self)
- elif is_xsan(bot_cfg):
- return xsan_flavor.XSanFlavorUtils(self)
- elif bot_cfg.get('configuration') == CONFIG_COVERAGE:
- return coverage_flavor.CoverageFlavorUtils(self)
- else:
- return default_flavor.DefaultFlavorUtils(self)
-
- def run(self, cmd, env=None, cwd=None):
- _env = {}
- _env.update(self.default_env)
- _env.update(env or {})
- cwd = cwd or self.skia_dir
- print '============'
- print 'CMD: %s' % cmd
- print 'CWD: %s' % cwd
- print 'ENV: %s' % _env
- print '============'
- subprocess.check_call(cmd, env=_env, cwd=cwd)
-
- def compile_steps(self):
- for t in self.build_targets:
- self.flavor.compile(t)
- dst = os.path.join(self.swarm_out_dir, 'out', self.configuration)
- os.makedirs(dst)
- for pattern in BUILD_PRODUCTS_WHITELIST:
- path = os.path.join(self.out_dir, self.configuration, pattern)
- for f in glob.glob(path):
- print 'Copying build product %s' % f
- shutil.copy(f, dst)
- self.cleanup()
-
- def _run_once(self, fn, *args, **kwargs):
- if not fn.__name__ in self._already_ran:
- self._already_ran[fn.__name__] = True
- fn(*args, **kwargs)
-
- def install(self):
- """Copy the required executables and files to the device."""
- self.device_dirs = self.flavor.get_device_dirs()
-
- # Run any device-specific installation.
- self.flavor.install()
-
- # TODO(borenet): Only copy files which have changed.
- # Resources
- self.flavor.copy_directory_contents_to_device(self.resource_dir,
- self.device_dirs.resource_dir)
-
- def _key_params(self):
- """Build a unique key from the builder name (as a list).
-
- E.g. arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6
- """
- # Don't bother to include role, which is always Test.
- # TryBots are uploaded elsewhere so they can use the same key.
- blacklist = ['role', 'is_trybot']
-
- flat = []
- for k in sorted(self.bot_cfg.keys()):
- if k not in blacklist:
- flat.append(k)
- flat.append(self.bot_cfg[k])
- return flat
-
- def test_steps(self, got_revision, master_name, slave_name, build_number,
- issue=None, patchset=None):
- """Run the DM test."""
- self.build_number = build_number
- self.got_revision = got_revision
- self.master_name = master_name
- self.slave_name = slave_name
- self._run_once(self.install)
-
- use_hash_file = False
- if self.upload_dm_results:
- # This must run before we write anything into self.device_dirs.dm_dir
- # or we may end up deleting our output on machines where they're the same.
- host_dm_dir = os.path.join(self.swarm_out_dir, 'dm')
- print 'host dm dir: %s' % host_dm_dir
- self.flavor.create_clean_host_dir(host_dm_dir)
- if str(host_dm_dir) != str(self.device_dirs.dm_dir):
- self.flavor.create_clean_device_dir(self.device_dirs.dm_dir)
-
- # Obtain the list of already-generated hashes.
- if not os.path.isdir(self.tmp_dir):
- os.makedirs(self.tmp_dir)
- hash_filename = 'uninteresting_hashes.txt'
- host_hashes_file = os.path.join(self.tmp_dir, hash_filename)
- hashes_file = self.flavor.device_path_join(
- self.device_dirs.tmp_dir, hash_filename)
-
- try:
- get_uninteresting_hashes(host_hashes_file)
- except Exception:
- pass
-
- if os.path.exists(host_hashes_file):
- self.flavor.copy_file_to_device(host_hashes_file, hashes_file)
- use_hash_file = True
-
- # Run DM.
- properties = [
- 'gitHash', self.got_revision,
- 'master', self.master_name,
- 'builder', self.name,
- 'build_number', self.build_number,
- ]
- if self.is_trybot:
- if not issue:
- raise Exception('issue is required for trybots.')
- if not patchset:
- raise Exception('patchset is required for trybots.')
- properties.extend([
- 'issue', issue,
- 'patchset', patchset,
- ])
-
- args = [
- 'dm',
- '--undefok', # This helps branches that may not know new flags.
- '--resourcePath', self.device_dirs.resource_dir,
- '--skps', self.device_dirs.skp_dir,
- '--images', self.flavor.device_path_join(
- self.device_dirs.images_dir, 'dm'),
- '--nameByHash',
- '--properties'
- ] + properties
-
- args.append('--key')
- args.extend(self._key_params())
- if use_hash_file:
- args.extend(['--uninterestingHashesFile', hashes_file])
- if self.upload_dm_results:
- args.extend(['--writePath', self.device_dirs.dm_dir])
-
- skip_flag = None
- if self.bot_cfg.get('cpu_or_gpu') == 'CPU':
- skip_flag = '--nogpu'
- elif self.bot_cfg.get('cpu_or_gpu') == 'GPU':
- skip_flag = '--nocpu'
- if skip_flag:
- args.append(skip_flag)
- args.extend(self.dm_flags)
-
- self.flavor.run(args, env=self.default_env)
-
- if self.upload_dm_results:
- # Copy images and JSON to host machine if needed.
- self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir,
- host_dm_dir)
-
- # See skia:2789.
- if ('Valgrind' in self.name and
- self.bot_cfg.get('cpu_or_gpu') == 'GPU'):
- abandonGpuContext = list(args)
- abandonGpuContext.append('--abandonGpuContext')
- self.flavor.run(abandonGpuContext)
- preAbandonGpuContext = list(args)
- preAbandonGpuContext.append('--preAbandonGpuContext')
- self.flavor.run(preAbandonGpuContext)
-
- self.cleanup()
-
- def perf_steps(self, got_revision, master_name, slave_name, build_number,
- issue=None, patchset=None):
- """Run Skia benchmarks."""
- self.build_number = build_number
- self.got_revision = got_revision
- self.master_name = master_name
- self.slave_name = slave_name
- self._run_once(self.install)
- if self.upload_perf_results:
- self.flavor.create_clean_device_dir(self.device_dirs.perf_data_dir)
-
- # Run nanobench.
- properties = [
- '--properties',
- 'gitHash', self.got_revision,
- 'build_number', self.build_number,
- ]
- if self.is_trybot:
- if not issue:
- raise Exception('issue is required for trybots.')
- if not patchset:
- raise Exception('patchset is required for trybots.')
- properties.extend([
- 'issue', issue,
- 'patchset', patchset,
- ])
-
- target = 'nanobench'
- if 'VisualBench' in self.name:
- target = 'visualbench'
- args = [
- target,
- '--undefok', # This helps branches that may not know new flags.
- '-i', self.device_dirs.resource_dir,
- '--skps', self.device_dirs.skp_dir,
- '--images', self.flavor.device_path_join(
- self.device_dirs.images_dir, 'dm'), # Using DM images for now.
- ]
-
- skip_flag = None
- if self.bot_cfg.get('cpu_or_gpu') == 'CPU':
- skip_flag = '--nogpu'
- elif self.bot_cfg.get('cpu_or_gpu') == 'GPU':
- skip_flag = '--nocpu'
- if skip_flag:
- args.append(skip_flag)
- args.extend(self.nanobench_flags)
-
- if self.upload_perf_results:
- json_path = self.flavor.device_path_join(
- self.device_dirs.perf_data_dir,
- 'nanobench_%s.json' % self.got_revision)
- args.extend(['--outResultsFile', json_path])
- args.extend(properties)
-
- keys_blacklist = ['configuration', 'role', 'is_trybot']
- args.append('--key')
- for k in sorted(self.bot_cfg.keys()):
- if not k in keys_blacklist:
- args.extend([k, self.bot_cfg[k]])
-
- self.flavor.run(args, env=self.default_env)
-
- # See skia:2789.
- if ('Valgrind' in self.name and
- self.bot_cfg.get('cpu_or_gpu') == 'GPU'):
- abandonGpuContext = list(args)
- abandonGpuContext.extend(['--abandonGpuContext', '--nocpu'])
- self.flavor.run(abandonGpuContext, env=self.default_env)
-
- # Copy results to host.
- if self.upload_perf_results:
- if not os.path.exists(self.perf_data_dir):
- os.makedirs(self.perf_data_dir)
- self.flavor.copy_directory_contents_to_host(
- self.device_dirs.perf_data_dir, self.perf_data_dir)
-
- self.cleanup()
-
- def cleanup(self):
- if sys.platform == 'win32':
- # Kill mspdbsrv.exe, which tends to hang around after the build finishes.
- for p in psutil.process_iter():
- try:
- if p.name == 'mspdbsrv.exe':
- p.kill()
- except psutil._error.AccessDenied:
- pass
- self.flavor.cleanup_steps()
'includes': [
'skia_repo.isolate',
],
- 'variables': {
- 'command': [
- 'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}',
- ],
- },
}
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import argparse
-import common
-import os
-import sys
-import utils
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--builder_name', required=True)
- parser.add_argument('--swarm_out_dir', required=True)
- args = parser.parse_args()
- with utils.print_timings():
- bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir))
- bot.compile_steps()
-
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import collections
-import json
-
-
-DEFAULT_SDK_ROOT = '/home/chrome-bot/android-sdk-linux'
-MAC_SDK_ROOT = '/Users/chrome-bot/adt-bundle-mac-x86_64-20140702/sdk'
-MACMINI_SDK_ROOT = '/Users/chrome-bot/android-sdk-macosx'
-
-SlaveInfo = collections.namedtuple('SlaveInfo',
- 'serial android_sdk_root has_root')
-
-SLAVE_INFO = {
- 'skiabot-mac-10_8-compile-000':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-001':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-002':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-003':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-004':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-005':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-006':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-007':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-008':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-mac-10_8-compile-009':
- SlaveInfo('noserial', MAC_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-androidone-001':
- SlaveInfo('AG86044202A04GC', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-androidone-002':
- SlaveInfo('AG8404EC06G02GC', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-androidone-003':
- SlaveInfo('AG8404EC0688EGC', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-galaxys3-001':
- SlaveInfo('4df713b8244a21cf', DEFAULT_SDK_ROOT, False),
- 'skiabot-shuttle-ubuntu12-galaxys3-002':
- SlaveInfo('32309a56e9b3a09f', DEFAULT_SDK_ROOT, False),
- 'skiabot-shuttle-ubuntu12-galaxys4-001':
- SlaveInfo('4d0032a5d8cb6125', MACMINI_SDK_ROOT, False),
- 'skiabot-shuttle-ubuntu12-galaxys4-002':
- SlaveInfo('4d00353cd8ed61c3', MACMINI_SDK_ROOT, False),
- 'skiabot-shuttle-ubuntu12-nexus5-001':
- SlaveInfo('03f61449437cc47b', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus5-002':
- SlaveInfo('018dff3520c970f6', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-nexus6-001':
- SlaveInfo('ZX1G22JJWS', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-nexus6-002':
- SlaveInfo('ZX1G22JN35', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-nexus6-003':
- SlaveInfo('ZX1G22JXXM', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus7-001':
- SlaveInfo('015d210a13480604', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus7-002':
- SlaveInfo('015d18848c280217', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus7-003':
- SlaveInfo('015d16897c401e17', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus9-001':
- SlaveInfo('HT43RJT00022', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus9-002':
- SlaveInfo('HT4AEJT03112', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus9-003':
- SlaveInfo('HT4ADJT03339', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus10-001':
- SlaveInfo('R32C801B5LH', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexus10-003':
- SlaveInfo('R32CB017X2L', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexusplayer-001':
- SlaveInfo('D76C708B', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu12-nexusplayer-002':
- SlaveInfo('8AB5139A', DEFAULT_SDK_ROOT, True),
- 'skiabot-shuttle-ubuntu15-nvidia-shield-001':
- SlaveInfo('04217150066510000078', MACMINI_SDK_ROOT, False),
- 'skiabot-linux-housekeeper-003':
- SlaveInfo('noserial', DEFAULT_SDK_ROOT, False),
- 'vm690-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
- 'vm691-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
- 'vm692-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
- 'vm693-m3': SlaveInfo('noserial', MACMINI_SDK_ROOT, False),
- 'default':
- SlaveInfo('noserial', DEFAULT_SDK_ROOT, False),
-}
-
-
-if __name__ == '__main__':
- print json.dumps(SLAVE_INFO) # pragma: no cover
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import android_devices
-import default_flavor
-import os
-
-
-"""Android flavor utils, used for building for and running tests on Android."""
-
-
-class _ADBWrapper(object):
- """Wrapper for ADB."""
- def __init__(self, path_to_adb, serial, android_flavor):
- self._adb = path_to_adb
- self._serial = serial
- self._wait_count = 0
- self._android_flavor = android_flavor
-
- def wait_for_device(self):
- """Run 'adb wait-for-device'."""
- self._wait_count += 1
- cmd = [
- os.path.join(self._android_flavor.android_bin, 'adb_wait_for_device'),
- '-s', self._serial,
- ]
- self._android_flavor._bot_info.run(
- cmd, env=self._android_flavor._default_env)
- cmd = [
- os.path.join(self._android_flavor.android_bin, 'adb_wait_for_charge'),
- '-s', self._serial,
- ]
- self._android_flavor._bot_info.run(
- cmd, env=self._android_flavor._default_env)
-
- def maybe_wait_for_device(self):
- """Run 'adb wait-for-device' if it hasn't already been run."""
- if self._wait_count == 0:
- self.wait_for_device()
-
- def __call__(self, *args, **kwargs):
- self.maybe_wait_for_device()
- return self._android_flavor._bot_info.run(self._adb + args, **kwargs)
-
-
-class AndroidFlavorUtils(default_flavor.DefaultFlavorUtils):
- def __init__(self, skia_api):
- super(AndroidFlavorUtils, self).__init__(skia_api)
- self.device = self._bot_info.spec['device_cfg']
- slave_info = android_devices.SLAVE_INFO.get(
- self._bot_info.slave_name,
- android_devices.SLAVE_INFO['default'])
- self.serial = slave_info.serial
- self.android_bin = os.path.join(
- self._bot_info.skia_dir, 'platform_tools', 'android', 'bin')
- self._android_sdk_root = slave_info.android_sdk_root
- self._adb = _ADBWrapper(
- os.path.join(self._android_sdk_root, 'platform-tools', 'adb'),
- self.serial,
- self)
- self._has_root = slave_info.has_root
- self._default_env = {'ANDROID_SDK_ROOT': self._android_sdk_root,
- 'ANDROID_HOME': self._android_sdk_root,
- 'SKIA_ANDROID_VERBOSE_SETUP': '1'}
-
- def step(self, name, cmd, env=None, **kwargs):
- self._adb.maybe_wait_for_device()
- args = [self.android_bin.join('android_run_skia'),
- '--verbose',
- '--logcat',
- '-d', self.device,
- '-s', self.serial,
- '-t', self._bot_info.configuration,
- ]
- env = dict(env or {})
- env.update(self._default_env)
-
- return self._bot_info.run(self._bot_info.m.step, name=name, cmd=args + cmd,
- env=env, **kwargs)
-
- def compile(self, target):
- """Build the given target."""
- env = dict(self._default_env)
- ccache = self._bot_info.ccache
- if ccache:
- env['ANDROID_MAKE_CCACHE'] = ccache
-
- cmd = [os.path.join(self.android_bin, 'android_ninja'), target,
- '-d', self.device]
- if 'Clang' in self._bot_info.name:
- cmd.append('--clang')
- self._bot_info.run(cmd, env=env)
-
- def device_path_join(self, *args):
- """Like os.path.join(), but for paths on a connected Android device."""
- return '/'.join(args)
-
- def device_path_exists(self, path):
- """Like os.path.exists(), but for paths on a connected device."""
- exists_str = 'FILE_EXISTS'
- return exists_str in self._adb(
- name='exists %s' % self._bot_info.m.path.basename(path),
- serial=self.serial,
- cmd=['shell', 'if', '[', '-e', path, '];',
- 'then', 'echo', exists_str + ';', 'fi'],
- stdout=self._bot_info.m.raw_io.output(),
- infra_step=True
- ).stdout
-
- def _remove_device_dir(self, path):
- """Remove the directory on the device."""
- self._adb(name='rmdir %s' % self._bot_info.m.path.basename(path),
- serial=self.serial,
- cmd=['shell', 'rm', '-r', path],
- infra_step=True)
- # Sometimes the removal fails silently. Verify that it worked.
- if self.device_path_exists(path):
- raise Exception('Failed to remove %s!' % path) # pragma: no cover
-
- def _create_device_dir(self, path):
- """Create the directory on the device."""
- self._adb(name='mkdir %s' % self._bot_info.m.path.basename(path),
- serial=self.serial,
- cmd=['shell', 'mkdir', '-p', path],
- infra_step=True)
-
- def copy_directory_contents_to_device(self, host_dir, device_dir):
- """Like shutil.copytree(), but for copying to a connected device."""
- self._bot_info.run(
- self._bot_info.m.step,
- name='push %s' % self._bot_info.m.path.basename(host_dir),
- cmd=[self.android_bin.join('adb_push_if_needed'), '--verbose',
- '-s', self.serial, host_dir, device_dir],
- env=self._default_env,
- infra_step=True)
-
- def copy_directory_contents_to_host(self, device_dir, host_dir):
- """Like shutil.copytree(), but for copying from a connected device."""
- self._bot_info.run(
- self._bot_info.m.step,
- name='pull %s' % self._bot_info.m.path.basename(device_dir),
- cmd=[self.android_bin.join('adb_pull_if_needed'), '--verbose',
- '-s', self.serial, device_dir, host_dir],
- env=self._default_env,
- infra_step=True)
-
- def copy_file_to_device(self, host_path, device_path):
- """Like shutil.copyfile, but for copying to a connected device."""
- self._adb(name='push %s' % self._bot_info.m.path.basename(host_path),
- serial=self.serial,
- cmd=['push', host_path, device_path],
- infra_step=True)
-
- def create_clean_device_dir(self, path):
- """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
- self._remove_device_dir(path)
- self._create_device_dir(path)
-
- def install(self):
- """Run device-specific installation steps."""
- if self._has_root:
- self._adb(name='adb root',
- serial=self.serial,
- cmd=['root'],
- infra_step=True)
- # Wait for the device to reconnect.
- self._bot_info.run(
- self._bot_info.m.step,
- name='wait',
- cmd=['sleep', '10'],
- infra_step=True)
- self._adb.wait_for_device()
-
- # TODO(borenet): Set CPU scaling mode to 'performance'.
- self._bot_info.run(self._bot_info.m.step,
- name='kill skia',
- cmd=[self.android_bin.join('android_kill_skia'),
- '--verbose', '-s', self.serial],
- env=self._default_env,
- infra_step=True)
- if self._has_root:
- self._adb(name='stop shell',
- serial=self.serial,
- cmd=['shell', 'stop'],
- infra_step=True)
-
- # Print out battery stats.
- self._adb(name='starting battery stats',
- serial=self.serial,
- cmd=['shell', 'dumpsys', 'batteryproperties'],
- infra_step=True)
-
- def cleanup_steps(self):
- """Run any device-specific cleanup steps."""
- self._adb(name='final battery stats',
- serial=self.serial,
- cmd=['shell', 'dumpsys', 'batteryproperties'],
- infra_step=True)
- self._adb(name='reboot',
- serial=self.serial,
- cmd=['reboot'],
- infra_step=True)
- self._bot_info.run(
- self._bot_info.m.step,
- name='wait for reboot',
- cmd=['sleep', '10'],
- infra_step=True)
- self._adb.wait_for_device()
-
- def read_file_on_device(self, path, *args, **kwargs):
- """Read the given file."""
- return self._adb(name='read %s' % self._bot_info.m.path.basename(path),
- serial=self.serial,
- cmd=['shell', 'cat', path],
- stdout=self._bot_info.m.raw_io.output(),
- infra_step=True).stdout.rstrip()
-
- def remove_file_on_device(self, path, *args, **kwargs):
- """Delete the given file."""
- return self._adb(name='rm %s' % self._bot_info.m.path.basename(path),
- serial=self.serial,
- cmd=['shell', 'rm', '-f', path],
- infra_step=True,
- *args,
- **kwargs)
-
- def get_device_dirs(self):
- """ Set the directories which will be used by the build steps."""
- device_scratch_dir = self._adb(
- name='get EXTERNAL_STORAGE dir',
- serial=self.serial,
- cmd=['shell', 'echo', '$EXTERNAL_STORAGE'],
- )
- prefix = self.device_path_join(device_scratch_dir, 'skiabot', 'skia_')
- return default_flavor.DeviceDirs(
- dm_dir=prefix + 'dm',
- perf_data_dir=prefix + 'perf',
- resource_dir=prefix + 'resources',
- images_dir=prefix + 'images',
- skp_dir=prefix + 'skp/skps',
- tmp_dir=prefix + 'tmp_dir')
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-import ssh_flavor
-
-
-"""Utils for building for and running tests on ChromeOS."""
-
-
-class ChromeOSFlavorUtils(ssh_flavor.SSHFlavorUtils):
- def __init__(self, bot_info):
- super(ChromeOSFlavorUtils, self).__init__(bot_info)
- self.board = self._bot_info.spec['device_cfg']
- self.device_root_dir = '/usr/local/skiabot'
- self.device_bin_dir = self.device_path_join(self.device_root_dir, 'bin')
-
- def step(self, name, cmd, **kwargs):
- """Wrapper for the Step API; runs a step as appropriate for this flavor."""
- local_path = self._bot_info.out_dir.join(
- 'config', 'chromeos-%s' % self.board,
- self._bot_info.configuration, cmd[0])
- remote_path = self.device_path_join(self.device_bin_dir, cmd[0])
- self.copy_file_to_device(local_path, remote_path)
- super(ChromeOSFlavorUtils, self).step(name=name,
- cmd=[remote_path]+cmd[1:],
- **kwargs)
-
- def compile(self, target):
- """Build the given target."""
- cmd = [os.path.join(self._bot_info.skia_dir, 'platform_tools', 'chromeos',
- 'bin', 'chromeos_make'),
- '-d', self.board,
- target]
- self._bot_info.run(cmd)
-
- def install(self):
- """Run any device-specific installation steps."""
- self.create_clean_device_dir(self.device_bin_dir)
-
- def get_device_dirs(self):
- """ Set the directories which will be used by the build steps."""
- prefix = self.device_path_join(self.device_root_dir, 'skia_')
- def join(suffix):
- return ''.join((prefix, suffix))
- return default_flavor.DeviceDirs(
- dm_dir=join('dm_out'), # 'dm' conflicts with the binary
- perf_data_dir=join('perf'),
- resource_dir=join('resources'),
- images_dir=join('images'),
- skp_dir=self.device_path_join(join('skp'), 'skps'),
- tmp_dir=join('tmp_dir'))
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-
-
-"""CMake flavor utils, used for building Skia with CMake."""
-
-
-class CMakeFlavorUtils(default_flavor.DefaultFlavorUtils):
- def compile(self, target):
- """Build Skia with CMake. Ignores `target`."""
- cmake_build = os.path.join(self._bot_info.skia_dir, 'cmake', 'cmake_build')
- self._bot_info.run([cmake_build, target])
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-import subprocess
-import time
-
-
-"""Utils for running coverage tests."""
-
-
-class CoverageFlavorUtils(default_flavor.DefaultFlavorUtils):
- def compile(self, target):
- """Build the given target."""
- cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
- 'llvm_coverage_build'),
- target]
- self._bot_info.run(cmd)
-
- def step(self, cmd, **kwargs):
- """Run the given step through coverage."""
- # Slice out the 'key' and 'properties' arguments to be reused.
- key = []
- properties = []
- current = None
- for i in xrange(0, len(cmd)):
- if isinstance(cmd[i], basestring) and cmd[i] == '--key':
- current = key
- elif isinstance(cmd[i], basestring) and cmd[i] == '--properties':
- current = properties
- elif isinstance(cmd[i], basestring) and cmd[i].startswith('--'):
- current = None
- if current is not None:
- current.append(cmd[i])
-
- results_dir = self._bot_info.out_dir.join('coverage_results')
- self.create_clean_host_dir(results_dir)
-
- # Run DM under coverage.
- report_file_basename = '%s.cov' % self._bot_info.got_revision
- report_file = os.path.join(results_dir, report_file_basename)
- args = [
- 'python',
- os.path.join(self._bot_info.skia_dir, 'tools', 'llvm_coverage_run.py'),
- ] + cmd + ['--outResultsFile', report_file]
- self._bot_info.run(args, **kwargs)
-
- # Generate nanobench-style JSON output from the coverage report.
- git_timestamp = subprocess.check_output(['git', 'log', '-n1',
- self._bot_info.got_revision, '--format=%%ci']).rstrip()
- nanobench_json = results_dir.join('nanobench_%s_%s.json' % (
- self._bot_info.got_revision, git_timestamp))
- line_by_line_basename = ('coverage_by_line_%s_%s.json' % (
- self._bot_info.got_revision, git_timestamp))
- line_by_line = results_dir.join(line_by_line_basename)
- args = [
- 'python',
- os.path.join(self._bot_info.skia_dir, 'tools',
- 'parse_llvm_coverage.py'),
- '--report', report_file, '--nanobench', nanobench_json,
- '--linebyline', line_by_line]
- args.extend(key)
- args.extend(properties)
- self._bot_info.run(args)
-
- # Upload raw coverage data.
- now = time.utcnow()
- gs_json_path = '/'.join((
- str(now.year).zfill(4), str(now.month).zfill(2),
- str(now.day).zfill(2), str(now.hour).zfill(2),
- self._bot_info.name,
- str(self._bot_info.build_number)))
- if self._bot_info.is_trybot:
- gs_json_path = '/'.join(('trybot', gs_json_path,
- str(self._bot_info.issue)))
-
- self._bot_info.gsutil_upload(
- 'upload raw coverage data',
- source=report_file,
- bucket='skia-infra',
- dest='/'.join(('coverage-raw-v1', gs_json_path, report_file_basename)))
-
- # Upload nanobench JSON data.
- gsutil_path = self._bot_info.m.path['depot_tools'].join(
- 'third_party', 'gsutil', 'gsutil')
- upload_args = [self._bot_info.name,
- self._bot_info.m.properties['buildnumber'],
- results_dir,
- self._bot_info.got_revision, gsutil_path]
- if self._bot_info.is_trybot:
- upload_args.append(self._bot_info.m.properties['issue'])
- self._bot_info.run(
- self._bot_info.m.python,
- 'upload nanobench coverage results',
- script=self._bot_info.resource('upload_bench_results.py'),
- args=upload_args,
- cwd=self._bot_info.m.path['checkout'],
- abort_on_failure=False,
- infra_step=True)
-
- # Upload line-by-line coverage data.
- self._bot_info.gsutil_upload(
- 'upload line-by-line coverage data',
- source=line_by_line,
- bucket='skia-infra',
- dest='/'.join(('coverage-json-v1', gs_json_path,
- line_by_line_basename)))
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Default flavor utils class, used for desktop bots."""
-
-
-import os
-import shutil
-import sys
-
-
-class DeviceDirs(object):
- def __init__(self,
- dm_dir,
- perf_data_dir,
- resource_dir,
- images_dir,
- skp_dir,
- tmp_dir):
- self._dm_dir = dm_dir
- self._perf_data_dir = perf_data_dir
- self._resource_dir = resource_dir
- self._images_dir = images_dir
- self._skp_dir = skp_dir
- self._tmp_dir = tmp_dir
-
- @property
- def dm_dir(self):
- """Where DM writes."""
- return self._dm_dir
-
- @property
- def perf_data_dir(self):
- return self._perf_data_dir
-
- @property
- def resource_dir(self):
- return self._resource_dir
-
- @property
- def images_dir(self):
- return self._images_dir
-
- @property
- def skp_dir(self):
- return self._skp_dir
-
- @property
- def tmp_dir(self):
- return self._tmp_dir
-
-
-class DefaultFlavorUtils(object):
- """Utilities to be used by build steps.
-
- The methods in this class define how certain high-level functions should
- work. Each build step flavor should correspond to a subclass of
- DefaultFlavorUtils which may override any of these functions as appropriate
- for that flavor.
-
- For example, the AndroidFlavorUtils will override the functions for
- copying files between the host and Android device, as well as the
- 'step' function, so that commands may be run through ADB.
- """
- def __init__(self, bot_info, *args, **kwargs):
- self._bot_info = bot_info
- self.chrome_path = os.path.join(self._bot_info.build_dir, 'src')
-
- def run(self, cmd, **kwargs):
- """Runs a step as appropriate for this flavor."""
- path_to_app = os.path.join(self._bot_info.out_dir,
- self._bot_info.configuration, cmd[0])
- if ('linux' in sys.platform and
- 'x86_64' in self._bot_info.name and
- not 'TSAN' in self._bot_info.name):
- new_cmd = ['catchsegv', path_to_app]
- else:
- if sys.platform == 'win32':
- path_to_app += '.exe'
- new_cmd = [path_to_app]
- new_cmd.extend(cmd[1:])
- return self._bot_info.run(new_cmd, **kwargs)
-
- def bootstrap_win_toolchain(self):
- """Run bootstrapping script for the Windows toolchain."""
- bootstrap_script = os.path.join(self._bot_info.infrabots_dir,
- 'bootstrap_win_toolchain_json.py')
- win_toolchain_json = os.path.join(
- self._bot_info.build_dir, 'src', 'build', 'win_toolchain.json')
- self._bot_info.run([
- 'python', bootstrap_script,
- '--win_toolchain_json', win_toolchain_json,
- '--depot_tools_parent_dir', self._bot_info.build_dir])
-
- def compile(self, target):
- """Build the given target."""
- # The CHROME_PATH environment variable is needed for bots that use
- # toolchains downloaded by Chrome.
- env = {}
- if sys.platform == 'win32':
- make_cmd = ['python', 'make.py']
- env['CHROME_PATH'] = self.chrome_path
- self._bot_info._run_once(self.bootstrap_win_toolchain)
- else:
- # Ensure depot_tools is in PATH.
- depot_tools = os.path.join('/b', 'depot_tools')
- env['PATH'] = os.pathsep.join([depot_tools, os.environ['PATH']])
- make_cmd = ['make']
- cmd = make_cmd + [target]
- self._bot_info.run(cmd, env=env)
-
- def device_path_join(self, *args):
- """Like os.path.join(), but for paths on a connected device."""
- return os.path.join(*args)
-
- def device_path_exists(self, path):
- """Like os.path.exists(), but for paths on a connected device."""
- return os.path.exists(path, infra_step=True) # pragma: no cover
-
- def copy_directory_contents_to_device(self, host_dir, device_dir):
- """Like shutil.copytree(), but for copying to a connected device."""
- # For "normal" bots who don't have an attached device, we expect
- # host_dir and device_dir to be the same.
- if str(host_dir) != str(device_dir):
- raise ValueError('For bots who do not have attached devices, copying '
- 'from host to device is undefined and only allowed if '
- 'host_path and device_path are the same (%s vs %s).' % (
- str(host_dir), str(device_dir))) # pragma: no cover
-
- def copy_directory_contents_to_host(self, device_dir, host_dir):
- """Like shutil.copytree(), but for copying from a connected device."""
- # For "normal" bots who don't have an attached device, we expect
- # host_dir and device_dir to be the same.
- if str(host_dir) != str(device_dir):
- raise ValueError('For bots who do not have attached devices, copying '
- 'from device to host is undefined and only allowed if '
- 'host_path and device_path are the same (%s vs %s).' % (
- str(host_dir), str(device_dir))) # pragma: no cover
-
- def copy_file_to_device(self, host_path, device_path):
- """Like shutil.copyfile, but for copying to a connected device."""
- # For "normal" bots who don't have an attached device, we expect
- # host_dir and device_dir to be the same.
- if str(host_path) != str(device_path): # pragma: no cover
- raise ValueError('For bots who do not have attached devices, copying '
- 'from host to device is undefined and only allowed if '
- 'host_path and device_path are the same (%s vs %s).' % (
- str(host_path), str(device_path)))
-
- def create_clean_device_dir(self, path):
- """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
- self.create_clean_host_dir(path)
-
- def create_clean_host_dir(self, path):
- """Convenience function for creating a clean directory."""
- if os.path.exists(path):
- shutil.rmtree(path)
- os.makedirs(path)
-
- def install(self):
- """Run device-specific installation steps."""
- pass
-
- def cleanup_steps(self):
- """Run any device-specific cleanup steps."""
- pass
-
- def get_device_dirs(self):
- """ Set the directories which will be used by the build steps.
-
- These refer to paths on the same device where the test executables will
- run, for example, for Android bots these are paths on the Android device
- itself. For desktop bots, these are just local paths.
- """
- join = lambda p: os.path.join(self._bot_info.build_dir, p)
- return DeviceDirs(
- dm_dir=os.path.join(self._bot_info.swarm_out_dir, 'dm'),
- perf_data_dir=self._bot_info.perf_data_dir,
- resource_dir=self._bot_info.resource_dir,
- images_dir=join('images'),
- skp_dir=self._bot_info.local_skp_dir,
- tmp_dir=join('tmp'))
-
- def __repr__(self):
- return '<%s object>' % self.__class__.__name__ # pragma: no cover
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-import subprocess
-
-
-"""iOS flavor utils, used for building for and running tests on iOS."""
-
-
-class iOSFlavorUtils(default_flavor.DefaultFlavorUtils):
- def __init__(self, bot_info):
- super(iOSFlavorUtils, self).__init__(bot_info)
- self.ios_bin = os.path.join(self._bot_info.skia_dir, 'platform_tools',
- 'ios', 'bin')
-
- def step(self, cmd, **kwargs):
- args = [os.path.join(self.ios_bin, 'ios_run_skia')]
-
- # Convert 'dm' and 'nanobench' from positional arguments
- # to flags, which is what iOSShell expects to select which
- # one is being run.
- cmd = ["--" + c if c in ['dm', 'nanobench'] else c
- for c in cmd]
- return self._bot_info.run(args + cmd, **kwargs)
-
- def compile(self, target):
- """Build the given target."""
- cmd = [os.path.join(self.ios_bin, 'ios_ninja')]
- self._bot_info.run(cmd)
-
- def device_path_join(self, *args):
- """Like os.path.join(), but for paths on a connected iOS device."""
- return '/'.join(args)
-
- def device_path_exists(self, path):
- """Like os.path.exists(), but for paths on a connected device."""
- return self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_path_exists'), path],
- ) # pragma: no cover
-
- def _remove_device_dir(self, path):
- """Remove the directory on the device."""
- return self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_rm'), path],
- )
-
- def _create_device_dir(self, path):
- """Create the directory on the device."""
- return self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_mkdir'), path],
- )
-
- def copy_directory_contents_to_device(self, host_dir, device_dir):
- """Like shutil.copytree(), but for copying to a connected device."""
- return self._bot_info.run([
- os.path.join(self.ios_bin, 'ios_push_if_needed'),
- host_dir, device_dir
- ])
-
- def copy_directory_contents_to_host(self, device_dir, host_dir):
- """Like shutil.copytree(), but for copying from a connected device."""
- return self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_pull_if_needed'),
- device_dir, host_dir],
- )
-
- def copy_file_to_device(self, host_path, device_path):
- """Like shutil.copyfile, but for copying to a connected device."""
- self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_push_file'), host_path, device_path],
- ) # pragma: no cover
-
- def create_clean_device_dir(self, path):
- """Like shutil.rmtree() + os.makedirs(), but on a connected device."""
- self._remove_device_dir(path)
- self._create_device_dir(path)
-
- def install(self):
- """Run device-specific installation steps."""
- self._bot_info.run([os.path.join(self.ios_bin, 'ios_install')])
-
- def cleanup_steps(self):
- """Run any device-specific cleanup steps."""
- self._bot_info.run([os.path.join(self.ios_bin, 'ios_restart')])
- self._bot_info.run(['sleep', '20'])
-
- def read_file_on_device(self, path):
- """Read the given file."""
- return subprocess.check_output(
- [os.path.join(self.ios_bin, 'ios_cat_file'), path]).rstrip()
-
- def remove_file_on_device(self, path):
- """Remove the file on the device."""
- return self._bot_info.run(
- [os.path.join(self.ios_bin, 'ios_rm'), path],
- )
-
- def get_device_dirs(self):
- """ Set the directories which will be used by the build steps."""
- prefix = self.device_path_join('skiabot', 'skia_')
- return default_flavor.DeviceDirs(
- dm_dir=prefix + 'dm',
- perf_data_dir=prefix + 'perf',
- resource_dir=prefix + 'resources',
- images_dir=prefix + 'images',
- skp_dir=prefix + 'skp/skps',
- tmp_dir=prefix + 'tmp_dir')
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import collections
-import json
-
-
-DEFAULT_PORT = '22'
-DEFAULT_USER = 'chrome-bot'
-
-
-SlaveInfo = collections.namedtuple('SlaveInfo',
- 'ssh_user ssh_host ssh_port')
-
-SLAVE_INFO = {
- 'skiabot-shuttle-ubuntu12-003':
- SlaveInfo('root', '192.168.1.123', DEFAULT_PORT),
- 'skiabot-shuttle-ubuntu12-004':
- SlaveInfo('root', '192.168.1.134', DEFAULT_PORT),
- 'default':
- SlaveInfo('nouser', 'noip', 'noport'),
-}
-
-
-if __name__ == '__main__':
- print json.dumps(SLAVE_INFO) # pragma: no cover
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-import posixpath
-import subprocess
-import ssh_devices
-
-
-"""Utils for running tests remotely over SSH."""
-
-
-class SSHFlavorUtils(default_flavor.DefaultFlavorUtils):
- def __init__(self, *args, **kwargs):
- super(SSHFlavorUtils, self).__init__(*args, **kwargs)
- slave_info = ssh_devices.SLAVE_INFO.get(self._bot_info.slave_name,
- ssh_devices.SLAVE_INFO['default'])
- self._host = slave_info.ssh_host
- self._port = slave_info.ssh_port
- self._user = slave_info.ssh_user
-
- @property
- def host(self):
- return self._host
-
- @property
- def port(self):
- return self._port
-
- @property
- def user(self):
- return self._user
-
- def ssh(self, cmd, **kwargs):
- """Run the given SSH command."""
- ssh_cmd = ['ssh']
- if self.port:
- ssh_cmd.extend(['-p', self.port])
- dest = self.host
- if self.user:
- dest = self.user + '@' + dest
- ssh_cmd.append(dest)
- ssh_cmd.extend(cmd)
- return self._bot_info.run(ssh_cmd, **kwargs)
-
- def step(self, *args, **kwargs):
- """Run the given step over SSH."""
- self.ssh(*args, **kwargs)
-
- def device_path_join(self, *args):
- """Like os.path.join(), but for paths on a remote machine."""
- return posixpath.join(*args)
-
- def device_path_exists(self, path): # pragma: no cover
- """Like os.path.exists(), but for paths on a remote device."""
- try:
- self.ssh(['test', '-e', path])
- return True
- except subprocess.CalledProcessError:
- return False
-
- def _remove_device_dir(self, path):
- """Remove the directory on the device."""
- self.ssh(['rm', '-rf', path])
-
- def _create_device_dir(self, path):
- """Create the directory on the device."""
- self.ssh(['mkdir', '-p', path])
-
- def create_clean_device_dir(self, path):
- """Like shutil.rmtree() + os.makedirs(), but on a remote device."""
- self._remove_device_dir(path)
- self._create_device_dir(path)
-
- def _make_scp_cmd(self, remote_path, recurse=True):
- """Prepare an SCP command.
-
- Returns a partial SCP command and an adjusted remote path.
- """
- cmd = ['scp']
- if recurse:
- cmd.append('-r')
- if self.port:
- cmd.extend(['-P', self.port])
- adj_remote_path = self.host + ':' + remote_path
- if self.user:
- adj_remote_path = self.user + '@' + adj_remote_path
- return cmd, adj_remote_path
-
- def copy_directory_contents_to_device(self, host_dir, device_dir):
- """Like shutil.copytree(), but for copying to a remote device."""
- _, remote_path = self._make_scp_cmd(device_dir)
- cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
- 'scp_dir_contents.sh'),
- host_dir, remote_path]
- self._bot_info.run(cmd)
-
- def copy_directory_contents_to_host(self, device_dir, host_dir):
- """Like shutil.copytree(), but for copying from a remote device."""
- _, remote_path = self._make_scp_cmd(device_dir)
- cmd = [os.path.join(self._bot_info.skia_dir, 'tools',
- 'scp_dir_contents.sh'),
- remote_path, host_dir]
- self._bot_info.run(cmd)
-
- def copy_file_to_device(self, host_path, device_path):
- """Like shutil.copyfile, but for copying to a connected device."""
- cmd, remote_path = self._make_scp_cmd(device_path, recurse=False)
- cmd.extend([host_path, remote_path])
- self._bot_info.run(cmd)
-
- def read_file_on_device(self, path):
- return self.ssh(['cat', path]).rstrip()
-
- def remove_file_on_device(self, path):
- """Delete the given file."""
- return self.ssh(['rm', '-f', path])
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import default_flavor
-import os
-
-
-"""Utils for running under Valgrind."""
-
-
-class ValgrindFlavorUtils(default_flavor.DefaultFlavorUtils):
- def __init__(self, *args, **kwargs):
- super(ValgrindFlavorUtils, self).__init__(*args, **kwargs)
- self._suppressions_file = os.path.join(self._bot_info.skia_dir,
- 'tools', 'valgrind.supp')
-
- def step(self, name, cmd, **kwargs):
- new_cmd = ['valgrind', '--gen-suppressions=all', '--leak-check=full',
- '--track-origins=yes', '--error-exitcode=1', '--num-callers=40',
- '--suppressions=%s' % self._suppressions_file]
- path_to_app = os.path.join(self._bot_info.out_dir,
- self._bot_info.configuration, cmd[0])
- new_cmd.append(path_to_app)
- new_cmd.extend(cmd[1:])
- return self._bot_info.run(new_cmd, **kwargs)
-
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Utils for running under *SAN"""
-
-
-import default_flavor
-import os
-
-
-class XSanFlavorUtils(default_flavor.DefaultFlavorUtils):
- def __init__(self, *args, **kwargs):
- super(XSanFlavorUtils, self).__init__(*args, **kwargs)
- self._sanitizer = {
- # We'd love to just pass 'address,undefined' and get all the checks, but
- # we're not anywhere close to being able to do that. Instead we start
- # with a set of checks that we know pass or nearly pass. See here for
- # more information:
- # http://clang.llvm.org/docs/UsersManual.html#controlling-code-generation
- 'ASAN': ('address,bool,function,integer-divide-by-zero,nonnull-attribute,'
- 'null,object-size,return,returns-nonnull-attribute,shift,'
- 'signed-integer-overflow,unreachable,vla-bound,vptr'),
- # MSAN and TSAN can't run together with ASAN, so they're their own bots.
- 'MSAN': 'memory',
- 'TSAN': 'thread',
- }[self._bot_info.bot_cfg['extra_config']]
-
- def compile(self, target):
- cmd = [os.path.join(self._bot_info.skia_dir, 'tools', 'xsan_build'),
- self._sanitizer, target]
- self._bot_info.run(cmd)
-
- def step(self, cmd, env=None, **kwargs):
- """Wrapper for the Step API; runs a step as appropriate for this flavor."""
- lsan_suppressions = self._bot_info.skia_dir.join('tools', 'lsan.supp')
- tsan_suppressions = self._bot_info.skia_dir.join('tools', 'tsan.supp')
- ubsan_suppressions = self._bot_info.skia_dir.join('tools', 'ubsan.supp')
- env = dict(env or {})
- env['ASAN_OPTIONS'] = 'symbolize=1 detect_leaks=1'
- env['LSAN_OPTIONS'] = ('symbolize=1 print_suppressions=1 suppressions=%s' %
- lsan_suppressions)
- env['TSAN_OPTIONS'] = 'suppressions=%s' % tsan_suppressions
- env['UBSAN_OPTIONS'] = 'suppressions=%s' % ubsan_suppressions
-
- path_to_app = os.path.join(self._bot_info.out_dir,
- self._bot_info.configuration, cmd[0])
- new_cmd = [path_to_app]
- new_cmd.extend(cmd[1:])
- return self._bot_info.run(new_cmd, env=env, **kwargs)
'variables': {
'files': [
'../../../images/',
+ '../../../tmp/SK_IMAGE_VERSION',
],
},
}
'resources.isolate',
'skps.isolate',
],
- 'variables': {
- 'command': [
- 'python', 'perf_skia.py',
- '--master_name', '<(MASTER_NAME)',
- '--builder_name', '<(BUILDER_NAME)',
- '--build_number', '<(BUILD_NUMBER)',
- '--slave_name', '<(SLAVE_NAME)',
- '--revision', '<(REVISION)',
- '--swarm_out_dir', '${ISOLATED_OUTDIR}',
- '--issue', '<(ISSUE)',
- '--patchset', '<(PATCHSET)',
- ],
- },
}
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import argparse
-import common
-import os
-import sys
-import utils
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--master_name', required=True)
- parser.add_argument('--builder_name', required=True)
- parser.add_argument('--build_number', required=True)
- parser.add_argument('--slave_name', required=True)
- parser.add_argument('--revision', required=True)
- parser.add_argument('--swarm_out_dir', required=True)
- parser.add_argument('--issue')
- parser.add_argument('--patchset')
- args = parser.parse_args()
- with utils.print_timings():
- bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir))
- bot.perf_steps(args.revision, args.master_name, args.slave_name,
- args.build_number, issue=args.issue or None,
- patchset=args.patchset or None)
-
-
-if __name__ == '__main__':
- main()
'variables': {
'files': [
'../../../skps/',
+ '../../../tmp/SKP_VERSION',
],
},
}
'resources.isolate',
'skps.isolate',
],
- 'variables': {
- 'command': [
- 'python', 'test_skia.py',
- '--master_name', '<(MASTER_NAME)',
- '--builder_name', '<(BUILDER_NAME)',
- '--build_number', '<(BUILD_NUMBER)',
- '--slave_name', '<(SLAVE_NAME)',
- '--revision', '<(REVISION)',
- '--swarm_out_dir', '${ISOLATED_OUTDIR}',
- '--issue', '<(ISSUE)',
- '--patchset', '<(PATCHSET)',
- ],
- },
}
+++ /dev/null
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import argparse
-import common
-import os
-import sys
-import utils
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--master_name', required=True)
- parser.add_argument('--builder_name', required=True)
- parser.add_argument('--build_number', required=True)
- parser.add_argument('--slave_name', required=True)
- parser.add_argument('--revision', required=True)
- parser.add_argument('--swarm_out_dir', required=True)
- parser.add_argument('--issue')
- parser.add_argument('--patchset')
- args = parser.parse_args()
- with utils.print_timings():
- bot = common.BotInfo(args.builder_name, os.path.abspath(args.swarm_out_dir))
- bot.test_steps(args.revision, args.master_name, args.slave_name,
- args.build_number, issue=args.issue or None,
- patchset=args.patchset or None)
-
-
-if __name__ == '__main__':
- main()
projects / platform / upstream / libSkiaSharp.git / commitdiff