2 # Copyright (c) 2013 The Native Client Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
13 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
14 import pynacl.platform
15 import pynacl.file_tools
20 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
21 NACL_DIR = os.path.dirname(SCRIPT_DIR)
22 TOOLCHAIN_BUILD_DIR = os.path.join(NACL_DIR, 'toolchain_build')
23 TOOLCHAIN_BUILD_OUT_DIR = os.path.join(TOOLCHAIN_BUILD_DIR, 'out')
25 TEMP_PACKAGES_FILE = os.path.join(TOOLCHAIN_BUILD_OUT_DIR, 'packages.txt')
27 BUILD_DIR = os.path.join(NACL_DIR, 'build')
28 PACKAGE_VERSION_DIR = os.path.join(BUILD_DIR, 'package_version')
29 PACKAGE_VERSION_SCRIPT = os.path.join(PACKAGE_VERSION_DIR, 'package_version.py')
31 # As this is a buildbot script, we want verbose logging. Note however, that
32 # toolchain_build has its own log settings, controlled by its CLI flags.
33 logging.getLogger().setLevel(logging.DEBUG)
35 parser = argparse.ArgumentParser(description='PNaCl toolchain buildbot script')
36 group = parser.add_mutually_exclusive_group()
37 group.add_argument('--buildbot', action='store_true',
38 help='Buildbot mode (build and archive the toolchain)')
39 group.add_argument('--trybot', action='store_true',
40 help='Trybot mode (build but do not archove the toolchain)')
41 parser.add_argument('--tests-arch', choices=['x86-32', 'x86-64'],
43 help='Host architecture for tests in buildbot_pnacl.sh')
44 args = parser.parse_args()
46 host_os = buildbot_lib.GetHostPlatform()
48 # This is a minimal context, not useful for running tests yet, but enough for
49 # basic Step handling.
50 context = buildbot_lib.BuildContext()
51 buildbot_lib.SetDefaultContextAttributes(context)
52 context['pnacl'] = True
53 status = buildbot_lib.BuildStatus(context)
55 toolchain_install_dir = os.path.join(
58 '%s_%s' % (host_os, pynacl.platform.GetArch()),
61 def ToolchainBuildCmd(python_executable=None, sync=False, extra_flags=[]):
62 executable = [python_executable] if python_executable else [sys.executable]
63 sync_flag = ['--sync'] if sync else []
65 # The path to the script is a relative path with forward slashes so it is
66 # interpreted properly when it uses __file__ inside cygwin
67 executable_args = ['toolchain_build/toolchain_build_pnacl.py',
68 '--verbose', '--clobber', '--build-64bit-host',
69 '--install', toolchain_install_dir]
72 executable_args.append('--buildbot')
74 executable_args.append('--trybot')
76 # Enabling LLVM assertions have a higher cost on Windows, particularly in the
77 # presence of threads. So disable them on windows but leave them on elsewhere
78 # to get the extra error checking.
79 # See https://code.google.com/p/nativeclient/issues/detail?id=3830
81 executable_args.append('--disable-llvm-assertions')
83 return executable + executable_args + sync_flag + extra_flags
86 # Clean out any installed toolchain parts that were built by previous bot runs.
87 with buildbot_lib.Step('Clobber TC install dir', status):
88 print 'Removing', toolchain_install_dir
89 pynacl.file_tools.RemoveDirectoryIfPresent(toolchain_install_dir)
92 # Run checkdeps so that the PNaCl toolchain trybots catch mistakes that would
93 # cause the normal NaCl bots to fail.
94 with buildbot_lib.Step('checkdeps', status):
98 os.path.join(NACL_DIR, 'tools', 'checkdeps', 'checkdeps.py')])
102 # On windows, sync with Windows git/svn rather than cygwin git/svn
103 with buildbot_lib.Step('Sync toolchain_build sources', status):
104 buildbot_lib.Command(
105 context, ToolchainBuildCmd(sync=True, extra_flags=['--sync-only']))
107 with buildbot_lib.Step('Update cygwin/check bash', status, halt_on_fail=True):
108 # Update cygwin if necessary.
110 if sys.platform == 'cygwin':
111 print 'This script does not support running from inside cygwin!'
113 subprocess.check_call(os.path.join(SCRIPT_DIR, 'cygwin_env.bat'))
114 saved_path = os.environ['PATH']
116 paths = saved_path.split(os.pathsep)
117 # Put path to cygwin tools at the beginning, so cygwin tools like python
118 # and cmake will supercede others (which do not understand cygwin paths)
119 paths = [os.path.join(NACL_DIR, 'cygwin', 'bin')] + paths
121 os.environ['PATH'] = os.pathsep.join(paths)
122 print os.environ['PATH']
123 bash = os.path.join(NACL_DIR, 'cygwin', 'bin', 'bash')
124 cygwin_python = os.path.join(NACL_DIR, 'cygwin', 'bin', 'python')
126 # Assume bash is in the path
130 print 'Bash version:'
132 subprocess.check_call([bash , '--version'])
133 except subprocess.CalledProcessError:
134 print 'Bash not found in path!'
135 raise buildbot_lib.StepFailed()
137 # toolchain_build outputs its own buildbot annotations, so don't use
138 # buildbot_lib.Step to run it here.
139 # Always run with the system python.
140 # TODO(dschuff): remove support for cygwin python once the mingw build is rolled
141 cmd = ToolchainBuildCmd(None,
142 host_os != 'win', # On Windows, we synced already
143 ['--packages-file', TEMP_PACKAGES_FILE])
144 logging.info('Running: ' + ' '.join(cmd))
145 subprocess.check_call(cmd)
147 if args.buildbot or args.trybot:
148 # Don't upload packages from the 32-bit linux bot to avoid racing on
149 # uploading the same packages as the 64-bit linux bot
150 if host_os != 'linux' or pynacl.platform.IsArch64Bit():
152 # Since we are currently running the build in cygwin, the filenames in
153 # TEMP_PACKAGES_FILE will have cygwin paths. Convert them to system
154 # paths so we dont' have to worry about running package_version tools
157 with open(TEMP_PACKAGES_FILE) as f:
160 subprocess.check_output(['cygpath', '-w', line]).strip())
161 with open(TEMP_PACKAGES_FILE, 'w') as f:
162 f.write('\n'.join(converted))
163 packages.UploadPackages(TEMP_PACKAGES_FILE, args.trybot)
167 # Since mac and windows bots don't build target libraries or run tests yet,
168 # Run a basic sanity check that tests the host components (LLVM, binutils,
170 if host_os == 'win' or host_os == 'mac':
171 with buildbot_lib.Step('Test host binaries and gold plugin', status,
173 buildbot_lib.Command(
176 os.path.join('tests', 'gold_plugin', 'gold_plugin_test.py'),
177 '--toolchaindir', toolchain_install_dir])
180 # TODO(dschuff): Fix windows regression test runner (upstream in the LLVM
181 # codebase or locally in the way we build LLVM) ASAP
182 with buildbot_lib.Step('LLVM Regression', status,
184 llvm_test = [sys.executable,
185 os.path.join(NACL_DIR, 'pnacl', 'scripts', 'llvm-test.py'),
188 buildbot_lib.Command(context, llvm_test)
191 # On Linux we build all toolchain components (driven from this script), and then
192 # call buildbot_pnacl.sh which builds the sandboxed translator and runs tests
193 # for all the components.
194 # For now, we only build the host toolchain components (binutils, llvm, driver)
195 # but no target ibraries on targets other than Linux, so we can't run the SCons
196 # tests (other than the gold_plugin_test) on those platforms yet.
197 # For now full test coverage is only achieved on the main waterfall bots.
198 # TODO(dschuff): enable building (but not uploading) or downloading of the
199 # target libraries on non-linux so we can run more tests on the toolchain
201 if host_os != 'linux':
204 # Now we run the PNaCl buildbot script. It in turn runs the PNaCl build.sh
205 # script (currently only for the sandboxed translator) and runs scons tests.
206 # TODO(dschuff): re-implement the test-running portion of buildbot_pnacl.sh
207 # using buildbot_lib, and use them here and in the non-toolchain builder.
208 buildbot_shell = os.path.join(NACL_DIR, 'buildbot', 'buildbot_pnacl.sh')
210 # Generate flags for buildbot_pnacl.sh
212 arch = 'x8664' if args.tests_arch == 'x86-64' else 'x8632'
215 trybot_mode = 'false'
219 platform_arg = 'mode-buildbot-tc-' + arch + '-linux'
221 command = [bash, buildbot_shell, platform_arg, trybot_mode]
222 logging.info('Running: ' + ' '.join(command))
223 subprocess.check_call(command)