From d2a42d7df68fa370853949509bb261887aa9b4f6 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 4 Jun 2021 17:02:48 -0700 Subject: [PATCH] Build clr/libs dependencies of test build locally (#53696) * Build clr/libs dependencies of test build locally - Stop relying on the actual product build for these - Should reduce long pole of running coreclr tests by removing need to wait for coreclr/libraries product builds to complete before test build can begin - Evidence shows that the native lib build can also be elided, but that requires more complex build work --- .../common/templates/runtimes/build-test-job.yml | 59 ++++++++++------------ src/tests/Common/override.targets | 2 +- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 50de94e..84cfd8e 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -7,8 +7,7 @@ parameters: testGroup: '' liveRuntimeBuildConfig: '' - # When set to a non-empty value (Debug / Release), it determines libraries - # build configuration to use for the tests. Setting this property implies + # Determines librariesbuild configuration to use for the tests. Setting this property implies # a dependency of this job on the appropriate libraries build and is used # to construct the name of the Azure artifact representing libraries build # to use for building the tests. @@ -47,9 +46,13 @@ jobs: testGroup: ${{ parameters.testGroup }} stagedBuild: ${{ parameters.stagedBuild }} liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - variables: ${{ parameters.variables }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + liveRuntimeBuildParams: ${{ format('clr.corelib+libs.ref+libs.native -rc {0} -c {1} -arch {2} -ci', coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig), parameters.liveLibrariesBuildConfig, parameters.archType) }} + ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64'))), not(eq(parameters.osGroup, 'OSX'))) }}: + compilerArg: '-clang9' + ${{ if not(and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc'), not(and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64'))), not(eq(parameters.osGroup, 'OSX')))) }}: + compilerArg: '' # Test jobs should continue on error for internal builds ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -70,22 +73,31 @@ jobs: ${{ if ne(parameters.dependsOn[0], '') }}: dependsOn: ${{ parameters.dependsOn }} - # TODO: Build of managed test components currently depends on the corresponding build job - # because it needs System.Private.Corelib; we should be able to remove this dependency - # by switching over to using reference assembly. - ${{ if and(ne(parameters.stagedBuild, true), eq(parameters.dependsOn[0], '')) }}: - dependsOn: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig)) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: timeoutInMinutes: 90 ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: timeoutInMinutes: 160 + variables: + - ${{ each variable in parameters.variables }}: + - ${{ variable }} + - name: liveRuntimeBuildParams + value: ${{ format('clr.corelib+libs.ref+libs.native -rc {0} -c {1} -arch {2} -ci', coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig), parameters.liveLibrariesBuildConfig, parameters.archType) }} + - name: compilerArg + value: '' + - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.compilerName, 'gcc')) }}: + - name: compilerArg + value: '-clang9' + # Building for x64 MUSL happens on Alpine Linux and we need to use the stable version available there + - ${{ if and(eq(parameters.osGroup, 'Linux'), eq(parameters.osSubgroup, '_musl'), eq(parameters.archType, 'x64')) }}: + - name: compilerArg + value: '' + # AppleClang has different version scheme, so we let complier introspection pick up the available clang from PATH + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - name: compilerArg + value: '' + steps: # Install test build dependencies @@ -97,24 +109,9 @@ jobs: - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force displayName: Install native dependencies - - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # We need to explictly download CoreCLR, even if building Mono because the CoreCLR tests depend on it - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(coreClrProductRootFolderPath) - artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)' - artifactName: '$(coreClrProductArtifactName)' - displayName: 'CoreCLR product build' + # Build core/libraries dependencies of test build + - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(compilerArg) + displayName: Build coreclr/libs components needed by test build - ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}: - script: | diff --git a/src/tests/Common/override.targets b/src/tests/Common/override.targets index 544d574..c646cf6 100644 --- a/src/tests/Common/override.targets +++ b/src/tests/Common/override.targets @@ -14,7 +14,7 @@ > - + -- 2.7.4