From ef7afc8e86eb1d8407a1c07ae735158efc5aca75 Mon Sep 17 00:00:00 2001 From: Egor Chesakov Date: Thu, 13 Dec 2018 20:05:38 -0800 Subject: [PATCH] Running and visualizing tests in AzureDevOps (Part 1) (#21502) * Enable Azure Pipelines reporter so the test results can be seen at one place (i.e. "Tests" tab) - TestRunNamePrefix should encode all the needed information for a test run to be identifiable. * Scenarios property should use comma (not semi-colon) to work correctly on non-Windows. * Specify timeout to test jobs (the numbers are taken from netci.groovy) - we would adjust them as soon as we get this running in each PR. * Specify timeout for Helix work items (the number is taken from buildpipeline) - same as above. * Specify helixType and helixRepo properties to the job and use _HelixType and _HelixSource variables in MSBuild project instead of defining them manually. * Limit Pri0 Checked test jobs to running against pull requests. This also should be limited in the future to "default" build definition name - so when we have PR workload selection feature working we can distinguish between default PR validation and user triggered jobs. * Remove "Prepare tests for helix" step - instead of archiving Core_Root and tests folders ourselves - directly specify them as correlation payload and work item payload directories and let the archiving be handled by Helix Sdk. --- azure-pipelines.yml | 28 +++++++++++++++------- eng/build-job.yml | 12 ++++------ eng/platform-matrix.yml | 15 +++++++----- eng/test-job.yml | 38 ++++++++++++++++-------------- eng/xplat-job.yml | 29 ++++++++++++----------- tests/helixpublishwitharcade.proj | 49 ++++++++++++++++++++++++--------------- 6 files changed, 100 insertions(+), 71 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 92de971..8957e23 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -110,27 +110,28 @@ jobs: # Checked test builds # -# Pri0 (CI) -- ${{ if eq(variables['System.TeamProject'], 'public') }}: +# Pri0 (PullRequest) +- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: priority: 0 + timeoutInMinutes: 120 # Pri1 (CI) -- ${{ if eq(variables['System.TeamProject'], 'public') }}: +- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml buildConfig: checked jobParameters: priority: 1 - scenarios: 'normal;jitstress2' + timeoutInMinutes: 240 # Pri1 crossgen (CI) -- ${{ if eq(variables['System.TeamProject'], 'public') }}: +- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml @@ -138,14 +139,25 @@ jobs: jobParameters: priority: 1 crossgen: true - scenarios: 'normal;jitstress2' + timeoutInMinutes: 240 + +# Pri1 (Manual) +- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'Manual')) }}: + - template: eng/platform-matrix.yml + parameters: + jobTemplate: test-job.yml + buildConfig: checked + jobParameters: + priority: 1 + scenarios: 'normal,jitstress1,jitstress2,gcstress0x3,gcstress0xc' + timeoutInMinutes: 300 # # Release test builds (Official Build) # # Pri1 -- ${{ if ne(variables['System.TeamProject'], 'public') }}: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml @@ -154,7 +166,7 @@ jobs: priority: 1 # Pri1 crossgen (Official Build) -- ${{ if eq(variables['System.TeamProject'], 'public') }}: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/platform-matrix.yml parameters: jobTemplate: test-job.yml diff --git a/eng/build-job.yml b/eng/build-job.yml index 3a5425f..98710b4 100644 --- a/eng/build-job.yml +++ b/eng/build-job.yml @@ -4,6 +4,7 @@ parameters: osGroup: '' osGroupName: '' containerName: '' + crossrootfsDir: '' ### Product build jobs: @@ -21,6 +22,7 @@ jobs: # Run all steps in the container. # Note that the containers are resources defined in azure-pipelines.yml containerName: ${{ parameters.containerName }} + crossrootfsDir: ${{ parameters.crossrootfsDir }} steps: @@ -36,14 +38,8 @@ jobs: displayName: Install native dependencies # Build - - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}: - - script: ./build.sh $(buildConfig) $(archType) -skiptests - displayName: Build product - - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}: - - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests - displayName: Build product - - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}: - - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skiptests + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests displayName: Build product - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters? diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml index 6d84bbb..647bd9c 100644 --- a/eng/platform-matrix.yml +++ b/eng/platform-matrix.yml @@ -17,6 +17,7 @@ jobs: osGroupName: Linux containerName: ubuntu_1404_arm_cross_build_image helixQueuesPublic: 'Ubuntu.1404.Arm32.Open' + crossrootfsDir: '/crossrootfs/arm' # Currently we don't have Linux/arm32 internal Helix queues ${{ insert }}: ${{ parameters.jobParameters }} @@ -28,8 +29,11 @@ jobs: osGroup: Linux osGroupName: Linux containerName: ubuntu_1604_arm64_cross_build_image - helixQueuesPublic: 'Debian.9.Arm64.Open,Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open' - helixQueuesInternal: 'Debian.9.Arm64,Ubuntu.1604.Arm64,Ubuntu.1804.Arm64' + # TODO: enable Debian.9.Arm64.Open and Debian.9.Arm64 queues + # when https://github.com/dotnet/core-eng/issues/4805 is resolved + helixQueuesPublic: 'Ubuntu.1604.Arm64.Open,Ubuntu.1804.Arm64.Open' + helixQueuesInternal: 'Ubuntu.1604.Arm64,Ubuntu.1804.Arm64' + crossrootfsDir: '/crossrootfs/arm64' ${{ insert }}: ${{ parameters.jobParameters }} # Linux musl @@ -40,8 +44,7 @@ jobs: osGroup: Linux osGroupName: Linux_musl containerName: musl_x64_build_image - helixQueuesPublic: 'Alpine.3.Arm64.Open' - helixQueuesInternal: 'Alpine.3.Arm64' + # TODO: add Alpine.Amd64 queues ${{ insert }}: ${{ parameters.jobParameters }} # RHEL 6 @@ -77,8 +80,8 @@ jobs: osGroup: Linux osGroupName: Linux containerName: ubuntu_1604_x64_build_image - helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Arm64.Open' - helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Arm64' + helixQueuesPublic: 'Debian.8.Amd64.Open,Ubuntu.1604.Amd64.Open,Ubuntu.1804.Amd64.Open' + helixQueuesInternal: 'Debian.8.Amd64,Ubuntu.1604.Amd64,Ubuntu.1804.Amd64' ${{ insert }}: ${{ parameters.jobParameters }} # macOS x64 diff --git a/eng/test-job.yml b/eng/test-job.yml index 72b37f6..218f8ab 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -8,6 +8,8 @@ parameters: scenarios: '' helixQueuesPublic: '' helixQueuesInternal: '' + timeoutInMinutes: '' + crossrootfsDir: '' ### Test job @@ -25,11 +27,18 @@ jobs: # Compute job name from template parameters ${{ if eq(parameters.crossgen, 'false') }}: name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test Pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.crossgen, 'true') }}: name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }} + ${{ if eq(parameters.crossgen, 'false') }}: + helixType: ${{ format('test/functional/cli/pri{0}', parameters.priority) }} + ${{ if eq(parameters.crossgen, 'true') }}: + helixType: ${{ format('test/functional/r2r/cli/pri{0}', parameters.priority) }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + variables: # Map template parameters to command line arguments - ${{ if eq(parameters.priority, '1') }}: @@ -61,6 +70,8 @@ jobs: - ${{ if ne(variables['System.TeamProject'], 'public') }}: - name: helixTargetQueuesArg value: ${{ format('/p:HelixTargetQueues=\"{0}\"', parameters.helixQueuesInternal) }} + - name: commonMSBuildArgs + value: ${{ format('/maxcpucount /p:__BuildOS={0} /p:__BuildArch={1} /p:__BuildType={2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }} # TODO: Enable crossgen in build-test.sh. It currently doesn't # accept a crossgen arg, so disable the macos/linux crossgen test @@ -75,6 +86,8 @@ jobs: # Note that the containers are resources defined in azure-pipelines.yml containerName: ${{ parameters.containerName }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + steps: # Install test build dependencies @@ -99,29 +112,18 @@ jobs: # Build tests - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: - # TODO: enable crossgen in build-test.sh - - script: ./build-test.sh $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) + # TODO: enable crossgen in build-test.sh + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) displayName: Build tests - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) displayName: Build tests - # Prepare tests for helix - - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: - - script: ./Tools/dotnetcli/dotnet msbuild tests/helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper)/Tests/Core_Root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true - # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj - displayName: Prepare test archives for Helix - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - # TODO: remove UsePython argument once we've removed generated wrappers in helixprep.proj - - script: .\Tools\dotnetcli\dotnet.exe msbuild tests\helixprep.proj /p:CORE_ROOT=$(Build.SourcesDirectory)\bin\tests\Windows_NT.$(archType).$(buildConfigUpper)\tests\core_root /p:__BuildType=$(buildConfig) /p:__BuildArch=$(archType) /p:UsePython=true - displayName: Prepare test archives for Helix - - # Send tests to helix - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}: - - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg) + - script: ./Tools/dotnetcli/dotnet msbuild tests/helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg) displayName: Send test jobs to Helix env: ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -130,8 +132,9 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'public') }}: # Access token variable for public project HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj /maxcpucount $(scenariosArg) $(helixTargetQueuesArg) + - script: .\Tools\dotnetcli\dotnet msbuild tests\helixpublishwitharcade.proj $(commonMSBuildArgs) $(scenariosArg) $(helixTargetQueuesArg) displayName: Send test jobs to Helix env: ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -140,3 +143,4 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'public') }}: # Access token variable for public project HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token) + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml index d9d45d1..db8cf3b 100644 --- a/eng/xplat-job.yml +++ b/eng/xplat-job.yml @@ -8,6 +8,9 @@ parameters: condition: '' dependsOn: '' containerName: '' + timeoutInMinutes: '' + helixType: '' + crossrootfsDir: '' variables: {} ## any extra variables to add to the defaults defined below jobs: @@ -18,8 +21,11 @@ jobs: displayName: ${{ parameters.displayName }} condition: ${{ parameters.condition }} - dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + helixRepo: 'dotnet/coreclr' + helixType: ${{ parameters.helixType }} pool: ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}: @@ -34,7 +40,6 @@ jobs: name: dotnet-internal-temp ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}: name: dotnet-external-temp - timeoutInMinutes: 240 workspace: clean: all @@ -74,17 +79,15 @@ jobs: - name: osGroupName value: ${{ parameters.osGroupName }} - # Crossbuild specific variables. - - ${{ if eq(parameters.archType, 'arm') }}: - - name: rootfsDir - value: /crossrootfs/arm - - - name: cacRootfsDir - value: /crossrootfs/x86 - - - ${{ if eq(parameters.archType, 'arm64') }}: - - name: rootfsDir - value: /crossrootfs/arm64 + - ${{ if ne(parameters.crossrootfsDir, '') }}: + - name: crossArg + value: 'cross' + # This is only required for cross builds. + - name: ROOTFS_DIR + value: ${{ parameters.crossrootfsDir }} + - ${{ if eq(parameters.crossrootfsDir, '') }}: + - name: crossArg + value: '' - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/tests/helixpublishwitharcade.proj b/tests/helixpublishwitharcade.proj index d479969..df0724a 100644 --- a/tests/helixpublishwitharcade.proj +++ b/tests/helixpublishwitharcade.proj @@ -7,25 +7,38 @@ - - pr/coreclr/master - test/stuff + $(_HelixSource) + $(_HelixType) $(BUILD_BUILDNUMBER) - true + true true - $(MSBuildProjectDirectory)/.. + 1000 + $(TestWorkingDir)\Tests\Core_Root + + + + %HELIX_PYTHONPATH% + %HELIX_CORRELATION_PAYLOAD%\runtest_helix.py + + + + $HELIX_PYTHONPATH + $HELIX_CORRELATION_PAYLOAD/runtest_helix.py - - %(Identity) - + + + + + + - + Scenario=%(Scenarios.Identity) @@ -39,8 +52,7 @@ - + - - $(Scenario) + $(BuildOS) $(BuildArch) $(BuildType) @ + $(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ - - - %(Identity) - python runtest_helix.py -wrapper %(FileName).dll - python runtest_helix.py -scenario $(Scenario) -wrapper %(FileName).dll + + %(RootDir)%(Directory) + $(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension) + $(HelixPythonPath) $(RunnerScript) -wrapper %(FileName)%(Extension) -scenario $(Scenario) + $([System.TimeSpan]::FromSeconds($(TimeoutInSeconds))) - -- 2.7.4