From: Tomáš Rylek Date: Fri, 23 Aug 2019 23:24:35 +0000 (+0200) Subject: Revert "Split test build and run into two jobs (dotnet/coreclr#26191)" (dotnet/corecl... X-Git-Tag: submit/tizen/20210909.063632~11030^2~737 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=c77966579fd06ab101cf33a6c0f4c0a214734278;p=platform%2Fupstream%2Fdotnet%2Fruntime.git Revert "Split test build and run into two jobs (dotnet/coreclr#26191)" (dotnet/coreclr#26348) This reverts commit dotnet/coreclr@2c01bb82f77e88e58c777d2e1995bb32dd0d8777. Commit migrated from https://github.com/dotnet/coreclr/commit/db9f5bb91455225ab448e263e940c637dfa83c64 --- diff --git a/eng/build-test-job.yml b/eng/build-test-job.yml deleted file mode 100644 index bbb48ab..0000000 --- a/eng/build-test-job.yml +++ /dev/null @@ -1,164 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osIdentifier: '' - container: '' - testGroup: '' - readyToRun: false - crossrootfsDir: '' - # If true, run the corefx tests instead of the coreclr ones - corefxTests: false - displayNameArgs: '' - condition: true - -### Test build job - -### Each test build job depends on a corresponding build job with the same -### buildConfig and archType. - -jobs: -- template: xplat-test-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osIdentifier: ${{ parameters.osIdentifier }} - readyToRun: ${{ parameters.readyToRun }} - - # Test jobs should continue on error for internal builds - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - continueOnError: true - - # Compute job name from template parameters - ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: - name: ${{ format('build_test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - - ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: - name: ${{ format('build_test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - - ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: - name: ${{ format('build_test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: - name: ${{ format('build_test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Build Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - - crossrootfsDir: ${{ parameters.crossrootfsDir }} - - variables: - # Map template parameters to command line arguments - - name: priorityArg - value: '' - - ${{ if ne(parameters.testGroup, 'innerloop') }}: - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - name: priorityArg - value: 'priority1' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - name: priorityArg - value: '-priority=1' - - - name: crossgenArg - value: '' - - ${{ if eq(parameters.readyToRun, true) }}: - - name: crossgenArg - value: 'crossgen' - - - name: clangArg - value: '' - # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly. - - ${{ if eq(parameters.osGroup, 'FreeBSD') }}: - - name: clangArg - value: '-clang6.0' - - ${{ if eq(parameters.archType, 'arm64') }}: - - name: clangArg - value: '-clang5.0' - - - name: testhostArg - value: '' - - ${{ if eq(parameters.corefxTests, true) }}: - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - name: testhostArg - value: 'generatetesthostonly' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - name: testhostArg - value: 'buildtesthostonly' - - # Since the condition is being altered, merge the default with the additional conditions. - # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions - condition: and(succeeded(), ${{ parameters.condition }}) - - # Test job depends on the corresponding build job - dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: 90 - - steps: - - # Install test build dependencies - - ${{ if eq(parameters.osGroup, 'OSX') }}: - - script: sh eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies - - - # Download product binaries directory - - task: DownloadBuildArtifacts@0 - displayName: Download product build - inputs: - buildType: current - downloadType: single - artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - downloadPath: $(System.ArtifactsDirectory) - - - # Populate Product directory - - task: CopyFiles@2 - displayName: Populate Product directory - inputs: - sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} - contents: '**' - targetFolder: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) - - - # Build tests - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) $(clangArg) $(testhostArg) ci - displayName: Build tests - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) $(testhostArg) ci - displayName: Build tests - - - # Zip and publish tests - - template: /eng/upload-artifact-step.yml - parameters: - rootFolder: $(testRootFolderPath) - includeRootFolder: false - archiveFile: $(testRootFolderPath)$(archiveExtension) - archiveType: $(archiveType) - tarCompression: $(tarCompression) - artifactName: $(testArtifactName) - - - # Publish Logs - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - pathtoPublish: $(Build.SourcesDirectory)/bin/Logs - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - artifactName: ${{ format('TestBuildLogs_r2r_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} - ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - artifactName: ${{ format('TestBuildLogs_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - artifactName: ${{ format('TestBuildLogs_r2r_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} - ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - artifactName: ${{ format('TestBuildLogs_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} - continueOnError: true - condition: always() diff --git a/eng/download-artifact-step.yml b/eng/download-artifact-step.yml deleted file mode 100644 index a0d1027..0000000 --- a/eng/download-artifact-step.yml +++ /dev/null @@ -1,22 +0,0 @@ -parameters: - downloadFolder: '' - unpackFolder: '' - artifactFileName: '' - artifactName: '' - -steps: - # Download artifact - - task: DownloadBuildArtifacts@0 - displayName: Download artifacts - inputs: - buildType: current - downloadType: single - downloadPath: '${{ parameters.downloadFolder }}' - artifactName: '${{ parameters.artifactName }}' - - # Unzip artifact - - task: ExtractFiles@1 - displayName: Unzip test artifacts - inputs: - archiveFilePatterns: ${{ parameters.downloadFolder }}/${{ parameters.artifactName }}/${{ parameters.artifactFileName }} - destinationFolder: ${{ parameters.unpackFolder }} diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index a970d98..01a9e3d 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -56,6 +56,7 @@ jobs: helixQueueGroup: ci jobParameters: testGroup: outerloop + publishTestArtifacts: true # # ReadyToRun test jobs @@ -80,6 +81,7 @@ jobs: testGroup: outerloop readyToRun: true displayNameArgs: R2R + publishTestArtifacts: true # # Crossgen-comparison jobs diff --git a/eng/pipelines/coreclr/pr.yml b/eng/pipelines/coreclr/pr.yml index 1785e93..fe35ebd 100644 --- a/eng/pipelines/coreclr/pr.yml +++ b/eng/pipelines/coreclr/pr.yml @@ -71,6 +71,7 @@ jobs: helixQueueGroup: pr jobParameters: testGroup: innerloop + publishTestArtifacts: true # # ReadyToRun test jobs @@ -89,6 +90,7 @@ jobs: testGroup: innerloop readyToRun: true displayNameArgs: R2R + publishTestArtifacts: true # # CoreFX test runs against CoreCLR diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml deleted file mode 100644 index 56476ac..0000000 --- a/eng/run-test-job.yml +++ /dev/null @@ -1,255 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osIdentifier: '' - container: '' - testGroup: '' - readyToRun: false - helixQueues: '' - # If true, run the corefx tests instead of the coreclr ones - corefxTests: false - displayNameArgs: '' - runInUnloadableContext: false - -### Test run job - -### Each test run job depends on a corresponding test build job with the same -### buildConfig and archType. - -jobs: -- template: xplat-test-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osIdentifier: ${{ parameters.osIdentifier }} - readyToRun: ${{ parameters.readyToRun }} - helixType: 'build/tests/' - - # Test jobs should continue on error for internal builds - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - continueOnError: true - - # Compute job name from template parameters - ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: - name: 'run_test_p0_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - dependsOn: 'build_test_p0_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - displayName: 'Run Test Pri0 ${{ parameters.osIdentifier }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' - - ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: - name: 'run_test_p1_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - dependsOn: 'build_test_p1_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - displayName: 'Run Test Pri1 ${{ parameters.osIdentifier }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' - - ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: - name: 'run_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - dependsOn: 'build_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - displayName: 'Run Test Pri0 ${{ parameters.displayNameArgs }} ${{ parameters.osIdentifier }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' - - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: - name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - dependsOn: 'build_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osIdentifier }}_${{ parameters.archType }}_${{parameters.buildConfig }}' - displayName: 'Run Test Pri1 ${{ parameters.displayNameArgs }} ${{ parameters.osIdentifier }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' - - variables: - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNet-HelixApi-Access - - # TODO: update these numbers as they were determined long ago - ${{ if eq(parameters.testGroup, 'innerloop') }}: - timeoutInMinutes: 150 - ${{ if in(parameters.testGroup, 'outerloop') }}: - timeoutInMinutes: 270 - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}: - timeoutInMinutes: 390 - ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}: - timeoutInMinutes: 510 - - steps: - - # Download and unzip tests - - template: /eng/download-artifact-step.yml - parameters: - downloadFolder: $(binTestsPath) - unpackFolder: $(testRootFolderPath) - artifactFileName: $(osGroup).$(archType).$(buildConfigUpper)$(archiveExtension) - artifactName: $(testArtifactName) - - - # Send tests to Helix - - template: /eng/send-to-helix-step.yml - parameters: - displayName: Send tests to Helix - buildConfig: $(buildConfigUpper) - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - creator: $(Build.DefinitionName) - - helixBuild: $(Build.BuildNumber) - helixSource: $(_HelixSource) - - # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it, - # but don't add it to new ones. - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - helixType: 'test/functional/r2r_corefx/' - ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - helixType: 'test/functional/corefx/' - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - helixType: 'test/functional/r2r/cli/' - ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - helixType: 'test/functional/cli/' - - helixQueues: ${{ parameters.helixQueues }} - - # This tests whether an array is empty - ${{ if eq(join('', parameters.helixQueues), '') }}: - condition: false - - publishTestResults: true - - # Set job timeouts - # - # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. - # In case xUnit test wrappers get refactored this number should also be adjusted. - # - # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting - # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the - # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. - - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: - timeoutPerTestCollectionInMinutes: 120 - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: - timeoutPerTestCollectionInMinutes: 30 - timeoutPerTestInMinutes: 10 - ${{ if in(parameters.testGroup, 'outerloop') }}: - timeoutPerTestCollectionInMinutes: 120 - timeoutPerTestInMinutes: 10 - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}: - timeoutPerTestCollectionInMinutes: 120 - timeoutPerTestInMinutes: 30 - ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: - timeoutPerTestCollectionInMinutes: 240 - timeoutPerTestInMinutes: 60 - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}: - timeoutPerTestCollectionInMinutes: 300 - timeoutPerTestInMinutes: 90 - - runCrossGen: ${{ parameters.readyToRun }} - runInUnloadableContext: ${{ parameters.runInUnloadableContext }} - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - # Access token variable for internal project from the - # DotNet-HelixApi-Access variable group - helixAccessToken: $(HelixApiAccessToken) - - # Choose which tests to send to Helix: CoreFX or CoreCLR. - ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: 'eng/helixcorefxtests.proj' - ${{ if ne(parameters.corefxTests, true) }}: - helixProjectArguments: 'tests/helixpublishwitharcade.proj' - - ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: - scenarios: - - normal - - no_tiered_compilation - ${{ if in(parameters.testGroup, 'jitstress') }}: - scenarios: - - jitminopts - - jitstress1 - - jitstress1_tiered - - jitstress2 - - jitstress2_tiered - - zapdisable - - tailcallstress - ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}: - scenarios: - - jitstress_isas_incompletehwintrinsic - - jitstress_isas_nohwintrinsic - - jitstress_isas_nohwintrinsic_nosimd - - jitstress_isas_nosimd - ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}: - scenarios: - - jitstress_isas_incompletehwintrinsic - - jitstress_isas_nohwintrinsic - - jitstress_isas_nohwintrinsic_nosimd - - jitstress_isas_nosimd - - jitstress_isas_x86_noaes - - jitstress_isas_x86_noavx - - jitstress_isas_x86_noavx2 - - jitstress_isas_x86_nobmi1 - - jitstress_isas_x86_nobmi2 - - jitstress_isas_x86_nofma - - jitstress_isas_x86_nohwintrinsic - - jitstress_isas_x86_nolzcnt - - jitstress_isas_x86_nopclmulqdq - - jitstress_isas_x86_nopopcnt - - jitstress_isas_x86_nosse - - jitstress_isas_x86_nosse2 - - jitstress_isas_x86_nosse3 - - jitstress_isas_x86_nosse3_4 - - jitstress_isas_x86_nosse41 - - jitstress_isas_x86_nosse42 - - jitstress_isas_x86_nossse3 - ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}: - scenarios: - - jitstressregs1_x86_noavx - - jitstressregs2_x86_noavx - - jitstressregs3_x86_noavx - - jitstressregs4_x86_noavx - - jitstressregs8_x86_noavx - - jitstressregs0x10_x86_noavx - - jitstressregs0x80_x86_noavx - - jitstressregs0x1000_x86_noavx - ${{ if in(parameters.testGroup, 'jitstressregs' ) }}: - scenarios: - - jitstressregs1 - - jitstressregs2 - - jitstressregs3 - - jitstressregs4 - - jitstressregs8 - - jitstressregs0x10 - - jitstressregs0x80 - - jitstressregs0x1000 - ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}: - scenarios: - - jitstress2_jitstressregs1 - - jitstress2_jitstressregs2 - - jitstress2_jitstressregs3 - - jitstress2_jitstressregs4 - - jitstress2_jitstressregs8 - - jitstress2_jitstressregs0x10 - - jitstress2_jitstressregs0x80 - - jitstress2_jitstressregs0x1000 - ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: - scenarios: - - gcstress0x3 - - gcstress0xc - ${{ if in(parameters.testGroup, 'gcstress-extra') }}: - scenarios: - - heapverify1 - - gcstress0xc_zapdisable - - gcstress0xc_zapdisable_jitstress2 - - gcstress0xc_zapdisable_heapverify1 - - gcstress0xc_jitstress1 - - gcstress0xc_jitstress2 - - gcstress0xc_jitminopts_heapverify1 - ${{ if in(parameters.testGroup, 'r2r-extra') }}: - scenarios: - - jitstress1 - - jitstress2 - - jitstress1_tiered - - jitstress2_tiered - - jitstressregs1 - - jitstressregs2 - - jitstressregs3 - - jitstressregs4 - - jitstressregs8 - - jitstressregs0x10 - - jitstressregs0x80 - - jitstressregs0x1000 - - jitminopts - - forcerelocs - - gcstress15 diff --git a/eng/test-job.yml b/eng/test-job.yml index 6fb9661..5bb3667 100644 --- a/eng/test-job.yml +++ b/eng/test-job.yml @@ -12,6 +12,7 @@ parameters: corefxTests: false displayNameArgs: '' runInUnloadableContext: false + publishTestArtifacts: false condition: true ### Test job @@ -20,30 +21,356 @@ parameters: ### buildConfig and archType. jobs: -- template: build-test-job.yml +- template: xplat-job.yml parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osIdentifier: ${{ parameters.osIdentifier }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - readyToRun: ${{ parameters.readyToRun }} - crossrootfsDir: ${{ parameters.crossrootfsDir }} - corefxTests: ${{ parameters.coreFxTests }} - displayNameArgs: ${{ parameters.displayNameArgs }} - condition: ${{ parameters.condition }} - -- template: run-test-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osIdentifier: ${{ parameters.osIdentifier }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - readyToRun: ${{ parameters.readyToRun }} - helixQueues: ${{ parameters.helixQueues }} - corefxTests: ${{ parameters.coreFxTests }} - displayNameArgs: ${{ parameters.displayNameArgs }} - runInUnloadableContext: ${{ parameters.runInUnloadableContext }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osIdentifier: ${{ parameters.osIdentifier }} + helixType: 'build/tests/' + + # Test jobs should continue on error for internal builds + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + continueOnError: true + + # Compute job name from template parameters + ${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}', 'p0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri0', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(eq(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri0', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.displayNameArgs, '')) }}: + name: ${{ format('test_{0}_{1}_{2}_{3}_{4}', 'p1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test {0} {1} {2} {3} {4}', 'Pri1', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + + variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-HelixApi-Access + + # Map template parameters to command line arguments + - name: priorityArg + value: '' + - ${{ if ne(parameters.testGroup, 'innerloop') }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - name: priorityArg + value: 'priority1' + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - name: priorityArg + value: '-priority=1' + + - name: crossgenArg + value: '' + - ${{ if eq(parameters.readyToRun, true) }}: + - name: crossgenArg + value: 'crossgen' + + - name: clangArg + value: '' + # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly. + - ${{ if eq(parameters.osGroup, 'FreeBSD') }}: + - name: clangArg + value: '-clang6.0' + - ${{ if eq(parameters.archType, 'arm64') }}: + - name: clangArg + value: '-clang5.0' + + - name: testhostArg + value: '' + - ${{ if eq(parameters.corefxTests, true) }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - name: testhostArg + value: 'generatetesthostonly' + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - name: testhostArg + value: 'buildtesthostonly' + + # Since the condition is being altered, merge the default with the additional conditions. + # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions + condition: and(succeeded(), ${{ parameters.condition }}) + + # Test job depends on the corresponding build job + dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + + # Run all steps in the container. + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} + + # "Total" means building tests, waiting for a queue in Helix to become available, and running the tests. + # In case test-job.yml gets split into two separate jobs (e.g. build-test-job.yml and run-test.yml) + # this number should be adjusted accordingly. + ${{ if eq(parameters.testGroup, 'innerloop') }}: + timeoutInMinutes: 240 + ${{ if in(parameters.testGroup, 'outerloop') }}: + timeoutInMinutes: 360 + ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}: + timeoutInMinutes: 480 + ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}: + timeoutInMinutes: 600 + + steps: + + # Install test build dependencies + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - script: sh eng/install-native-dependencies.sh $(osGroup) + displayName: Install native dependencies + + + # Download product binaries directory + - task: DownloadBuildArtifacts@0 + displayName: Download product build + inputs: + buildType: current + downloadType: single + artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + downloadPath: $(System.ArtifactsDirectory) + + + # Populate Product directory + - task: CopyFiles@2 + displayName: Populate Product directory + inputs: + sourceFolder: $(System.ArtifactsDirectory)/${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }} + contents: '**' + targetFolder: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper) + + + # Build tests + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) $(clangArg) $(testhostArg) ci + displayName: Build tests + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) $(testhostArg) ci + displayName: Build tests + + + # Send tests to Helix + - template: /eng/send-to-helix-step.yml + parameters: + displayName: Send tests to Helix + buildConfig: $(buildConfigUpper) + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + + ${{ if eq(variables['System.TeamProject'], 'public') }}: + creator: $(Build.DefinitionName) + + helixBuild: $(Build.BuildNumber) + helixSource: $(_HelixSource) + + # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it, + # but don't add it to new ones. + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + helixType: 'test/functional/r2r_corefx/' + ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + helixType: 'test/functional/corefx/' + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + helixType: 'test/functional/r2r/cli/' + ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + helixType: 'test/functional/cli/' + + helixQueues: ${{ parameters.helixQueues }} + + # This tests whether an array is empty + ${{ if eq(join('', parameters.helixQueues), '') }}: + condition: false + + publishTestResults: true + + # Set job timeouts + # + # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. + # In case xUnit test wrappers get refactored this number should also be adjusted. + # + # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting + # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the + # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. + + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: + timeoutPerTestCollectionInMinutes: 120 + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: + timeoutPerTestCollectionInMinutes: 30 + timeoutPerTestInMinutes: 10 + ${{ if in(parameters.testGroup, 'outerloop') }}: + timeoutPerTestCollectionInMinutes: 120 + timeoutPerTestInMinutes: 10 + ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}: + timeoutPerTestCollectionInMinutes: 120 + timeoutPerTestInMinutes: 30 + ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: + timeoutPerTestCollectionInMinutes: 240 + timeoutPerTestInMinutes: 60 + ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}: + timeoutPerTestCollectionInMinutes: 300 + timeoutPerTestInMinutes: 90 + + runCrossGen: ${{ parameters.readyToRun }} + runInUnloadableContext: ${{ parameters.runInUnloadableContext }} + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # Access token variable for internal project from the + # DotNet-HelixApi-Access variable group + helixAccessToken: $(HelixApiAccessToken) + + # Choose which tests to send to Helix: CoreFX or CoreCLR. + ${{ if eq(parameters.corefxTests, true) }}: + helixProjectArguments: 'eng/helixcorefxtests.proj' + ${{ if ne(parameters.corefxTests, true) }}: + helixProjectArguments: 'tests/helixpublishwitharcade.proj' + + ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: + scenarios: + - normal + - no_tiered_compilation + ${{ if in(parameters.testGroup, 'jitstress') }}: + scenarios: + - jitminopts + - jitstress1 + - jitstress1_tiered + - jitstress2 + - jitstress2_tiered + - zapdisable + - tailcallstress + ${{ if in(parameters.testGroup, 'jitstress-isas-arm') }}: + scenarios: + - jitstress_isas_incompletehwintrinsic + - jitstress_isas_nohwintrinsic + - jitstress_isas_nohwintrinsic_nosimd + - jitstress_isas_nosimd + ${{ if in(parameters.testGroup, 'jitstress-isas-x86') }}: + scenarios: + - jitstress_isas_incompletehwintrinsic + - jitstress_isas_nohwintrinsic + - jitstress_isas_nohwintrinsic_nosimd + - jitstress_isas_nosimd + - jitstress_isas_x86_noaes + - jitstress_isas_x86_noavx + - jitstress_isas_x86_noavx2 + - jitstress_isas_x86_nobmi1 + - jitstress_isas_x86_nobmi2 + - jitstress_isas_x86_nofma + - jitstress_isas_x86_nohwintrinsic + - jitstress_isas_x86_nolzcnt + - jitstress_isas_x86_nopclmulqdq + - jitstress_isas_x86_nopopcnt + - jitstress_isas_x86_nosse + - jitstress_isas_x86_nosse2 + - jitstress_isas_x86_nosse3 + - jitstress_isas_x86_nosse3_4 + - jitstress_isas_x86_nosse41 + - jitstress_isas_x86_nosse42 + - jitstress_isas_x86_nossse3 + ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}: + scenarios: + - jitstressregs1_x86_noavx + - jitstressregs2_x86_noavx + - jitstressregs3_x86_noavx + - jitstressregs4_x86_noavx + - jitstressregs8_x86_noavx + - jitstressregs0x10_x86_noavx + - jitstressregs0x80_x86_noavx + - jitstressregs0x1000_x86_noavx + ${{ if in(parameters.testGroup, 'jitstressregs' ) }}: + scenarios: + - jitstressregs1 + - jitstressregs2 + - jitstressregs3 + - jitstressregs4 + - jitstressregs8 + - jitstressregs0x10 + - jitstressregs0x80 + - jitstressregs0x1000 + ${{ if in(parameters.testGroup, 'jitstress2-jitstressregs') }}: + scenarios: + - jitstress2_jitstressregs1 + - jitstress2_jitstressregs2 + - jitstress2_jitstressregs3 + - jitstress2_jitstressregs4 + - jitstress2_jitstressregs8 + - jitstress2_jitstressregs0x10 + - jitstress2_jitstressregs0x80 + - jitstress2_jitstressregs0x1000 + ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: + scenarios: + - gcstress0x3 + - gcstress0xc + ${{ if in(parameters.testGroup, 'gcstress-extra') }}: + scenarios: + - heapverify1 + - gcstress0xc_zapdisable + - gcstress0xc_zapdisable_jitstress2 + - gcstress0xc_zapdisable_heapverify1 + - gcstress0xc_jitstress1 + - gcstress0xc_jitstress2 + - gcstress0xc_jitminopts_heapverify1 + ${{ if in(parameters.testGroup, 'r2r-extra') }}: + scenarios: + - jitstress1 + - jitstress2 + - jitstress1_tiered + - jitstress2_tiered + - jitstressregs1 + - jitstressregs2 + - jitstressregs3 + - jitstressregs4 + - jitstressregs8 + - jitstressregs0x10 + - jitstressregs0x80 + - jitstressregs0x1000 + - jitminopts + - forcerelocs + - gcstress15 + + # Zip tests + - ${{ if and(eq(parameters.publishTestArtifacts, true), ne(parameters.corefxTests, true)) }}: + - task: ArchiveFiles@2 + displayName: Zip Tests + inputs: + rootFolderOrFile: $(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper) + + ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + archiveType: tar + tarCompression: gz + archiveFile: $(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper).tar.gz + ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + archiveType: zip + archiveFile: $(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper).zip + includeRootFolder: true + + - ${{ if and(eq(parameters.publishTestArtifacts, true), ne(parameters.corefxTests, true)) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Test Build + inputs: + ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + pathtoPublish: $(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper).tar.gz + ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + pathtoPublish: $(Build.SourcesDirectory)/bin/tests/$(osGroup).$(archType).$(buildConfigUpper).zip + + ${{ if eq(parameters.readyToRun, true) }}: + artifactName: ${{ format('Tests_r2r_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if ne(parameters.readyToRun, true) }}: + artifactName: ${{ format('Tests_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + + # Publish Logs + - task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + pathtoPublish: $(Build.SourcesDirectory)/bin/Logs + ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + artifactName: ${{ format('TestLogs_r2r_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + artifactName: ${{ format('TestLogs_corefx_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + artifactName: ${{ format('TestLogs_r2r_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + artifactName: ${{ format('TestLogs_{0}_{1}_{2}_{3}', parameters.osIdentifier, parameters.archType, parameters.buildConfig, parameters.testGroup) }} + continueOnError: true + condition: always() diff --git a/eng/upload-artifact-step.yml b/eng/upload-artifact-step.yml deleted file mode 100644 index 4106b31..0000000 --- a/eng/upload-artifact-step.yml +++ /dev/null @@ -1,24 +0,0 @@ -parameters: - rootFolder: '' - includeRootFolder: false - archiveFile: '' - archiveType: '' - tarCompression: '' - artifactName: '' - -steps: - # Zip Artifact - - task: ArchiveFiles@2 - displayName: Zip Tests - inputs: - rootFolderOrFile: ${{ parameters.rootFolder }} - archiveFile: ${{ parameters.archiveFile }} - archiveType: ${{ parameters.archiveType }} - tarCompression: ${{ parameters.tarCompression }} - includeRootFolder: ${{ parameters.includeRootFolder }} - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact - inputs: - pathtoPublish: ${{ parameters.archiveFile }} - artifactName: ${{ parameters.artifactName }} diff --git a/eng/xplat-test-job.yml b/eng/xplat-test-job.yml deleted file mode 100644 index 68312a7..0000000 --- a/eng/xplat-test-job.yml +++ /dev/null @@ -1,86 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osIdentifier: '' - name: '' - helixType: '(unspecified)' - container: '' - crossrootfsDir: '' - readyToRun: false - - # arcade-specific parameters - condition: '' - continueOnError: false - dependsOn: '' - displayName: '' - timeoutInMinutes: '' - enableMicrobuild: '' - gatherAssetManifests: false - - variables: {} ## any extra variables to add to the defaults defined below - -jobs: -- template: /eng/xplat-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osIdentifier: ${{ parameters.osIdentifier }} - name: ${{ parameters.name }} - helixType: ${{ parameters.helixType }} - container: ${{ parameters.container }} - crossrootfsDir: ${{ parameters.crossrootfsDir }} - - # arcade-specific parameters - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - dependsOn: ${{ parameters.dependsOn }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: ${{ parameters.enableMicrobuild }} - gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - - variables: - - ${{ if ne(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.osIdentifier}}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.testGroup }} - - - ${{ if eq(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.osIdentifier}}_${{ parameters.archType }}_${{ parameters.buildConfig }} - - - name: binTestsPath - value: $(Build.SourcesDirectory)/bin/tests - - - name: testRootFolderPath - value: $(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper) - - - ${{ if ne(parameters.readyToRun, true) }}: - - name: testArtifactName - value: Tests_$(testArtifactRootName) - - - ${{ if eq(parameters.readyToRun, true) }}: - - name: testArtifactName - value: Tests_r2r_$(testArtifactRootName) - - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - name: archiveExtension - value: '.zip' - - name: archiveType - value: zip - - name: tarCompression - value: '' - - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - name: archiveExtension - value: '.tar.gz' - - name: archiveType - value: tar - - name: tarCompression - value: gz - - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - steps: ${{ parameters.steps }}