From 9929c07b6125ce9197e72d9b746966e6781a5ec3 Mon Sep 17 00:00:00 2001 From: Bruce Forstall Date: Tue, 16 Jun 2020 09:10:23 -0700 Subject: [PATCH] Add JIT stress mode testing using libraries tests (#36486) * Add JIT stress mode testing using libraries tests There are three separate pipelines, with different sets of JIT stress modes enabled for each: - libraries-jitstress.yml - libraries-jitstress2-jitstressregs.yml - libraries-jitstressregs.yml The live built Release libraries are used with a Checked CoreCLR. The non-stress configuration is tested during normal PR testing. The libraries `sendtohelix.proj` file is changed to take an optional list of CoreCLR stress scenarios to run. If given, a set of environment setting scripts is created and added to the Helix correlation payload. Then, an auxiliary `sendtohelixhelp.proj` file is invoked for each scenario (or just the default, empty scenario) to create and run the Helix tasks. * Remove remnants of old corefx-on-coreclr test run infrastructure * Rename corefx-*.yml to libraries-*.yml * Disable tests ``` src/libraries/System.Numerics.Vectors/tests/Matrix3x2Tests.cs https://github.com/dotnet/runtime/issues/36587 Matrix3x2CreateRotationCenterTest() https://github.com/dotnet/runtime/issues/36587 Matrix3x2CreateScaleCenterTest1() https://github.com/dotnet/runtime/issues/36587 Matrix3x2CreateScaleCenterTest3() src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs https://github.com/dotnet/runtime/issues/36586 Matrix4x4CreateFromAxisAngleTest() ``` * Fix run-test-job.yml * Fix BuildAllConfigurations case --- eng/pipelines/common/platform-matrix.yml | 1 - .../common/templates/runtimes/run-test-job.yml | 86 +++----- eng/pipelines/coreclr/corefx-jitstress.yml | 51 ----- .../coreclr/corefx-jitstress2-jitstressregs.yml | 51 ----- eng/pipelines/coreclr/corefx-jitstressregs.yml | 51 ----- eng/pipelines/coreclr/corefx.yml | 51 ----- eng/pipelines/coreclr/libraries-jitstress.yml | 71 +++++++ .../coreclr/libraries-jitstress2-jitstressregs.yml | 71 +++++++ eng/pipelines/coreclr/libraries-jitstressregs.yml | 71 +++++++ eng/pipelines/coreclr/templates/build-job.yml | 1 + .../coreclr/templates/xplat-pipeline-job.yml | 1 - eng/pipelines/libraries/helix.yml | 3 +- eng/pipelines/libraries/run-test-job.yml | 47 +++++ src/coreclr/build-test.cmd | 4 +- src/coreclr/build-test.sh | 8 +- src/coreclr/tests/Directory.Build.props | 8 - src/coreclr/tests/bringup_runtest.sh | 125 ------------ src/coreclr/tests/runtest.sh | 138 ------------- .../tests/GenericVectorTests.cs | 1 + .../tests/Matrix3x2Tests.cs | 3 + .../tests/Matrix4x4Tests.cs | 1 + src/libraries/sendtohelix.proj | 220 +++++++++++++-------- src/libraries/sendtohelixhelp.proj | 174 ++++++++++++++++ 23 files changed, 607 insertions(+), 631 deletions(-) delete mode 100644 eng/pipelines/coreclr/corefx-jitstress.yml delete mode 100644 eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml delete mode 100644 eng/pipelines/coreclr/corefx-jitstressregs.yml delete mode 100644 eng/pipelines/coreclr/corefx.yml create mode 100644 eng/pipelines/coreclr/libraries-jitstress.yml create mode 100644 eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml create mode 100644 eng/pipelines/coreclr/libraries-jitstressregs.yml create mode 100644 src/libraries/sendtohelixhelp.proj diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 04ca970..ecb21d5 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -12,7 +12,6 @@ parameters: # 'pr' - the queues used for a pull request for the platform. Typically a small set. # 'ci' - the queues used for a CI (post-merge) test run. # 'all' - the queues used for non-PR, non-CI test runs, e.g., Manual or Scheduled runs. Typically this is all available queues. - # 'corefx' - the queues used for a corefx test run. helixQueueGroup: 'pr' # helixQueuesTemplate is a yaml template which will be expanded in order to set up the helix queues # for the given platform and helixQueueGroup. diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index a4fe9f7..8fc0cfe 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -11,8 +11,6 @@ parameters: crossgen2: false compositeBuildMode: false helixQueues: '' - # If true, run the corefx tests instead of the coreclr ones - corefxTests: false condition: true stagedBuild: false displayNameArgs: '' @@ -38,7 +36,6 @@ jobs: container: ${{ parameters.container }} testGroup: ${{ parameters.testGroup }} crossrootfsDir: ${{ parameters.crossrootfsDir }} - corefxTests: ${{ parameters.corefxTests }} stagedBuild: ${{ parameters.stagedBuild }} liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} helixType: 'build/tests/' @@ -51,11 +48,10 @@ jobs: continueOnError: true dependsOn: - - ${{ if ne(parameters.corefxTests, true) }}: - - ${{ if eq(parameters.testGroup, 'innerloop') }}: - - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}' - - ${{ if ne(parameters.testGroup, 'innerloop') }}: - - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' + - ${{ if eq(parameters.testGroup, 'innerloop') }}: + - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}' + - ${{ if ne(parameters.testGroup, 'innerloop') }}: + - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - ${{ if ne(parameters.stagedBuild, true) }}: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: @@ -71,11 +67,6 @@ jobs: displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}' variables: - - name: testhostArg - value: 'generatetesthostonly' - - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - - name: testhostArg - value: 'buildtesthostonly' - name: runtimeFlavorArgs value: '' @@ -110,7 +101,7 @@ jobs: # # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the - # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. + # xunit test dlls that invokes the actual tests). # # Please note that for Crossgen / Crossgen2 R2R runs, the "test running time" also includes the # time needed to compile the test into native code with the Crossgen compiler. @@ -119,9 +110,6 @@ jobs: value: 10 - name: timeoutPerTestCollectionInMinutes value: 30 - - ${{ if eq(parameters.corefxTests, true) }}: - - name: timeoutPerTestCollectionInMinutes - value: 10 - ${{ if in(parameters.testGroup, 'outerloop') }}: - name: timeoutPerTestCollectionInMinutes value: 120 @@ -187,13 +175,12 @@ jobs: # Download and unzip managed test artifacts - - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(managedTestArtifactRootFolderPath)' - artifactFileName: '$(managedGenericTestArtifactName).tar.gz' - artifactName: '$(managedGenericTestArtifactName)' - displayName: 'generic managed test artifacts' + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(managedTestArtifactRootFolderPath)' + artifactFileName: '$(managedGenericTestArtifactName).tar.gz' + artifactName: '$(managedGenericTestArtifactName)' + displayName: 'generic managed test artifacts' # Download product binaries directory @@ -217,23 +204,21 @@ jobs: # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing # ilproj test projects during copynativeonly. - - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(microsoftNetSdkIlFolderPath)' - artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' - artifactName: '$(microsoftNetSdkIlArtifactName)' - displayName: 'Microsoft.NET.Sdk.IL package' + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(microsoftNetSdkIlFolderPath)' + artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' + artifactName: '$(microsoftNetSdkIlArtifactName)' + displayName: 'Microsoft.NET.Sdk.IL package' # Download and unzip native test artifacts - - ${{ if ne(parameters.corefxTests, true) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(nativeTestArtifactRootFolderPath)' - artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' - artifactName: '$(nativeTestArtifactName)' - displayName: 'native test artifacts' + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(nativeTestArtifactRootFolderPath)' + artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' + artifactName: '$(nativeTestArtifactName)' + displayName: 'native test artifacts' # Publish native test components to test output folder. Sadly we cannot do this @@ -241,15 +226,8 @@ jobs: # and directly unzip them there after download). Unfortunately the logic to copy # the native artifacts to the final test folders is dependent on availability of the # managed test artifacts. - - ${{ if ne(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipstressdependencies copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg) - displayName: Copy native test components to test output folder - - - # Generate test host - - ${{ if eq(parameters.corefxTests, true) }}: - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(testhostArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) - displayName: Generate test host + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipstressdependencies copynativeonly $(crossgenArg) $(buildConfig) $(archType) $(priorityArg) $(librariesOverrideArg) + displayName: Copy native test components to test output folder # Generate test wrappers. This is the step that examines issues.targets to exclude tests. @@ -296,13 +274,9 @@ jobs: # REVIEW: not sure why "cli" is part of the names here. Leave it for the ones that already had it, # but don't add it to new ones. - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: - helixType: 'test/functional/r2r_corefx/' - ${{ if and(eq(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: - helixType: 'test/functional/corefx/' - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.readyToRun, true)) }}: + ${{ if eq(parameters.readyToRun, true) }}: helixType: 'test/functional/r2r/cli/' - ${{ if and(ne(parameters.corefxTests, true), ne(parameters.readyToRun, true)) }}: + ${{ if ne(parameters.readyToRun, true) }}: helixType: 'test/functional/cli/' helixQueues: ${{ parameters.helixQueues }} @@ -326,11 +300,7 @@ jobs: # DotNet-HelixApi-Access variable group helixAccessToken: $(HelixApiAccessToken) - # Choose which tests to send to Helix: CoreFX or CoreCLR. - ${{ if eq(parameters.corefxTests, true) }}: - helixProjectArguments: '$(Build.SourcesDirectory)/eng/helixcorefxtests.proj' - ${{ if ne(parameters.corefxTests, true) }}: - helixProjectArguments: '$(coreClrRepoRoot)/tests/helixpublishwitharcade.proj' + helixProjectArguments: '$(coreClrRepoRoot)/tests/helixpublishwitharcade.proj' ${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}: scenarios: diff --git a/eng/pipelines/coreclr/corefx-jitstress.yml b/eng/pipelines/coreclr/corefx-jitstress.yml deleted file mode 100644 index 07acd41..0000000 --- a/eng/pipelines/coreclr/corefx-jitstress.yml +++ /dev/null @@ -1,51 +0,0 @@ -trigger: none - -pr: none - -schedules: -- cron: "0 7 * * *" - displayName: Mon through Sun at 11:00 PM (UTC-8:00) - branches: - include: - - master - always: true - -jobs: -# -# Checkout repository -# -- template: /eng/pipelines/common/checkout-job.yml - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: jitstress - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: corefx - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress - corefxTests: true - displayNameArgs: CoreFX diff --git a/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml deleted file mode 100644 index c58f8f5..0000000 --- a/eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml +++ /dev/null @@ -1,51 +0,0 @@ -trigger: none - -pr: none - -schedules: -- cron: "0 9 * * *" - displayName: Mon through Sun at 1:00 AM (UTC-8:00) - branches: - include: - - master - always: true - -jobs: -# -# Checkout repository -# -- template: /eng/pipelines/common/checkout-job.yml - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: jitstress - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: corefx - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress2-jitstressregs - corefxTests: true - displayNameArgs: CoreFX diff --git a/eng/pipelines/coreclr/corefx-jitstressregs.yml b/eng/pipelines/coreclr/corefx-jitstressregs.yml deleted file mode 100644 index ce064af..0000000 --- a/eng/pipelines/coreclr/corefx-jitstressregs.yml +++ /dev/null @@ -1,51 +0,0 @@ -trigger: none - -pr: none - -schedules: -- cron: "0 12 * * *" - displayName: Mon through Sun at 4:00 AM (UTC-8:00) - branches: - include: - - master - always: true - -jobs: -# -# Checkout repository -# -- template: /eng/pipelines/common/checkout-job.yml - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: jitstressregs - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: corefx - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstressregs - corefxTests: true - displayNameArgs: CoreFX diff --git a/eng/pipelines/coreclr/corefx.yml b/eng/pipelines/coreclr/corefx.yml deleted file mode 100644 index 2f70f3f..0000000 --- a/eng/pipelines/coreclr/corefx.yml +++ /dev/null @@ -1,51 +0,0 @@ -trigger: none - -pr: none - -schedules: -- cron: "0 11 * * *" - displayName: Mon through Sun at 3:00 AM (UTC-8:00) - branches: - include: - - master - always: true - -jobs: -# -# Checkout repository -# -- template: /eng/pipelines/common/checkout-job.yml - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - jobParameters: - testGroup: outerloop - -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this. - - Linux_arm - - Linux_arm64 - - Linux_x64 - # TODO: re-enable Windows_NT_arm when https://github.com/dotnet/corefx/issues/38863 is resolved - - Windows_NT_x64 - - Windows_NT_x86 - helixQueueGroup: corefx - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - corefxTests: true - displayNameArgs: CoreFX diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml new file mode 100644 index 0000000..ff7f25b --- /dev/null +++ b/eng/pipelines/coreclr/libraries-jitstress.yml @@ -0,0 +1,71 @@ +trigger: none + +pr: none + +schedules: +- cron: "0 7 * * *" + displayName: Mon through Sun at 11:00 PM (UTC-8:00) + branches: + include: + - master + always: true + +jobs: +# +# Checkout repository +# +- template: /eng/pipelines/common/checkout-job.yml + +# +# Build CoreCLR checked and libraries Release +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: checked + platforms: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + +# +# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + buildConfig: Release + platforms: + - Linux_x64 + - Windows_NT_x64 + jobParameters: + liveRuntimeBuildConfig: checked + testScope: innerloop + +# +# Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - Windows_NT_arm64 + - Windows_NT_x64 + - Windows_NT_x86 + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 300 + testScope: innerloop + liveRuntimeBuildConfig: checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + coreclrTestGroup: jitstress diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml new file mode 100644 index 0000000..656d139 --- /dev/null +++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml @@ -0,0 +1,71 @@ +trigger: none + +pr: none + +schedules: +- cron: "0 9 * * 6" + displayName: Sat at 1:00 AM (UTC-8:00) + branches: + include: + - master + always: true + +jobs: +# +# Checkout repository +# +- template: /eng/pipelines/common/checkout-job.yml + +# +# Build CoreCLR checked and libraries Release +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: checked + platforms: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + +# +# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + buildConfig: Release + platforms: + - Linux_x64 + - Windows_NT_x64 + jobParameters: + liveRuntimeBuildConfig: checked + testScope: innerloop + +# +# Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - Windows_NT_arm64 + - Windows_NT_x64 + - Windows_NT_x86 + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 300 + testScope: innerloop + liveRuntimeBuildConfig: checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + coreclrTestGroup: jitstress2-jitstressregs diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml new file mode 100644 index 0000000..2e62420 --- /dev/null +++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml @@ -0,0 +1,71 @@ +trigger: none + +pr: none + +schedules: +- cron: "0 9 * * 0" + displayName: Sun at 1:00 AM (UTC-8:00) + branches: + include: + - master + always: true + +jobs: +# +# Checkout repository +# +- template: /eng/pipelines/common/checkout-job.yml + +# +# Build CoreCLR checked and libraries Release +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: checked + platforms: + - Linux_x64 + - Linux_arm + - Linux_arm64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + +# +# Libraries Test Build - Release innerloop. All libraries are built on x64 and reused on all platforms. +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-test-job.yml + buildConfig: Release + platforms: + - Linux_x64 + - Windows_NT_x64 + jobParameters: + liveRuntimeBuildConfig: checked + testScope: innerloop + +# +# Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes +# +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - Windows_NT_arm64 + - Windows_NT_x64 + - Windows_NT_x86 + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 300 + testScope: innerloop + liveRuntimeBuildConfig: checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + coreclrTestGroup: jitstressregs diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index e1df064..cd52dad 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -180,6 +180,7 @@ jobs: artifactName: $(buildProductArtifactName) displayName: 'product build' + - ${{ if and(ne(parameters.compilerName, 'gcc'), ne(parameters.testGroup, '')) }}: # Publish test native components for consumption by test execution. - ${{ if ne(parameters.isOfficialBuild, true) }}: - template: /eng/pipelines/common/upload-artifact-step.yml diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 191b05d..115599a 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -8,7 +8,6 @@ parameters: container: '' testGroup: '' crossrootfsDir: '' - corefxTests: false liveLibrariesBuildConfig: '' stagedBuild: false strategy: '' diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index 10e4550..af5bdea 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -12,11 +12,11 @@ parameters: interpreter: '' condition: always() extraHelixArguments: '' + scenarios: '' steps: - script: ${{ parameters.msbuildScript }} $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj - /t:test /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archGroup }} /p:Configuration=${{ parameters.configuration }} @@ -33,3 +33,4 @@ steps: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops HelixTargetQueues: ${{ parameters.helixQueues }} # Pass queues to MSBuild as env var to avoid need of escaping them + _Scenarios: ${{ join(',', parameters.scenarios) }} # Pass scenarios to MSBuild as env var to avoid need of escaping comma separated list diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 9a82ead..ee446ac 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -18,6 +18,10 @@ parameters: dependsOnTestArchitecture: x64 condition: true variables: {} + # coreclrTestGroup: if empty, then a normal, default test run is created. If set, it indicates a set of + # stress modes that each test will be run with. This is the same usage as 'testGroup' in + # eng/pipelines/common/templates/runtimes/run-test-job.yml. + coreclrTestGroup: '' jobs: - template: /eng/pipelines/libraries/base-job.yml @@ -100,3 +104,46 @@ jobs: helixToken: '' testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix) extraHelixArguments: $(_extraHelixArguments) + + # coreclrTestGroup: The following mappings of 'coreclrTestGroup' to 'scenarios' is copied from + # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup' + # for clarity), and should remain in sync. This is only a subset; only the testGroups that are + # used to test the libraries have been added here. More could be added if we decided to test the + # libraries with more stress modes (e.g., GCStress). The scenario tags are interpreted by + # src\coreclr\tests\testenvironment.proj. + # + # The one difference here compared to eng/pipelines/common/templates/runtimes/run-test-job.yml is + # that 'jitstress' contains 'no_tiered_compilation'. The 'normal' (default) test mode + # is run in a regular CI job, so there is no need to duplicate it here. So, add 'no_tiered_compilation' + # to the 'jitstress' job instead of adding a new job just for 'no_tiered_compilation'. + + ${{ if in(parameters.coreclrTestGroup, 'jitstress') }}: + scenarios: + - no_tiered_compilation + - jitminopts + - jitstress1 + - jitstress1_tiered + - jitstress2 + - jitstress2_tiered + - zapdisable + - tailcallstress + ${{ if in(parameters.coreclrTestGroup, 'jitstressregs' ) }}: + scenarios: + - jitstressregs1 + - jitstressregs2 + - jitstressregs3 + - jitstressregs4 + - jitstressregs8 + - jitstressregs0x10 + - jitstressregs0x80 + - jitstressregs0x1000 + ${{ if in(parameters.coreclrTestGroup, 'jitstress2-jitstressregs') }}: + scenarios: + - jitstress2_jitstressregs1 + - jitstress2_jitstressregs2 + - jitstress2_jitstressregs3 + - jitstress2_jitstressregs4 + - jitstress2_jitstressregs8 + - jitstress2_jitstressregs0x10 + - jitstress2_jitstressregs0x80 + - jitstress2_jitstressregs0x1000 diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index a3e16dc..ca0ad53 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -108,7 +108,6 @@ if /i "%1" == "skipgeneratelayout" (set __SkipGenerateLayout=1&set processedA if /i "%1" == "copynativeonly" (set __CopyNativeTestBinaries=1&set __SkipStressDependencies=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set __SkipGenerateLayout=1&set __SkipCrossgenFramework=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "generatelayoutonly" (set __SkipManaged=1&set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) -if /i "%1" == "buildtesthostonly" (set __SkipNative=1&set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set __SkipGenerateLayout=1&set __SkipStressDependencies=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgenframeworkonly" (set __SkipRestorePackages=1&set __SkipStressDependencies=1&set __SkipNative=1&set __SkipManaged=1&set __SkipGenerateLayout=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -591,7 +590,6 @@ echo Build architecture: one of x64, x86, arm, arm64 ^(default: x64^). echo Build type: one of Debug, Checked, Release ^(default: Debug^). echo skipmanaged: skip the managed tests build echo skipnative: skip the native tests build -echo buildtesthostonly: build the CoreFX testhost only echo skiprestorepackages: skip package restore echo runtimeid ^: Builds a test overlay for the specified OS ^(Only supported when building against packages^). Supported IDs are: echo alpine.3.4.3-x64: Builds overlay for Alpine 3.4.3 @@ -609,7 +607,7 @@ echo win-x64: Builds overlay for portable Windows echo win7-x64: Builds overlay for Windows 7 echo crossgen: Precompiles the framework managed assemblies echo copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests. -echo skipgeneratelayout: Do not generate the Core_Root layout or the CoreFX testhost. +echo skipgeneratelayout: Do not generate the Core_Root layout echo generatelayoutonly: Generate the Core_Root layout without building managed or native test components echo targetsNonWindows: echo Exclude- Optional parameter - specify location of default exclusion file ^(defaults to tests\issues.targets if not specified^) diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index fb4fe83..eefde57 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -514,7 +514,6 @@ usage_list+=("-skiptestwrappers: Don't generate test wrappers.") usage_list+=("-buildtestwrappersonly: only build the test wrappers.") usage_list+=("-copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests.") -usage_list+=("-generatetesthostonly: only generate the test host.") usage_list+=("-generatelayoutonly: only pull down dependencies and build coreroot.") usage_list+=("-crossgenframeworkonly: only compile the framework in CORE_ROOT with Crossgen / Crossgen2.") @@ -575,10 +574,6 @@ handle_arguments_local() { __TestBuildMode=crossgen2 ;; - generatetesthostonly|-generatetesthostonly) - __GenerateTestHostOnly=1 - ;; - generatelayoutonly|-generatelayoutonly) __GenerateLayoutOnly=1 ;; @@ -645,7 +640,6 @@ __DoCrossgen2=0 __CompositeBuildMode=0 __DotNetCli="$__RepoRootDir/dotnet.sh" __GenerateLayoutOnly= -__GenerateTestHostOnly= __IsMSBuildOnNETCoreSupported=0 __MSBCleanBuildArgs= __NativeTestIntermediatesDir= @@ -711,7 +705,7 @@ if [[ -z "$HOME" ]]; then echo "HOME not defined; setting it to $HOME" fi -if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then +if [[ (-z "$__GenerateLayoutOnly") && (-z "$__BuildTestWrappersOnly") ]]; then build_Tests elif [[ ! -z "$__BuildTestWrappersOnly" ]]; then build_test_wrappers diff --git a/src/coreclr/tests/Directory.Build.props b/src/coreclr/tests/Directory.Build.props index 5aec9a8..bfdb2b3 100644 --- a/src/coreclr/tests/Directory.Build.props +++ b/src/coreclr/tests/Directory.Build.props @@ -65,12 +65,4 @@ - - - $(ProductVersion) - $([MSBuild]::NormalizeDirectory('$(TestWorkingDir)', 'testhost')) - $([MSBuild]::NormalizeDirectory('$(TestHostRootPath)', 'host', 'fxr', '$(TestHostVersion)')) - $([MSBuild]::NormalizeDirectory('$(TestHostRootPath)', 'shared', 'Microsoft.NETCore.App', '$(TestHostVersion)')) - - diff --git a/src/coreclr/tests/bringup_runtest.sh b/src/coreclr/tests/bringup_runtest.sh index 8b7fa33..e875992 100755 --- a/src/coreclr/tests/bringup_runtest.sh +++ b/src/coreclr/tests/bringup_runtest.sh @@ -67,13 +67,6 @@ function print_usage { echo ' is zero when launching this script. This option is intended for use in CI.' echo ' --xunitOutputPath= : Create xUnit XML report at the specifed path (default: /coreclrtests.xml)' echo '' - echo 'CoreFX Test Options ' - echo ' --corefxtests : Runs CoreFX tests' - echo ' --corefxtestsall : Runs all available CoreFX tests' - echo ' --corefxtestlist= : Runs the CoreFX tests specified in the passed list' - echo ' --testHostDir= : Directory containing a built test host including core binaries, test dependencies' - echo ' and a dotnet executable' - echo '' echo 'Runtime Code Coverage options:' echo ' --coreclr-coverage : Optional argument to get coreclr code coverage reports' echo ' --coreclr-objs= : Location of root of the object directory' @@ -441,80 +434,6 @@ function create_core_overlay { copy_test_native_bin_to_test_root $coreOverlayDir } -function create_testhost -{ - if [ ! -d "$testHostDir" ]; then - exit_with_error "$errorSource" "Did not find the test host directory: $testHostDir" - fi - - # Initialize test variables - local dotnetExe=$coreClrSrc/../../dotnet.sh - local coreClrSrcTestDir=$coreClrSrc/tests - - if [ -z $coreClrBinDir ]; then - local coreClrBinDir=${coreClrSrc}/bin - export __CoreFXTestDir=${coreClrSrc}/artifacts/tests/CoreFX - else - export __CoreFXTestDir=${coreClrBinDir}/tests/CoreFX - fi - - local coreFXTestSetupUtilityName=CoreFX.TestUtils.TestFileSetup - local coreFXTestSetupUtility="${coreClrSrcTestDir}/src/Common/CoreFX/TestFileSetup/${coreFXTestSetupUtilityName}.csproj" - local coreFXTestSetupUtilityOutputPath=${__CoreFXTestDir}/TestUtilities - local coreFXTestBinariesOutputPath=${__CoreFXTestDir}/tests_downloaded - - if [ -z $CoreFXTestList]; then - local CoreFXTestList="${coreClrSrcTestDir}/CoreFX/CoreFX.issues.json" - fi - - case "${OSName}" in - # Check if we're running under OSX - Darwin) - local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_OSX.txt) - local coreFXTestExclusionDef=nonosxtests - ;; - # Default to Linux - *) - local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_Linux.txt) - local coreFXTestExclusionDef=nonlinuxtests - ;; - esac - - local coreFXTestExecutable=xunit.console.netcore.exe - local coreFXLogDir=${coreClrBinDir}/log/CoreFX/ - local coreFXTestExecutableArgs="--notrait category=nonnetcoreapptests --notrait category=${coreFXTestExclusionDef} --notrait category=failing --notrait category=IgnoreForCI --notrait category=OuterLoop --notrait Benchmark=true" - - chmod +x ${dotnetExe} - resetCommandArgs=("msbuild /t:Restore ${coreFXTestSetupUtility}") - echo "${dotnetExe} $resetCommandArgs" - "${dotnetExe}" $resetCommandArgs - - buildCommandArgs=("msbuild ${coreFXTestSetupUtility} /p:OutputPath=${coreFXTestSetupUtilityOutputPath} /p:Platform=${_arch} /p:Configuration=Release") - echo "${dotnetExe} $buildCommandArgs" - "${dotnetExe}" $buildCommandArgs - - if [ "${RunCoreFXTestsAll}" == "1" ]; then - local coreFXRunCommand=--runAllTests - else - local coreFXRunCommand=--runSpecifiedTests - fi - - local buildTestSetupUtilArgs=("${coreFXTestSetupUtilityOutputPath}/${coreFXTestSetupUtilityName}.dll --clean --outputDirectory ${coreFXTestBinariesOutputPath} --testListJsonPath ${CoreFXTestList} ${coreFXRunCommand} --dotnetPath ${testHostDir}/dotnet --testUrl ${coreFXTestRemoteURL} --executable ${coreFXTestExecutable} --log ${coreFXLogDir} ${coreFXTestExecutableArgs}") - echo "${dotnetExe} $buildTestSetupUtilArgs" - "${dotnetExe}" $buildTestSetupUtilArgs - - local exitCode=$? - if [ $exitCode != 0 ]; then - echo Running CoreFX tests finished with failures - else - echo Running CoreFX tests finished successfully - fi - - echo Check ${coreFXLogDir} for test run logs - - exit ${exitCode} -} - declare -a skipCrossGenFiles function is_skip_crossgen_test { @@ -1188,20 +1107,6 @@ do --runcrossgentests) export RunCrossGen=1 ;; - --corefxtests) - export RunCoreFXTests=1 - ;; - --corefxtestsall) - export RunCoreFXTests=1 - export RunCoreFXTestsAll=1 - ;; - --corefxtestlist) - export RunCoreFXTests=1 - export CoreFXTestList=${i#*=} - ;; - --testHostDir=*) - export testHostDir=${i#*=} - ;; --sequential) ((maxProcesses = 1)) ;; @@ -1272,36 +1177,6 @@ fi export COMPlus_gcServer="$serverGC" -if [ "$RunCoreFXTests" == 1 ]; -then - if [ -z "$coreClrSrc" ] - then - echo "Coreclr src files are required to run CoreFX tests" - echo "Coreclr src files root path can be passed using '--coreclr-src' argument" - print_usage - exit $EXIT_CODE_EXCEPTION - fi - - if [ -z "$testHostDir" ]; then - echo "--testHostDir is required to run CoreFX tests" - print_usage - exit $EXIT_CODE_EXCEPTION - fi - - if [ ! -f "$testHostDir/dotnet" ]; then - echo "Executable dotnet not found in $testHostDir" - exit $EXIT_CODE_EXCEPTION - fi - - if [ ! -d "$testHostDir" ]; then - echo "Directory specified by --testHostDir does not exist: $testRootDir" - exit $EXIT_CODE_EXCEPTION - fi - - create_testhost - exit 0 -fi - if [ -z "$testRootDir" ]; then echo "--testRootDir is required." print_usage diff --git a/src/coreclr/tests/runtest.sh b/src/coreclr/tests/runtest.sh index 95437d4..857cda1 100755 --- a/src/coreclr/tests/runtest.sh +++ b/src/coreclr/tests/runtest.sh @@ -41,88 +41,6 @@ function print_usage { echo ' --xunitOutputPath= : Create xUnit XML report at the specifed path (default: /coreclrtests.xml)' echo ' --printLastResultsOnly : Print the results of the last run' echo ' --runincontext : Run each tests in an unloadable AssemblyLoadContext' - echo '' - echo 'CoreFX Test Options ' - echo ' --corefxtests : Runs CoreFX tests' - echo ' --corefxtestsall : Runs all available CoreFX tests' - echo ' --corefxtestlist= : Runs the CoreFX tests specified in the passed list' - echo ' --testHostDir= : Directory containing a built test host including core binaries, test dependencies' - echo ' and a dotnet executable' - echo ' --coreclr-src= : Specify the CoreCLR root directory. Required to build the TestFileSetup tool for CoreFX testing.' -} - -function create_testhost -{ - if [ ! -d "$testHostDir" ]; then - exit_with_error "$errorSource" "Did not find the test host directory: $testHostDir" - fi - - # Initialize test variables - local dotnetExe=$coreClrSrc/../../dotnet.sh - local coreClrSrcTestDir=$coreClrSrc/tests - - if [ -z $coreClrBinDir ]; then - local coreClrBinDir=${coreClrSrc}/../../artifacts/bin/coreclr - export __CoreFXTestDir=${coreClrSrc}/../../artifacts/tests/CoreFX - else - export __CoreFXTestDir=${coreClrBinDir}/tests/CoreFX - fi - - local coreFXTestSetupUtilityName=CoreFX.TestUtils.TestFileSetup - local coreFXTestSetupUtility="${coreClrSrcTestDir}/src/Common/CoreFX/TestFileSetup/${coreFXTestSetupUtilityName}.csproj" - local coreFXTestSetupUtilityOutputPath=${__CoreFXTestDir}/TestUtilities - local coreFXTestBinariesOutputPath=${__CoreFXTestDir}/tests_downloaded - - if [ -z $CoreFXTestList]; then - local CoreFXTestList="${coreClrSrcTestDir}/CoreFX/CoreFX.issues.json" - fi - - case "${OSName}" in - # Check if we're running under OSX - Darwin) - local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_OSX.txt) - local coreFXTestExclusionDef=nonosxtests - ;; - # Default to Linux - *) - local coreFXTestRemoteURL=$(<${coreClrSrcTestDir}/CoreFX/CoreFXTestListURL_Linux.txt) - local coreFXTestExclusionDef=nonlinuxtests - ;; - esac - - local coreFXTestExecutable=xunit.console.netcore.exe - local coreFXLogDir=${coreClrBinDir}/log/CoreFX/ - local coreFXTestExecutableArgs="--notrait category=nonnetcoreapptests --notrait category=${coreFXTestExclusionDef} --notrait category=failing --notrait category=IgnoreForCI --notrait category=OuterLoop --notrait Benchmark=true" - - chmod +x ${dotnetExe} - resetCommandArgs=("msbuild /t:Restore ${coreFXTestSetupUtility}") - echo "${dotnetExe} $resetCommandArgs" - "${dotnetExe}" $resetCommandArgs - - buildCommandArgs=("msbuild ${coreFXTestSetupUtility} /p:OutputPath=${coreFXTestSetupUtilityOutputPath} /p:Platform=${_arch} /p:Configuration=Release") - echo "${dotnetExe} $buildCommandArgs" - "${dotnetExe}" $buildCommandArgs - - if [ "${RunCoreFXTestsAll}" == "1" ]; then - local coreFXRunCommand=--runAllTests - else - local coreFXRunCommand=--runSpecifiedTests - fi - - local buildTestSetupUtilArgs=("${coreFXTestSetupUtilityOutputPath}/${coreFXTestSetupUtilityName}.dll --clean --outputDirectory ${coreFXTestBinariesOutputPath} --testListJsonPath ${CoreFXTestList} ${coreFXRunCommand} --dotnetPath ${testHostDir}/dotnet --testUrl ${coreFXTestRemoteURL} --executable ${coreFXTestExecutable} --log ${coreFXLogDir} ${coreFXTestExecutableArgs}") - echo "${dotnetExe} $buildTestSetupUtilArgs" - "${dotnetExe}" $buildTestSetupUtilArgs - - local exitCode=$? - if [ $exitCode != 0 ]; then - echo Running CoreFX tests finished with failures - else - echo Running CoreFX tests finished successfully - fi - - echo Check ${coreFXLogDir} for test run logs - - exit ${exitCode} } function set_up_core_dump_generation { @@ -197,9 +115,7 @@ testNativeBinDir= coreOverlayDir= coreClrBinDir= mscorlibDir= -coreFxBinDir= coreClrObjs= -coreClrSrc= coverageOutputDir= testEnv= playlistFile= @@ -301,9 +217,6 @@ do --mscorlibDir=*) mscorlibDir=${i#*=} ;; - --coreFxBinDir=*) - coreFxBinDir=${i#*=} - ;; --testDir=*) testDirectories[${#testDirectories[@]}]=${i#*=} ;; @@ -322,20 +235,6 @@ do --runcrossgen2tests) export RunCrossGen2=1 ;; - --corefxtests) - export RunCoreFXTests=1 - ;; - --corefxtestsall) - export RunCoreFXTests=1 - export RunCoreFXTestsAll=1 - ;; - --corefxtestlist) - export RunCoreFXTests=1 - export CoreFXTestList=${i#*=} - ;; - --testHostDir=*) - export testHostDir=${i#*=} - ;; --sequential) runSequential=1 ;; @@ -357,9 +256,6 @@ do --coreclr-objs=*) coreClrObjs=${i#*=} ;; - --coreclr-src=*) - coreClrSrc=${i#*=} - ;; --coverage-output-dir=*) coverageOutputDir=${i#*=} ;; @@ -406,40 +302,6 @@ fi export COMPlus_gcServer="$serverGC" ################################################################################ -# CoreFX -################################################################################ - -if [ "$RunCoreFXTests" == 1 ]; -then - if [ -z "$coreClrSrc" ] - then - echo "Coreclr src files are required to run CoreFX tests" - echo "Coreclr src files root path can be passed using '--coreclr-src' argument" - print_usage - exit $EXIT_CODE_EXCEPTION - fi - - if [ -z "$testHostDir" ]; then - echo "--testHostDir is required to run CoreFX tests" - print_usage - exit $EXIT_CODE_EXCEPTION - fi - - if [ ! -f "$testHostDir/dotnet" ]; then - echo "Executable dotnet not found in $testHostDir" - exit $EXIT_CODE_EXCEPTION - fi - - if [ ! -d "$testHostDir" ]; then - echo "Directory specified by --testHostDir does not exist: $testRootDir" - exit $EXIT_CODE_EXCEPTION - fi - - create_testhost - exit 0 -fi - -################################################################################ # Runtest.py ################################################################################ diff --git a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs index f349927..8bf10b8 100644 --- a/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs +++ b/src/libraries/System.Numerics.Vectors/tests/GenericVectorTests.cs @@ -3031,6 +3031,7 @@ namespace System.Numerics.Tests } [Fact] + [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36614", RuntimeTestModes.JitStress)] public void NarrowDouble() { double[] source1 = GenerateRandomValuesForVector(); diff --git a/src/libraries/System.Numerics.Vectors/tests/Matrix3x2Tests.cs b/src/libraries/System.Numerics.Vectors/tests/Matrix3x2Tests.cs index c2a128f..b7a0a88 100644 --- a/src/libraries/System.Numerics.Vectors/tests/Matrix3x2Tests.cs +++ b/src/libraries/System.Numerics.Vectors/tests/Matrix3x2Tests.cs @@ -187,6 +187,7 @@ namespace System.Numerics.Tests // A test for CreateRotation (float, Vector2f) [Fact] + [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36587", RuntimeTestModes.JitStress)] public void Matrix3x2CreateRotationCenterTest() { float radians = MathHelper.ToRadians(30.0f); @@ -685,6 +686,7 @@ namespace System.Numerics.Tests // A test for CreateScale (Vector2f, Vector2f) [Fact] + [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36587", RuntimeTestModes.JitStress)] public void Matrix3x2CreateScaleCenterTest1() { Vector2 scale = new Vector2(3, 4); @@ -744,6 +746,7 @@ namespace System.Numerics.Tests // A test for CreateScale (float, float, Vector2f) [Fact] + [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36587", RuntimeTestModes.JitStress)] public void Matrix3x2CreateScaleCenterTest3() { Vector2 scale = new Vector2(3, 4); diff --git a/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs b/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs index fc82fb8..e51357e 100644 --- a/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs +++ b/src/libraries/System.Numerics.Vectors/tests/Matrix4x4Tests.cs @@ -500,6 +500,7 @@ namespace System.Numerics.Tests // A test for CreateFromAxisAngle(Vector3f,float) [Fact] + [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/36586", RuntimeTestModes.JitStress)] public void Matrix4x4CreateFromAxisAngleTest() { float radians = MathHelper.ToRadians(-30.0f); diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index fc0f3f0..908921f 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -1,85 +1,144 @@ - - - - $(BUILD_BUILDNUMBER) - default - - 1800 - 900 - <_timeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds))) - - - true - - $(TestArchiveRuntimeRoot)test-runtime-$(BuildSettings).zip - $(TestArchiveRuntimeRoot)packages-testPayload-$(Configuration).zip + - $(TestArchiveRuntimeFile) - $(TestArchiveTestsRoot)**/*.zip + In the simple case, no scenarios are specified and the default CoreCLR optimization configuration + is used to run the tests. If a set of comma-separated scenarios are specified in the `_Scenarios` + property, then each test is run for each of the specified CoreCLR scenarios (e.g., COMPlus_JitStress=1; + COMPlus_JitStressRegs=4; COMPlus_JitStress=2 + COMPlus_JitStressRegs=0x1000). The set of acceptable + scenario names is defined and interpreted by src\coreclr\tests\testenvironment.proj. - $(Configuration) - $(TargetArchitecture) + "RunInParallelForEachScenario" is the "root" target for this Project. It first creates the + "correlation payload", which is the set of files used by all Helix submissions + (which we compress into a single file). +--> - - $(BuildSettings)- - $(TestRunNamePrefix)$(TestRunNamePrefixSuffix)- + - $(WaitForWorkItemCompletion) - + + + + + + + + + <_PropertiesToPass> + RuntimeFlavor=$(RuntimeFlavor); + TargetArchitecture=$(TargetArchitecture); + Configuration=$(Configuration); + TargetOS=$(TargetOS); + TestRunNamePrefixSuffix=$(TestRunNamePrefixSuffix); + Creator=$(Creator); + HelixAccessToken=$(HelixAccessToken); + HelixTargetQueues=$(HelixTargetQueues); + BuildTargetFramework=$(BuildTargetFramework); + BuildSettings=$(BuildSettings) + + + + + + + + + + + + - - - true - sdk + + - packaging- - $([System.IO.File]::ReadAllText('$(RepoRoot)global.json')) - $([System.Text.RegularExpressions.Regex]::Match($(GlobalJsonContent), '(%3F<="dotnet": ").*(%3F=")')) - + + $(MSBuildProjectDirectory)\sendtohelixhelp.proj + - - - innerloop - test/functional/cli/$(TestScope)/ - test/functional/packaging/ - + + <_Scenarios Include="$(_Scenarios.Split(','))" /> + + + <_ProjectsToBuild Include="$(PerScenarioProjectFile)"> + $(_PropertiesToPass);Scenario=%(_Scenarios.Identity);TestArchiveRuntimeFile=$(TestArchiveRuntimeFile) + + + + + <_BuildInParallel>false + <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' > '1' ">true + + + + + - - set DOTNET_CLI_TELEMETRY_OPTOUT=1;set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1;set DOTNET_MULTILEVEL_LOOKUP=0 + + true - dotnet msbuild %HELIX_CORRELATION_PAYLOAD%\test.msbuild - $(HelixCommand) /warnaserror - $(HelixCommand) /p:PackageTestProjectsDir=%HELIX_WORKITEM_PAYLOAD% - $(HelixCommand) /p:RestorePackagesPath=%HELIX_WORKITEM_PAYLOAD%\packages - $(HelixCommand) /p:LocalPackagesPath="%HELIX_CORRELATION_PAYLOAD%\packages\" + + $(TestArchiveRuntimeRoot)test-runtime-$(BuildSettings).zip + $(TestArchiveRuntimeRoot)packages-testPayload-$(Configuration).zip - - set MONO_ENV_OPTIONS='$(MonoEnvOptions)' - export MONO_ENV_OPTIONS='$(MonoEnvOptions)' - - - call RunTests.cmd --runtime-path %HELIX_CORRELATION_PAYLOAD% - ./RunTests.sh --runtime-path "$HELIX_CORRELATION_PAYLOAD" + SetStressModes_$(Scenario).cmd + SetStressModes_$(Scenario).sh + + + + + + + $(TestHostRootPath)$(TestEnvFileName) + + + + <_ProjectsToBuild Include="$(RepoRoot)\src\coreclr\tests\testenvironment.proj"> + Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows) + + + + + + + + + + + + + + + + + <_Scenario Include="$(Scenarios.Split(','))" /> + <_ProjectsToBuild Include="$(MSBuildProjectFile)"> + Scenario=%(_Scenario.Identity) + + + + + + <_RuntimeInputs Include="$(TestHostRootPath)**/*.dll" /> - - - + + <_RuntimeInputs Condition=" '$(Scenarios)' != '' and '$(TargetsWindows)' == 'true' " Include="$(TestHostRootPath)**/*.cmd" /> + <_RuntimeInputs Condition=" '$(Scenarios)' != '' and '$(TargetsWindows)' != 'true' " Include="$(TestHostRootPath)**/*.sh" /> + + + + + + + - - - - - - <_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" /> - - - %(Identity) - $(HelixCommand) - $(_timeoutSpan) - - - - - - - + + + - - - - diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj new file mode 100644 index 0000000..fd47a71 --- /dev/null +++ b/src/libraries/sendtohelixhelp.proj @@ -0,0 +1,174 @@ + + + + + + true + + + $(BUILD_BUILDNUMBER) + default + + + 1800 + 3600 + 900 + 1800 + <_timeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds))) + + + true + + + + $(TestArchiveRuntimeFile) + $(TestArchiveTestsRoot)**/*.zip + + $(Configuration) + $(TargetArchitecture) + + + $(BuildSettings)- + $(TestRunNamePrefix)$(TestRunNamePrefixSuffix)- + $(TestRunNamePrefix)$(Scenario)- + + $(WaitForWorkItemCompletion) + + + + + true + sdk + + packaging- + $([System.IO.File]::ReadAllText('$(RepoRoot)global.json')) + $([System.Text.RegularExpressions.Regex]::Match($(GlobalJsonContent), '(%3F<="dotnet": ").*(%3F=")')) + + + + + innerloop + test/functional/cli/$(TestScope)/ + test/functional/packaging/ + + + + + + + + + + dotnet msbuild %HELIX_CORRELATION_PAYLOAD%\test.msbuild + $(HelixCommand) /warnaserror + $(HelixCommand) /p:PackageTestProjectsDir=%HELIX_WORKITEM_PAYLOAD% + $(HelixCommand) /p:RestorePackagesPath=%HELIX_WORKITEM_PAYLOAD%\packages + $(HelixCommand) /p:LocalPackagesPath="%HELIX_CORRELATION_PAYLOAD%\packages\" + + + + + + + + + + + + + SetStressModes_$(Scenario).cmd + SetStressModes_$(Scenario).sh + + + + + + + + + + + + + + + + + + + + @(HelixPreCommand) + + + + + call RunTests.cmd --runtime-path %HELIX_CORRELATION_PAYLOAD% + ./RunTests.sh --runtime-path "$HELIX_CORRELATION_PAYLOAD" + + + + + + + + + + + + + + + + + + + + + + + <_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" /> + + + %(Identity) + $(HelixCommand) + $(_timeoutSpan) + + + + + + + + + + + + + + + + + -- 2.7.4