From aa5ff31ecefd0fb5946af06412d1dbb0a2677424 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 26 Jul 2022 19:18:51 -0400 Subject: [PATCH] [wasm] Add `runtime-wasm-perf` pipeline (#72119) - This adds a new `runtime-wasm-perf` pipeline, which can be run manually with `/azp run runtime-wasm-perf` - Essentially, splits the perf jobs into `perf-wasm-jobs.yml`, and `perf-non-wasm-jobs.yml` - It does not change the existing perf pipeline - this will run all the wasm benchmarks, but it doesn't support comparing results yet - A follow up PR will add perf jobs to the runtime pipelines, for sanity check, and won't run the full set of benchmarks. --- .../common/download-specific-artifact-step.yml | 12 +- eng/pipelines/common/perf-variables.yml | 15 + eng/pipelines/common/variables.yml | 2 + eng/pipelines/coreclr/perf-non-wasm-jobs.yml | 452 +++++++++++++++ eng/pipelines/coreclr/perf-wasm-jobs.yml | 162 ++++++ eng/pipelines/coreclr/perf.yml | 605 +-------------------- eng/pipelines/coreclr/templates/perf-job.yml | 82 +-- .../coreclr/templates/perf-send-to-helix.yml | 2 + .../coreclr/templates/run-scenarios-job.yml | 6 +- eng/pipelines/runtime-extra-platforms-wasm.yml | 12 + eng/pipelines/runtime-wasm-perf.yml | 26 + eng/testing/performance/microbenchmarks.proj | 7 +- eng/testing/performance/performance-setup.sh | 6 + src/mono/wasm/build/WasmApp.props | 1 + 14 files changed, 752 insertions(+), 638 deletions(-) create mode 100644 eng/pipelines/common/perf-variables.yml create mode 100644 eng/pipelines/coreclr/perf-non-wasm-jobs.yml create mode 100644 eng/pipelines/coreclr/perf-wasm-jobs.yml create mode 100644 eng/pipelines/runtime-wasm-perf.yml diff --git a/eng/pipelines/common/download-specific-artifact-step.yml b/eng/pipelines/common/download-specific-artifact-step.yml index 180eb88..9ccf241 100644 --- a/eng/pipelines/common/download-specific-artifact-step.yml +++ b/eng/pipelines/common/download-specific-artifact-step.yml @@ -4,6 +4,10 @@ parameters: artifactFileName: '' artifactName: '' displayName: '' + buildId: '' + branchName: '' + pipeline: '' + project: 'public' # 'internal' or 'public' steps: # Download artifact @@ -11,11 +15,11 @@ steps: displayName: 'Download specific ${{ parameters.displayName }}' inputs: buildType: specific - project: 'public' # 'internal' or 'public' - pipeline: 'Antigen' + project: ${{ parameters.project }} + pipeline: ${{ parameters.pipeline }} buildVersionToDownload: specific - branchName: 'refs/pull/59376/head' - buildId: '1387378' + branchName: ${{ parameters.branchName }} + buildId: ${{ parameters.buildId }} downloadType: single downloadPath: '$(Build.SourcesDirectory)/__download__' artifactName: '${{ parameters.artifactName }}' diff --git a/eng/pipelines/common/perf-variables.yml b/eng/pipelines/common/perf-variables.yml new file mode 100644 index 0000000..c40e917 --- /dev/null +++ b/eng/pipelines/common/perf-variables.yml @@ -0,0 +1,15 @@ +variables: + - name: _wasmCollectHelixLogsScript + value: >- + test "$_commandExitCode" -eq 0 || ( + test -d "$HELIX_WORKITEM_UPLOAD_ROOT" && ( + export _PERF_DIR=$HELIX_WORKITEM_ROOT/performance; + mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; + mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; + mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; + mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/for-running; + find $_PERF_DIR -name '*.binlog' | xargs -I{} cp {} $HELIX_WORKITEM_UPLOAD_ROOT/log; + cp -R $_PERF_DIR/artifacts/obj/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; + cp -R $_PERF_DIR/artifacts/bin/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; + cp -R $_PERF_DIR/artifacts/obj/BenchmarkDotNet.Autogenerated $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; + cp -R $_PERF_DIR/artifacts/bin/for-running $HELIX_WORKITEM_UPLOAD_ROOT/log/for-running)) diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index 44bde20..76156f4 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -33,3 +33,5 @@ variables: value: Release ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: value: Debug + +- template: /eng/pipelines/common/perf-variables.yml diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml new file mode 100644 index 0000000..9cc0e60 --- /dev/null +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -0,0 +1,452 @@ +jobs: + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: + + # build mono + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/mono/templates/build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: + - Linux_x64 + + # build coreclr and libraries + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: release + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: + + # build coreclr and libraries + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: release + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + - Linux_musl_x64 + jobParameters: + testGroup: perf + + # build mono for AOT + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: AOT + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: AOT Mono Artifacts + artifactName: LinuxMonoAOTx64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz + + # build mono Android scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Android_arm64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: AndroidMono + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: Android Mono Artifacts + artifactName: AndroidMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz + + # build mono iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - iOS_arm64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: iOSMono + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: iOS Mono Artifacts + artifactName: iOSMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz + + # build mono + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/mono/templates/build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: + - Linux_x64 + + # run mono and maui android scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Windows_x64 + variables: + - name: mauiVersion + value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] + jobParameters: + testGroup: perf + runtimeType: AndroidMono + projectFile: android_scenarios.proj + runKind: android_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfpixel4a' + additionalSetupParameters: "-MauiVersion $env:mauiVersion" + + # run mono iOS scenarios and maui iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - OSX_x64 + variables: + - name: mauiVersion + value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] + jobParameters: + testGroup: perf + runtimeType: iOSMono + projectFile: ios_scenarios.proj + runKind: ios_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfiphone12mini' + iOSLlvmBuild: False + additionalSetupParameters: "--mauiversion $(mauiVersion)" + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - OSX_x64 + variables: + - name: mauiVersion + value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] + jobParameters: + testGroup: perf + runtimeType: iOSMono + projectFile: ios_scenarios.proj + runKind: ios_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfiphone12mini' + iOSLlvmBuild: True + additionalSetupParameters: "--mauiversion $(mauiVersion)" + + # run maui android scenarios for net6 + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Windows_x64 + variables: + - name: mauiVersion + value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet6.outputs['getMauiVersion.mauiVersion'] ] + jobParameters: + testGroup: perf + runtimeType: AndroidMobileNet6 + projectFile: android_scenarios_net6.proj + runKind: android_scenarios_net6 + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfpixel4a' + additionalSetupParameters: "-MauiVersion $env:mauiVersion" + + # run maui iOS scenarios for net6 (Maui doesn't need Llmv true build (for net6)) + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - OSX_x64 + variables: + - name: mauiVersion + value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet6.outputs['getMauiVersion.mauiVersion'] ] + jobParameters: + testGroup: perf + runtimeType: iOSMobileNet6 + projectFile: ios_scenarios_net6.proj + runKind: ios_scenarios_net6 + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfiphone12mini' + iOSLlvmBuild: False + additionalSetupParameters: "--mauiversion $(mauiVersion)" + + # run mono microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + + # run mono interpreter perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + + # run coreclr perftiger microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + - Linux_musl_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + + # run coreclr perftiger microbenchmarks pgo perf jobs + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -NoPgo + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -DynamicPgo + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -FullPgo + + # run coreclr perfowl microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perfowl' + + # run coreclr crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: crossgen_perf.proj + runKind: crossgen_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perftiger' + + # Uncomment to reenable package replacement + ## build maui runtime packs + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Android_x86 + # - Android_x64 + # - Android_arm + # - Android_arm64 + # - MacCatalyst_x64 + # - iOSSimulator_x64 + # - iOS_arm64 + # - iOS_arm + # jobParameters: + # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + # nameSuffix: Maui_Packs_Mono + # isOfficialBuild: false + # extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + # extraStepsParameters: + # name: MonoRuntimePacks + + # build maui app net7.0 + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - iOS_arm64 + jobParameters: + # Uncomment to reenable package replacement for main + #dependsOn: + # - Build_Android_arm_release_Maui_Packs_Mono + # - Build_Android_arm64_release_Maui_Packs_Mono + # - Build_Android_x86_release_Maui_Packs_Mono + # - Build_Android_x64_release_Maui_Packs_Mono + # - Build_MacCatalyst_x64_release_Maui_Packs_Mono + # - Build_iOSSimulator_x64_release_Maui_Packs_Mono + # - Build_iOS_arm_release_Maui_Packs_Mono + # - Build_iOS_arm64_release_Maui_Packs_Mono + buildArgs: -s mono -c $(_BuildConfig) + nameSuffix: MACiOSAndroidMauiNet7 + isOfficialBuild: false + pool: + vmImage: 'macos-11' + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-maui-apps-net7.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: MAC, iOS, and Android Maui Artifacts Net7 + artifactName: MACiOSAndroidMauiArmNet7 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz + + # build maui app net6.0 + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - iOS_arm64 + jobParameters: + buildArgs: -s mono -c $(_BuildConfig) + nameSuffix: MACiOSAndroidMauiNet6 + isOfficialBuild: false + pool: + vmImage: 'macos-11' + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-maui-apps-net6.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: MAC, iOS, and Android Maui Artifacts Net6 + artifactName: MACiOSAndroidMauiArmNet6 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml new file mode 100644 index 0000000..9c84f1d --- /dev/null +++ b/eng/pipelines/coreclr/perf-wasm-jobs.yml @@ -0,0 +1,162 @@ +# This contains all the wasm related perf jobs +# + +parameters: + runProfile: 'non-v8' + compare: false + onlySanityCheck: false + downloadSpecificBuild: null # { buildId, pipeline, branchName, project } + collectHelixLogsScript: '' + +jobs: + +- ${{ if eq(parameters.runProfile, 'non-v8') }}: + - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + # build mono on wasm - if not using an existing build + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release + runtimeFlavor: mono + platforms: + - Browser_wasm + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: wasm + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml + extraStepsParameters: + configForBuild: Release + + #run mono wasm microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: Release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptEngine: 'javascriptcore' + collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + #downloadSpecificBuild: + #buildId: '1693181' + #pipeline: 'perf-wasm' + #branchName: 'refs/pull/67312/head' + + #run mono wasm aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: Release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + testgroup: perf + livelibrariesbuildconfig: Release + skipLiveLibrariesDownload: true + runtimetype: wasm + codegentype: 'aot' + projectfile: microbenchmarks.proj + runkind: micro + runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptengine: 'javascriptcore' + collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + +- ${{ if eq(parameters.runProfile, 'v8') }}: + - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + # build mono on wasm - if not using an existing build + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release + runtimeFlavor: mono + platforms: + - Browser_wasm + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: wasm + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml + extraStepsParameters: + configForBuild: Release + + # run mono wasm interpreter (default) microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + skipLiveLibrariesDownload: true + runtimeType: wasm + codeGenType: 'wasm' + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptEngine: 'v8' + collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} + compare: ${{ parameters.compare }} + onlySanityCheck: ${{ parameters.onlySanityCheck }} + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + + #run mono wasm aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + testgroup: perf + livelibrariesbuildconfig: Release + skipLiveLibrariesDownload: true + runtimetype: wasm + codegentype: 'aot' + projectfile: microbenchmarks.proj + runkind: micro + runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptEngine: 'v8' + collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} + compare: ${{ parameters.compare }} + onlySanityCheck: ${{ parameters.onlySanityCheck }} + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + + # run mono wasm blazor perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + skipLiveLibrariesDownload: true + runtimeType: wasm + projectFile: blazor_perf.proj + runKind: blazor_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + #additionalSetupParameters: '--latestdotnet' - doesn't do anything, IIUC. see performance-setup.sh + logicalmachine: 'perftiger' + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index c61fa4c..f1dfcce 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -19,20 +19,6 @@ trigger: variables: - template: /eng/pipelines/common/variables.yml - - name: _wasmCollectHelixLogsScript - value: >- - test "$_commandExitCode" -eq 0 || ( - test -d "$HELIX_WORKITEM_UPLOAD_ROOT" && ( - export _PERF_DIR=$HELIX_WORKITEM_ROOT/performance; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; - mkdir -p $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/bin; - find $_PERF_DIR -name '*.binlog' | xargs -I{} cp {} $HELIX_WORKITEM_UPLOAD_ROOT/log; - cp -R $_PERF_DIR/artifacts/obj/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/obj; - cp -R $_PERF_DIR/artifacts/bin/MicroBenchmarks $HELIX_WORKITEM_UPLOAD_ROOT/log/MicroBenchmarks/bin; - cp -R $_PERF_DIR/artifacts/obj/BenchmarkDotNet.Autogenerated $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/obj; - cp -R $_PERF_DIR/artifacts/bin/BenchmarkDotNet.Autogenerated $HELIX_WORKITEM_UPLOAD_ROOT/log/BenchmarkDotNet.Autogenerated/bin)) schedules: - cron: "30 2 * * *" @@ -44,588 +30,11 @@ schedules: jobs: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: - - # build mono - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - Linux_x64 - - # build coreclr and libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: release - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - - # build mono on wasm - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - runtimeFlavor: mono - platforms: - - Browser_wasm - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: wasm - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml - extraStepsParameters: - configForBuild: Release - - #run mono wasm microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - buildConfig: Release - runtimeFlavor: wasm - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: wasm - codeGenType: 'wasm' - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptEngine: 'javascriptcore' - collectHelixLogsScript: ${{ variables['_wasmCollectHelixLogsScript'] }} - - #run mono wasm aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? - buildconfig: Release - runtimeflavor: wasm - platforms: - - linux_x64 - jobparameters: - testgroup: perf - livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true - runtimetype: wasm - codegentype: 'aot' - projectfile: microbenchmarks.proj - runkind: micro - runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptengine: 'javascriptcore' - collectHelixLogsScript: ${{ variables['_wasmCollectHelixLogsScript'] }} - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: - - # build coreclr and libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: release - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - - Linux_musl_x64 - jobParameters: - testGroup: perf - - # build mono on wasm - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - runtimeFlavor: mono - platforms: - - Browser_wasm - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: wasm - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml - extraStepsParameters: - configForBuild: Release - - # build mono for AOT - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: AOT - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: AOT Mono Artifacts - artifactName: LinuxMonoAOTx64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz - - # build mono Android scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Android_arm64 - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: AndroidMono - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: Android Mono Artifacts - artifactName: AndroidMonoarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz - - # build mono iOS scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - iOS_arm64 - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: iOSMono - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: iOS Mono Artifacts - artifactName: iOSMonoarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz - - # build mono - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - Linux_x64 - - # run mono and maui android scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Windows_x64 - variables: - - name: mauiVersion - value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] - jobParameters: - testGroup: perf - runtimeType: AndroidMono - projectFile: android_scenarios.proj - runKind: android_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfpixel4a' - additionalSetupParameters: "-MauiVersion $env:mauiVersion" - - # run mono iOS scenarios and maui iOS scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - OSX_x64 - variables: - - name: mauiVersion - value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] - jobParameters: - testGroup: perf - runtimeType: iOSMono - projectFile: ios_scenarios.proj - runKind: ios_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfiphone12mini' - iOSLlvmBuild: False - additionalSetupParameters: "--mauiversion $(mauiVersion)" - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - OSX_x64 - variables: - - name: mauiVersion - value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet7.outputs['getMauiVersion.mauiVersion'] ] - jobParameters: - testGroup: perf - runtimeType: iOSMono - projectFile: ios_scenarios.proj - runKind: ios_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfiphone12mini' - iOSLlvmBuild: True - additionalSetupParameters: "--mauiversion $(mauiVersion)" - - # run maui android scenarios for net6 - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Windows_x64 - variables: - - name: mauiVersion - value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet6.outputs['getMauiVersion.mauiVersion'] ] - jobParameters: - testGroup: perf - runtimeType: AndroidMobileNet6 - projectFile: android_scenarios_net6.proj - runKind: android_scenarios_net6 - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfpixel4a' - additionalSetupParameters: "-MauiVersion $env:mauiVersion" - - # run maui iOS scenarios for net6 (Maui doesn't need Llmv true build (for net6)) - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - OSX_x64 - variables: - - name: mauiVersion - value: $[ dependencies.Build_iOS_arm64_release_MACiOSAndroidMauiNet6.outputs['getMauiVersion.mauiVersion'] ] - jobParameters: - testGroup: perf - runtimeType: iOSMobileNet6 - projectFile: ios_scenarios_net6.proj - runKind: ios_scenarios_net6 - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfiphone12mini' - iOSLlvmBuild: False - additionalSetupParameters: "--mauiversion $(mauiVersion)" - - # run mono microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - - # run mono interpreter perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - codeGenType: 'Interpreter' - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - - # run mono wasm interpreter (default) microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - buildConfig: release - runtimeFlavor: wasm - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true - runtimeType: wasm - codeGenType: 'wasm' - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptEngine: 'v8' - collectHelixLogsScript: ${{ variables['_wasmCollectHelixLogsScript'] }} - - #run mono wasm aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? - buildconfig: release - runtimeflavor: wasm - platforms: - - linux_x64 - jobparameters: - testgroup: perf - livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true - runtimetype: wasm - codegentype: 'aot' - projectfile: microbenchmarks.proj - runkind: micro - runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptEngine: 'v8' - collectHelixLogsScript: ${{ variables['_wasmCollectHelixLogsScript'] }} - - # run mono aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - buildConfig: release - runtimeFlavor: aot - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - codeGenType: 'AOT' - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - - # run coreclr perftiger microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - - Linux_musl_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - - # run coreclr perftiger microbenchmarks pgo perf jobs - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -NoPgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -DynamicPgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -FullPgo - - # run coreclr perfowl microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perfowl' - - # run coreclr crossgen perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: crossgen_perf.proj - runKind: crossgen_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perftiger' - - # run mono wasm blazor perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: wasm - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true - runtimeType: wasm - projectFile: blazor_perf.proj - runKind: blazor_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - additionalSetupParameters: '--latestdotnet' - logicalmachine: 'perftiger' - - # Uncomment to reenable package replacement - ## build maui runtime packs - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/common/global-build-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Android_x86 - # - Android_x64 - # - Android_arm - # - Android_arm64 - # - MacCatalyst_x64 - # - iOSSimulator_x64 - # - iOS_arm64 - # - iOS_arm - # jobParameters: - # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - # nameSuffix: Maui_Packs_Mono - # isOfficialBuild: false - # extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - # extraStepsParameters: - # name: MonoRuntimePacks - - # build maui app net7.0 - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - iOS_arm64 - jobParameters: - # Uncomment to reenable package replacement for main - #dependsOn: - # - Build_Android_arm_release_Maui_Packs_Mono - # - Build_Android_arm64_release_Maui_Packs_Mono - # - Build_Android_x86_release_Maui_Packs_Mono - # - Build_Android_x64_release_Maui_Packs_Mono - # - Build_MacCatalyst_x64_release_Maui_Packs_Mono - # - Build_iOSSimulator_x64_release_Maui_Packs_Mono - # - Build_iOS_arm_release_Maui_Packs_Mono - # - Build_iOS_arm64_release_Maui_Packs_Mono - buildArgs: -s mono -c $(_BuildConfig) - nameSuffix: MACiOSAndroidMauiNet7 - isOfficialBuild: false - pool: - vmImage: 'macos-11' - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-maui-apps-net7.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: MAC, iOS, and Android Maui Artifacts Net7 - artifactName: MACiOSAndroidMauiArmNet7 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz - - # build maui app net6.0 - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - iOS_arm64 - jobParameters: - buildArgs: -s mono -c $(_BuildConfig) - nameSuffix: MACiOSAndroidMauiNet6 - isOfficialBuild: false - pool: - vmImage: 'macos-11' - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-maui-apps-net6.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: MAC, iOS, and Android Maui Artifacts Net6 - artifactName: MACiOSAndroidMauiArmNet6 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz +- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml + parameters: + ${{ and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: + runProfile: 'non-v8' + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: + runProfile: 'v8' +- template: /eng/pipelines/coreclr/perf-non-wasm-jobs.yml diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index c75a58f..6bc2131 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -22,6 +22,9 @@ parameters: skipLiveLibrariesDownload: false collectHelixLogsScript: '' timeoutInMinutes: 320 + compare: false + onlySanityCheck: false + downloadSpecificBuild: null # buildId, pipeline, branchName, project ### Perf job @@ -61,27 +64,28 @@ jobs: collectHelixLogsScript: ${{ parameters.collectHelixLogsScript }} # Test job depends on the corresponding build job - dependsOn: - - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'AndroidMobileNet6', 'iOSMobileNet6', 'wasm')) }}: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if eq(parameters.runtimeType, 'wasm')}}: - - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'Browser', '', 'wasm', 'Linux', parameters.buildConfig, parameters.runtimeType) }} - - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: - - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} - - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: - - ${{ 'build_Android_arm64_release_AndroidMono' }} - - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet7' }} - - ${{ if eq(parameters.runtimeType, 'AndroidMobileNet6')}}: - - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet6' }} - - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: - - ${{ 'build_iOS_arm64_release_iOSMono' }} - - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet7' }} - - ${{ if eq(parameters.runtimeType, 'iOSMobileNet6')}}: - - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet6' }} + ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + dependsOn: + - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'AndroidMobileNet6', 'iOSMobileNet6', 'wasm')) }}: + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: + - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if eq(parameters.runtimeType, 'wasm')}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'Browser', '', 'wasm', 'Linux', parameters.buildConfig, parameters.runtimeType) }} + - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} + - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: + - ${{ 'build_Android_arm64_release_AndroidMono' }} + - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet7' }} + - ${{ if eq(parameters.runtimeType, 'AndroidMobileNet6')}}: + - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet6' }} + - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: + - ${{ 'build_iOS_arm64_release_iOSMono' }} + - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet7' }} + - ${{ if eq(parameters.runtimeType, 'iOSMobileNet6')}}: + - ${{ 'Build_iOS_arm64_release_MACiOSAndroidMauiNet6' }} ${{ if and(eq(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'AndroidMobileNet6', 'iOSMobileNet6'))) }}: ${{ if eq(parameters.runtimeType, 'mono') }}: @@ -92,9 +96,9 @@ jobs: ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono ${{ if and(eq(parameters.runtimeType, 'wasm'), ne(parameters.codeGenType, 'AOT')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --javascriptengine ${{ parameters.javascriptEngine }} + extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --javascriptengine ${{ parameters.javascriptEngine }} $(extraSetupParametersSuffix) ${{ if and(eq(parameters.runtimeType, 'wasm'), eq(parameters.codeGenType, 'AOT')) }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --wasmaot --javascriptengine ${{ parameters.javascriptEngine }} + extraSetupParameters: --architecture ${{ parameters.archType }} --wasmbundle $(librariesDownloadDir)/bin/wasm --wasmaot --javascriptengine ${{ parameters.javascriptEngine }} $(extraSetupParametersSuffix) ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm')) }}: extraSetupParameters: --architecture ${{ parameters.archType }} --monoaot $(librariesDownloadDir)/bin/aot ${{ if and(eq(parameters.runtimeType, 'coreclr'), ne(parameters.osSubGroup, '_musl')) }}: @@ -106,7 +110,15 @@ jobs: ${{ if in(parameters.runtimeType, 'iosMono', 'iOSMobileNet6') }}: extraSetupParameters: --architecture ${{ parameters.archType }} --iosmono --iosllvmbuild ${{ parameters.iOSLlvmBuild }} - variables: ${{ parameters.variables }} + variables: + - ${{ each variable in parameters.variables }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - name: extraSetupParametersSuffix + ${{ if eq(parameters.compare, 'true') }}: + value: '--compare' + ${{ if eq(parameters.onlySanityCheck, 'true') }}: + value: '--only-sanity' frameworks: - ${{ parameters.framework }} @@ -144,12 +156,21 @@ jobs: # Download wasm - ${{ if eq(parameters.runtimeType, 'wasm') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir)/BrowserWasm - artifactFileName: BrowserWasm.zip - artifactName: BrowserWasm - displayName: BrowserWasm + - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.zip + artifactName: BrowserWasm + displayName: BrowserWasm + - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}: + - template: /eng/pipelines/common/download-specific-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.zip + artifactName: BrowserWasm + displayName: BrowserWasm + ${{ insert }}: ${{ parameters.downloadSpecificBuild }} - script: >- mkdir -p $(librariesDownloadDir)/bin/wasm/wasm-data && @@ -226,7 +247,6 @@ jobs: artifactFileName: 'MauiBlazorAndroidAppNet6.tar.gz' artifactName: 'MauiBlazorAndroidAppNet6' displayName: 'Maui Blazor Android App Net6' - # Download iOSMono tests and MauiiOS/MacCatalyst - ${{ if eq(parameters.runtimeType, 'iOSMono') }}: diff --git a/eng/pipelines/coreclr/templates/perf-send-to-helix.yml b/eng/pipelines/coreclr/templates/perf-send-to-helix.yml index a6f4636..44290b3 100644 --- a/eng/pipelines/coreclr/templates/perf-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/perf-send-to-helix.yml @@ -9,6 +9,7 @@ parameters: HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json @@ -40,6 +41,7 @@ steps: HelixPreCommands: ${{ parameters.HelixPreCommands }} HelixPostCommands: ${{ parameters.HelixPostCommands }} WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} diff --git a/eng/pipelines/coreclr/templates/run-scenarios-job.yml b/eng/pipelines/coreclr/templates/run-scenarios-job.yml index 79907ff..47426ab 100644 --- a/eng/pipelines/coreclr/templates/run-scenarios-job.yml +++ b/eng/pipelines/coreclr/templates/run-scenarios-job.yml @@ -34,7 +34,7 @@ jobs: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true continueOnError: ${{ parameters.continueOnError }} - + ${{ if ne(parameters.displayName, '') }}: displayName: '${{ parameters.displayName }}' ${{ if eq(parameters.displayName, '') }}: @@ -135,7 +135,7 @@ jobs: condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} # run ci-setup - - script: $(Python) $(PerformanceDirectory)\scripts\ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)\machine-setup.cmd + - script: $(Python) $(PerformanceDirectory)\scripts\ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)\machine-setup.cmd displayName: Run ci setup script (Windows) condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) $(ExtraSetupArguments) --output-file $(WorkItemDirectory)/machine-setup.sh @@ -196,7 +196,7 @@ jobs: includeRootFolder: false archiveFile: '$(WorkItemDirectory).zip' verbose: True - + # run perf testing in helix - template: /eng/pipelines/coreclr/templates/perf-send-to-helix.yml parameters: diff --git a/eng/pipelines/runtime-extra-platforms-wasm.yml b/eng/pipelines/runtime-extra-platforms-wasm.yml index 98a9079..70e8621 100644 --- a/eng/pipelines/runtime-extra-platforms-wasm.yml +++ b/eng/pipelines/runtime-extra-platforms-wasm.yml @@ -46,6 +46,12 @@ jobs: runAOT: true alwaysRun: true + # Disabled for now + #- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml + #parameters: + #runProfile: 'v8' + #onlySanityCheck: true + # # ********** For !rolling builds, IOW - PR builds ************* # - run everything, if relevant paths changed @@ -170,3 +176,9 @@ jobs: alwaysRun: ${{ parameters.isWasmOnlyBuild }} # ff tests are unstable currently shouldContinueOnError: true + + # Disable for now + #- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml + #parameters: + #runProfile: 'v8' + #onlySanityCheck: true diff --git a/eng/pipelines/runtime-wasm-perf.yml b/eng/pipelines/runtime-wasm-perf.yml new file mode 100644 index 0000000..0d554f0 --- /dev/null +++ b/eng/pipelines/runtime-wasm-perf.yml @@ -0,0 +1,26 @@ +# This is a wrapper yml for `perf-wasm-jobs`, which has all the perf related +# wasm jobs. This file is essentially so we can point the pipeline in azdo +# UI to this, and thus avoid any scheduled triggers + +trigger: none + +variables: + - template: /eng/pipelines/common/variables.yml + +jobs: + +# +# Evaluate paths +# +#- ${{ if eq(variables.dependOnEvaluatePaths, true) }}: + #- template: /eng/pipelines/common/evaluate-default-paths.yml + +- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml + parameters: + runProfile: 'v8' + collectHelixLogsScript: ${{ variables._wasmCollectHelixLogsScript }} + #onlySanityCheck: false + #downloadSpecificBuild: + #buildId: '1878694' + #pipeline: 'perf-wasm' + #branchName: 'refs/pull/72119/head' diff --git a/eng/testing/performance/microbenchmarks.proj b/eng/testing/performance/microbenchmarks.proj index e91f8e7..f5c924c 100644 --- a/eng/testing/performance/microbenchmarks.proj +++ b/eng/testing/performance/microbenchmarks.proj @@ -69,10 +69,10 @@ $(WorkItemCommand) $(CliArguments) - + 2:30 - 0:15 + 0:15 @@ -86,6 +86,9 @@ + + + diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh index 3c9776f..bb926ae 100755 --- a/eng/testing/performance/performance-setup.sh +++ b/eng/testing/performance/performance-setup.sh @@ -35,6 +35,7 @@ javascript_engine="v8" iosmono=false iosllvmbuild="" maui_version="" +only_sanity=false while (($# > 0)); do lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" @@ -157,6 +158,10 @@ while (($# > 0)); do maui_version=$2 shift 2 ;; + --only-sanity) + only_sanity=true + shift 1 + ;; *) echo "Common settings:" echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" @@ -413,3 +418,4 @@ Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variab Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false Write-PipelineSetVariable -Name 'iOSLlvmBuild' -Value "$iosllvmbuild" -is_multi_job_variable false +Write-PipelineSetVariable -name "OnlySanityCheck" -value "$only_sanity" -is_multi_job_variable false diff --git a/src/mono/wasm/build/WasmApp.props b/src/mono/wasm/build/WasmApp.props index 7fdde27..89ce619 100644 --- a/src/mono/wasm/build/WasmApp.props +++ b/src/mono/wasm/build/WasmApp.props @@ -4,6 +4,7 @@ Browser browser-wasm true + partial <_WasmBuildCoreDependsOn> _InitializeCommonProperties; -- 2.7.4