Add wasm runs (#39548)

* Add WASM microbenchmark runs

This commit adds WASM microbenchmark support to the perf lab.

* Increase timeout for first run

* Add runkind

* Update proj file selection

diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj
index 185522fc739c1f59a7eced6ab0dc77e47efbdb40..71114b074e1b268baec67a4a377a5bd1191d479e 100644 (file)
--- a/eng/common/performance/microbenchmarks.proj
+++ b/eng/common/performance/microbenchmarks.proj
@@ -41,6 +41,10 @@
     <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(WasmDotnet)' == 'true'">
+    <CliArguments>$(CliArguments) --wasm</CliArguments>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(MonoDotnet)' == 'true' and '$(AGENT_OS)' == 'Windows_NT'">
     <CoreRunArgument>--corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe</CoreRunArgument>
   </PropertyGroup>
@@ -65,7 +69,7 @@
   </PropertyGroup>
   
   <PropertyGroup>
-    <WorkItemTimeout>2:30</WorkItemTimeout>
+    <WorkItemTimeout>12:30</WorkItemTimeout>
     <WorkItemTimeout Condition="'$(HelixSourcePrefix)' != 'official'">0:15</WorkItemTimeout>
   </PropertyGroup>
 

diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index c87cbf0fc2326539e437fc1cddc8b4a49bac0cb1..66f63a397a0fc08f86edab04acad7c797acd615f 100755 (executable)
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -24,6 +24,8 @@ run_from_perf_repo=false
 use_core_run=true
 use_baseline_core_run=true
 using_mono=false
+wasm_runtime_loc=
+using_wasm=false
 
 while (($# > 0)); do
   lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -70,7 +72,7 @@ while (($# > 0)); do
       ;;
     --kind)
       kind=$2
-      configurations="CompliationMode=$compilation_mode RunKind=$kind"
+      configurations="CompilationMode=$compilation_mode RunKind=$kind"
       shift 2
       ;;
     --runcategories)
@@ -101,6 +103,10 @@ while (($# > 0)); do
       mono_dotnet=$2
       shift 2
       ;;
+    --wasm)
+      wasm_runtime_loc=$2
+      shift 2
+      ;;
     --compare)
       compare=true
       shift 1
@@ -130,6 +136,7 @@ while (($# > 0)); do
       echo "  --runcategories <value>        Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\""
       echo "  --internal                     If the benchmarks are running as an official job."
       echo "  --monodotnet                   Pass the path to the mono dotnet for mono performance testing."
+      echo "  --wasm                         Path to the unpacled wasm runtime pack."
       echo ""
       exit 0
       ;;
@@ -141,7 +148,7 @@ if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-perfo
 fi
 
 if [ -z "$configurations" ]; then
-    configurations="CompliationMode=$compilation_mode"
+    configurations="CompilationMode=$compilation_mode"
 fi
 
 if [ -z "$core_root_directory" ]; then
@@ -191,11 +198,15 @@ if [[ "$mono_dotnet" != "" ]]; then
     configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
 fi
 
+if [[ "$wasm_runtime_loc" != "" ]]; then
+    configurations="CompilationMode=wasm;RunKind=micro"
+fi
+
 if [[ "$monointerpreter" == "true" ]]; then
-    extra_benchmark_dotnet_arguments="--category-exclusion-filter NoInterpreter"
+    extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter"
 fi
 
-common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
+common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs \"$configurations\" --architecture $architecture"
 setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
 
 
@@ -217,6 +228,17 @@ else
     mv $docs_directory $workitem_directory
 fi
 
+if [[ "$wasm_runtime_loc" != "" ]]; then
+    using_wasm=true
+    
+    wasm_dotnet_path=$payload_directory/dotnet-wasm
+    
+    mv $wasm_runtime_loc $wasm_dotnet_path
+    
+    extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
+    
+fi
+
 if [[ "$mono_dotnet" != "" ]]; then
     using_mono=true
     mono_dotnet_path=$payload_directory/dotnet-mono
@@ -259,3 +281,4 @@ Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable fal
 Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
 Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
 Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false
+Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false

diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml
index cb97c408514fde3831192d16f6171efc05c0aa52..0cbf812e809d82360ed4a51228db7ae7d7815497 100644 (file)
--- a/eng/pipelines/coreclr/perf.yml
+++ b/eng/pipelines/coreclr/perf.yml
@@ -56,6 +56,21 @@ jobs:
       testGroup: perf
 
 # build mono
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    buildConfig: release
+    runtimeFlavor: mono
+    platforms:
+    - Browser_wasm
+    jobParameters:
+      buildArgs: -s mono+libs+installer -c $(_BuildConfig)
+      nameSuffix: wasm
+      isOfficialBuild: ${{ variables.isOfficialBuild }}
+      extraStepsTemplate: /eng/pipelines/common/upload-unsigned-artifacts-step.yml
+      extraStepsParameters:
+        name: MonoRuntimePacks
+
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/mono/templates/build-job.yml
@@ -97,6 +112,21 @@ jobs:
         runKind: micro_mono
 
 # run coreclr microbenchmarks perf job
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml
+    buildConfig: release
+    runtimeFlavor: wasm
+    platforms:
+    - Linux_x64
+    jobParameters:
+      testGroup: perf
+      liveLibrariesBuildConfig: Release
+      runtimeType: wasm
+      codeGenType: 'wasm'
+      projectFile: microbenchmarks.proj
+      runKind: micro
+
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml

diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml
index 57df1c0dcb638b502dcc4156482882b595d5b115..21799d64354d5cd20db08848e34fed8963df865c 100644 (file)
--- a/eng/pipelines/coreclr/templates/perf-job.yml
+++ b/eng/pipelines/coreclr/templates/perf-job.yml
@@ -43,6 +43,8 @@ jobs:
       - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
     - ${{ if eq(parameters.runtimeType, 'mono') }}:
       - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    - ${{ if eq(parameters.runtimeType, 'wasm') }}:
+      - ${{ format('build_{0}{1}_{2}_{3}_{4}', 'Browser', '', 'wasm', parameters.buildConfig, parameters.runtimeType) }}
 
     ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       ${{ if eq(parameters.runtimeType, 'mono') }}:
@@ -52,7 +54,9 @@ jobs:
     ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       ${{ if eq(parameters.runtimeType, 'mono') }}:
         extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono --kind micro_mono
-      ${{ if ne(parameters.runtimeType, 'mono') }}:
+      ${{ if eq(parameters.runtimeType, 'wasm') }}:
+        extraSetupParameters: --architecture ${{ parameters.archType }} --wasm $(librariesDownloadDir)/bin/wasm --kind micro
+      ${{ if eq(parameters.runtimeType, 'coreclr') }}:
         extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }}
 
     variables: ${{ parameters.variables }}
@@ -90,6 +94,19 @@ jobs:
           artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)'
           displayName: 'Mono runtime'
 
+    - ${{ if eq(parameters.runtimeType, 'wasm') }}:
+      # Download artifact
+      - task: DownloadBuildArtifacts@0
+        displayName: 'Download WASM Runtime Pack'
+        inputs:
+          buildType: current
+          downloadType: single
+          downloadPath: '$(Build.SourcesDirectory)/__download__'
+          artifactName: 'IntermediateUnsignedArtifacts'
+
+      - script: "mkdir $(librariesDownloadDir)/bin/wasm;unzip -o $(Build.SourcesDirectory)/__download__/IntermediateUnsignedArtifacts/MonoRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.browser-wasm.5.0.0-ci.nupkg data/* runtimes/* -d $(librariesDownloadDir)/bin/wasm;cp src/mono/wasm/runtime-test.js $(librariesDownloadDir)/bin/wasm/runtime-test.js;find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \\;"
+        displayName: "Create wasm directory (Linux)"
+
     # Create Core_Root
     - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
       displayName: Create Core_Root

diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml
index 3d34f5a67a142b6a1bb9d81e15a787c4ad4d8fd0..0e44fbb7cd28a158c0c1cc718c8659f7e7ef943f 100644 (file)
--- a/eng/pipelines/coreclr/templates/run-performance-job.yml
+++ b/eng/pipelines/coreclr/templates/run-performance-job.yml
@@ -53,7 +53,7 @@ jobs:
     - IsInternal: ''
     - HelixApiAccessToken: ''
     - HelixPreCommandStemWindows: 'py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install --user azure.storage.blob==12.0.0 --force-reinstall;py -3 -m pip install --user azure.storage.queue==12.0.0 --force-reinstall;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
-    - HelixPreCommandStemLinux: 'sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
+    - HelixPreCommandStemLinux: 'sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/Scripts/activate;export PYTHONPATH=;pip3 install --user azure.storage.blob==12.0.0 --force-reinstall;pip3 install --user azure.storage.queue==12.0.0 --force-reinstall;npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
     - ExtraMSBuildLogsWindows: 'set MSBUILDDEBUGCOMM=1;set "MSBUILDDEBUGPATH=%HELIX_WORKITEM_UPLOAD_ROOT%"'
     - ExtraMSBuildLogsLinux: 'export MSBUILDDEBUGCOMM=1;export "MSBUILDDEBUGPATH=$HELIX_WORKITEM_UPLOAD_ROOT"'
     - HelixPreCommand: ''
@@ -73,7 +73,7 @@ jobs:
       - ${{ if eq( parameters.osGroup, 'Windows_NT') }}:
         - HelixPreCommand: $(ExtraMSBuildLogsWindows)
       - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-          - HelixPreCommand: $(ExtraMSBuildLogsLinux)
+          - HelixPreCommand: $(ExtraMSBuildLogsLinux);npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8
     
 
     - ${{ if and(eq(parameters.codeGenType, 'Interpreter'), eq(parameters.runtimeType, 'mono')) }}: