Add non-LLVM iOS size scenario (#54585)

* Add non-LLVM iOS size scenario

* add category

* remove parens

* variables in the wrong place.

* move setting configs to before it is written

diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml
index e125f17..32ddf22 100644 (file)
--- a/eng/pipelines/coreclr/perf.yml
+++ b/eng/pipelines/coreclr/perf.yml
@@ -215,7 +215,28 @@ jobs:
           archiveExtension: '.tar.gz'
           archiveType: tar
           tarCompression: gz
-  
+    
+  # build mono iOS scenarios
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/common/global-build-job.yml
+      buildConfig: release
+      runtimeFlavor: mono
+      platforms:
+      - iOS_arm64
+      jobParameters:
+        buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
+        nameSuffix: iOSMono
+        isOfficialBuild: false
+        extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
+        extraStepsParameters:
+          rootFolder: '$(Build.SourcesDirectory)/artifacts/'
+          includeRootFolder: true
+          displayName: iOS Mono Artifacts
+          artifactName: iOSMonoarm64
+          archiveExtension: '.tar.gz'
+          archiveType: tar
+          tarCompression: gz
 
   # build mono
   - template: /eng/pipelines/common/platform-matrix.yml
@@ -242,6 +263,23 @@ jobs:
         runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
         logicalmachine: 'perfpixel4a'
 
+  # run mono iOS scenarios
+  - template: /eng/pipelines/common/platform-matrix.yml
+    parameters:
+      jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml
+      buildConfig: release
+      runtimeFlavor: mono
+      platforms:
+        - Windows_x64
+      jobParameters:
+        testGroup: perf
+        runtimeType: iOSMono
+        projectFile: ios_scenarios.proj
+        runKind: ios_scenarios
+        runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml
+        logicalmachine: 'perfpixel4a'
+        iosLlvmBuild: False
+
   # run mono microbenchmarks perf job
   - template: /eng/pipelines/common/platform-matrix.yml
     parameters:

diff --git a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
index 2b556af..6bfd863 100644 (file)
--- a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
+++ b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml
@@ -23,6 +23,27 @@ steps:
     - script: make run MONO_ARCH=arm64 DEPLOY_AND_RUN=false
       workingDirectory: $(Build.SourcesDirectory)/src/mono/sample/Android
       displayName: Build HelloAndroid sample app
+  - ${{ if eq(parameters.osGroup, 'iOS') }}:
+    - script: make build-appbundle TARGET=iOS MONO_ARCH=arm64 MONO_CONFIG=Release AOT=True USE_LLVM=False
+      env:
+        DevTeamProvisioning: '-'
+      workingDirectory: $(Build.SourcesDirectory)/src/mono/sample/iOS
+      displayName: Build HelloiOS AOT sample app LLVM=False
+    - task: PublishBuildArtifacts@1
+      condition: succeededOrFailed()
+      displayName: 'Publish binlog'
+      inputs:
+        pathtoPublish: $(Build.SourcesDirectory)/src/mono/sample/iOS/msbuild.binlog
+        artifactName:  ${{ parameters.artifactName }}
+    - template: /eng/pipelines/common/upload-artifact-step.yml
+      parameters:
+          rootFolder: $(Build.SourcesDirectory)/src/mono/sample/iOS/bin/ios-arm64/publish/app/HelloiOS/Release-iphoneos/HelloiOS.app
+          includeRootFolder: true
+          displayName: iOS Sample App NoLLVM
+          artifactName: iOSSampleAppNoLLVM
+          archiveExtension: '.tar.gz'
+          archiveType: tar
+          tarCompression: gz
 
   - template: /eng/pipelines/common/upload-artifact-step.yml
     parameters:
@@ -42,4 +63,5 @@ steps:
       artifactName:  ${{ parameters.artifactName }}
       archiveExtension:  ${{ parameters.archiveExtension }}
       archiveType:  ${{ parameters.archiveType }}
-      tarCompression:  ${{ parameters.tarCompression }}
\ No newline at end of file
+      tarCompression:  ${{ parameters.tarCompression }}
+  

diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml
index 5347d85..3dee80b 100644 (file)
--- a/eng/pipelines/coreclr/templates/perf-job.yml
+++ b/eng/pipelines/coreclr/templates/perf-job.yml
@@ -18,6 +18,7 @@ parameters:
   logicalMachine: ''
   pgoRunType: ''
   javascriptEngine: 'NoJS'
+  iOSLlvmBuild: 'False'
 
 ### Perf job
 
@@ -28,8 +29,8 @@ jobs:
 - template: ${{ parameters.runJobTemplate }}
   parameters:
     # Compute job name from template parameters
-    jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}_{9}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType) }}
-    displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5} {6} {7} {8} {9}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType) }}
+    jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}_{9}_{10}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType, parameters.iosLlvmBuild) }}
+    displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5} {6} {7} {8} {9} {10}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType, parameters.iosLlvmBuild) }}
     pool: ${{ parameters.pool }}
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -46,9 +47,10 @@ jobs:
     logicalmachine: ${{ parameters.logicalmachine }}
     pgoRunType: ${{ parameters.pgoRunType }}
     javascriptEngine: ${{ parameters.javascriptEngine }}
+    iosLlvmBuild: ${{ parameters.iosLlvmBuild }}
     # Test job depends on the corresponding build job
     dependsOn:
-    - ${{ if ne(parameters.runtimeType, 'AndroidMono')}}:
+    - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}:
       - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
     - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
       - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
@@ -60,13 +62,15 @@ jobs:
       - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }}
     - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}:
       - ${{ 'build_Android_arm64_release_AndroidMono' }}
+    - ${{ if eq(parameters.runtimeType, 'iOSMono')}}:
+      - ${{ 'build_iOS_arm64_release_iOSMono' }}
 
-    ${{ if and(eq(parameters.osGroup, 'windows'), ne(parameters.runtimeType, 'AndroidMono')) }}:
+    ${{ if and(eq(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono'))) }}:
       ${{ if eq(parameters.runtimeType, 'mono') }}:
         extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono
       ${{ if eq(parameters.runtimeType, 'coreclr') }}:
         extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }}
-    ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.runtimeType, 'AndroidMono')) }}:
+    ${{ if and(ne(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono'))) }}:
       ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}:
         extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono
       ${{ if eq(parameters.runtimeType, 'wasm') }}:
@@ -79,6 +83,8 @@ jobs:
         extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} --alpine
     ${{ if eq(parameters.runtimeType, 'AndroidMono') }}:
       extraSetupParameters: -Architecture ${{ parameters.archType }} -AndroidMono
+    ${{ if eq(parameters.runtimeType, 'iosMono') }}:
+      extraSetupParameters: -Architecture ${{ parameters.archType }} -iOSMono -iOSLlvmBuild:$${{ parameters.iOSLlvmBuild }}
 
     variables: ${{ parameters.variables }}
 
@@ -98,7 +104,7 @@ jobs:
           displayName: 'live-built libraries'
 
     # Download coreclr
-    - ${{ if ne(parameters.runtimeType, 'AndroidMono') }}:
+    - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: $(buildProductRootFolderPath)
@@ -149,6 +155,16 @@ jobs:
           artifactFileName: 'AndroidMonoarm64.tar.gz'
           artifactName: 'AndroidMonoarm64'
           displayName: 'Mono Android runtime'
+    
+    # Download iOSMono tests
+    - ${{ if eq(parameters.runtimeType, 'iOSMono') }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(Build.SourcesDirectory)/iosHelloWorld/nollvm
+          cleanUnpackFolder: false
+          artifactFileName: 'iOSSampleAppNoLLVM.tar.gz'
+          artifactName: 'iOSSampleAppNoLLVM'
+          displayName: 'iOS Sample App NoLLVM'
 
 
     # Create Core_Root
@@ -159,8 +175,8 @@ jobs:
     # Copy the runtime directory into the testhost folder to include OOBs.
     - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\runtime\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\shared\\Microsoft.NETCore.App\\6.0.0 /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\6.0.0\\corerun.exe"
       displayName: "Create mono dotnet (Windows)"
-      condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), ne('${{ parameters.runtimeType }}', 'AndroidMono'))
+      condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono')))
 
     - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/runtime/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/shared/Microsoft.NETCore.App/6.0.0 -rf;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun"
       displayName: "Create mono dotnet (Linux)"
-      condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), ne('${{ parameters.runtimeType }}', 'AndroidMono'))
+      condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono')))

diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml
index de523bb..e031d0f 100644 (file)
--- a/eng/pipelines/coreclr/templates/run-performance-job.yml
+++ b/eng/pipelines/coreclr/templates/run-performance-job.yml
@@ -23,6 +23,7 @@ parameters:
   runKind: ''                     # required -- test category
   logicalMachine: ''              # required -- Used to specify a which pool of machines the test should run against
   javascriptEngine: 'NoJS'
+  iosLlvmBuild: 'False'
   
 jobs:
 - template: xplat-pipeline-job.yml
@@ -142,6 +143,6 @@ jobs:
       displayName: Publish Logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.runKind }}_${{ parameters.logicalMachine }}_${{ parameters.javascriptEngine }}_${{ parameters.pgoRunType }}'
+        artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.runKind }}_${{ parameters.logicalMachine }}_${{ parameters.javascriptEngine }}_${{ parameters.pgoRunType }}_${{ parameters.iosLlvmBuild }}'
       continueOnError: true
       condition: always()

diff --git a/eng/testing/performance/ios_scenarios.proj b/eng/testing/performance/ios_scenarios.proj
new file mode 100644 (file)
index 0000000..c7fbf1e
--- /dev/null
+++ b/eng/testing/performance/ios_scenarios.proj
@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <Python>python3</Python>
+    <HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk</HelixPreCommands>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <PropertyGroup>
+    <LlvmPath>nollvm</LlvmPath>
+    <LlvmPath Condition="'$(iOSLlvmBuild)' == 'True'">llvm</LlvmPath>
+  </PropertyGroup>
+  
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
+    
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="SOD - iOS HelloWorld .app Size">
+        <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+        <PreCommands>cd $(ScenarioDirectory)helloios;xcopy %HELIX_CORRELATION_PAYLOAD%\iosHelloWorld\$(LlvmPath) .\app\/e;$(Python) pre.py</PreCommands>
+        <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
+        <PostCommands>$(Python) post.py</PostCommands>
+    </HelixWorkItem>
+  </ItemGroup>
+</Project>
\ No newline at end of file

diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1
index 3fc5a44..03aff42 100644 (file)
--- a/eng/testing/performance/performance-setup.ps1
+++ b/eng/testing/performance/performance-setup.ps1
@@ -21,9 +21,11 @@ Param(
     [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind",
     [string] $LogicalMachine="",
     [switch] $AndroidMono,
+    [switch] $iOSMono,
     [switch] $NoPGO,
     [switch] $DynamicPGO,
-    [switch] $FullPGO
+    [switch] $FullPGO,
+    [switch] $iOSLlvmBuild
 )
 
 $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
@@ -90,6 +92,10 @@ elseif($FullPGO)
     $Configurations += " PGOType=fullpgo"
 }
 
+if ($iOSMono) {
+    $Configurations += " iOSLlvmBuild=$iOSLlvmBuild"
+}
+
 # FIX ME: This is a workaround until we get this from the actual pipeline
 $CommonSetupArguments="--channel main --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture"
 $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
@@ -141,6 +147,15 @@ if ($AndroidMono) {
     $SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
 }
 
+if ($iOSMono) {
+    if(!(Test-Path $WorkItemDirectory))
+    {
+        mkdir $WorkItemDirectory
+    }
+    Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse
+    $SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
+}
+
 $DocsDir = (Join-Path $PerformanceDirectory "docs")
 robocopy $DocsDir $WorkItemDirectory
 
@@ -168,6 +183,7 @@ Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun
 Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'iOSLlvmBuild' -Value "$iOSLlvmBuild" -IsMultiJobVariable $false
 
 # Helix Arguments
 Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false

diff --git a/src/mono/sample/iOS/Makefile b/src/mono/sample/iOS/Makefile
index 0cea321..e3b7578 100644 (file)
--- a/src/mono/sample/iOS/Makefile
+++ b/src/mono/sample/iOS/Makefile
@@ -1,8 +1,9 @@
-MONO_CONFIG=Debug
-MONO_ARCH=x64
+MONO_CONFIG?=Debug
+MONO_ARCH?=x64
 DOTNET := ../../../../dotnet.sh
 USE_LLVM=true
-AOT=false
+AOT?=false
+TARGET?=iOSSimulator
 
 #If DIAGNOSTIC_PORTS is enabled, RUNTIME_COMPONENTS must also be enabled.
 #If RUNTIME_COMPONENTS is enabled, DIAGNOSTIC_PORTS is optional.
@@ -20,7 +21,7 @@ appbuilder:
        $(DOTNET) build -c Debug $(TOOLS_DIR)/AppleAppBuilder/AppleAppBuilder.csproj
 
 runtimepack:
-       ../../../../build.sh Mono+Libs -os iOSSimulator -arch $(MONO_ARCH) -c $(MONO_CONFIG)
+       ../../../../build.sh Mono+Libs -os $(TARGET)  -arch $(MONO_ARCH) -c $(MONO_CONFIG)
 
 run: clean appbuilder
        $(DOTNET) publish \
@@ -41,6 +42,10 @@ run-sim: clean appbuilder
        '/p:RuntimeComponents="$(RUNTIME_COMPONENTS)"' \
        '/p:DiagnosticPorts="$(DIAGNOSTIC_PORTS)"'
 
+build-appbundle: clean appbuilder
+       $(DOTNET) publish -c $(MONO_CONFIG) /p:TargetOS=$(TARGET) /p:TargetArchitecture=$(MONO_ARCH) \
+       /p:UseLLVM=$(USE_LLVM) /p:ForceAOT=$(AOT) /bl \
+
 run-catalyst:
        $(DOTNET) publish \
        -c $(MONO_CONFIG) \