From 1c81556da6058abe2fe2d9c9e96a58933af980fe Mon Sep 17 00:00:00 2001 From: Bill Wert Date: Thu, 24 Jun 2021 10:46:31 -0700 Subject: [PATCH] Add non-LLVM iOS size scenario (#54585) * Add non-LLVM iOS size scenario * add category * remove parens * variables in the wrong place. * move setting configs to before it is written --- eng/pipelines/coreclr/perf.yml | 40 +++++++++++++++++++++- .../coreclr/templates/build-perf-sample-apps.yml | 24 ++++++++++++- eng/pipelines/coreclr/templates/perf-job.yml | 32 ++++++++++++----- .../coreclr/templates/run-performance-job.yml | 3 +- eng/testing/performance/ios_scenarios.proj | 34 ++++++++++++++++++ eng/testing/performance/performance-setup.ps1 | 18 +++++++++- src/mono/sample/iOS/Makefile | 13 ++++--- 7 files changed, 148 insertions(+), 16 deletions(-) create mode 100644 eng/testing/performance/ios_scenarios.proj diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index e125f17..32ddf22 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -215,7 +215,28 @@ jobs: archiveExtension: '.tar.gz' archiveType: tar tarCompression: gz - + + # build mono iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - iOS_arm64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: iOSMono + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: iOS Mono Artifacts + artifactName: iOSMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono - template: /eng/pipelines/common/platform-matrix.yml @@ -242,6 +263,23 @@ jobs: runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfpixel4a' + # run mono iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Windows_x64 + jobParameters: + testGroup: perf + runtimeType: iOSMono + projectFile: ios_scenarios.proj + runKind: ios_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfpixel4a' + iosLlvmBuild: False + # run mono microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml index 2b556af..6bfd863 100644 --- a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml +++ b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml @@ -23,6 +23,27 @@ steps: - script: make run MONO_ARCH=arm64 DEPLOY_AND_RUN=false workingDirectory: $(Build.SourcesDirectory)/src/mono/sample/Android displayName: Build HelloAndroid sample app + - ${{ if eq(parameters.osGroup, 'iOS') }}: + - script: make build-appbundle TARGET=iOS MONO_ARCH=arm64 MONO_CONFIG=Release AOT=True USE_LLVM=False + env: + DevTeamProvisioning: '-' + workingDirectory: $(Build.SourcesDirectory)/src/mono/sample/iOS + displayName: Build HelloiOS AOT sample app LLVM=False + - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + displayName: 'Publish binlog' + inputs: + pathtoPublish: $(Build.SourcesDirectory)/src/mono/sample/iOS/msbuild.binlog + artifactName: ${{ parameters.artifactName }} + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/src/mono/sample/iOS/bin/ios-arm64/publish/app/HelloiOS/Release-iphoneos/HelloiOS.app + includeRootFolder: true + displayName: iOS Sample App NoLLVM + artifactName: iOSSampleAppNoLLVM + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -42,4 +63,5 @@ steps: artifactName: ${{ parameters.artifactName }} archiveExtension: ${{ parameters.archiveExtension }} archiveType: ${{ parameters.archiveType }} - tarCompression: ${{ parameters.tarCompression }} \ No newline at end of file + tarCompression: ${{ parameters.tarCompression }} + diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 5347d85..3dee80b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -18,6 +18,7 @@ parameters: logicalMachine: '' pgoRunType: '' javascriptEngine: 'NoJS' + iOSLlvmBuild: 'False' ### Perf job @@ -28,8 +29,8 @@ jobs: - template: ${{ parameters.runJobTemplate }} parameters: # Compute job name from template parameters - jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}_{9}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType) }} - displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5} {6} {7} {8} {9}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType) }} + jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}_{5}_{6}_{7}_{8}_{9}_{10}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType, parameters.iosLlvmBuild) }} + displayName: ${{ format('Performance {0}{1} {2} {3} {4} {5} {6} {7} {8} {9} {10}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind, parameters.logicalMachine, parameters.javascriptEngine, parameters.pgoRunType, parameters.iosLlvmBuild) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -46,9 +47,10 @@ jobs: logicalmachine: ${{ parameters.logicalmachine }} pgoRunType: ${{ parameters.pgoRunType }} javascriptEngine: ${{ parameters.javascriptEngine }} + iosLlvmBuild: ${{ parameters.iosLlvmBuild }} # Test job depends on the corresponding build job dependsOn: - - ${{ if ne(parameters.runtimeType, 'AndroidMono')}}: + - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}: - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} @@ -60,13 +62,15 @@ jobs: - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: - ${{ 'build_Android_arm64_release_AndroidMono' }} + - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: + - ${{ 'build_iOS_arm64_release_iOSMono' }} - ${{ if and(eq(parameters.osGroup, 'windows'), ne(parameters.runtimeType, 'AndroidMono')) }}: + ${{ if and(eq(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono'))) }}: ${{ if eq(parameters.runtimeType, 'mono') }}: extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono ${{ if eq(parameters.runtimeType, 'coreclr') }}: extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} - ${{ if and(ne(parameters.osGroup, 'windows'), ne(parameters.runtimeType, 'AndroidMono')) }}: + ${{ if and(ne(parameters.osGroup, 'windows'), not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono'))) }}: ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono ${{ if eq(parameters.runtimeType, 'wasm') }}: @@ -79,6 +83,8 @@ jobs: extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} --alpine ${{ if eq(parameters.runtimeType, 'AndroidMono') }}: extraSetupParameters: -Architecture ${{ parameters.archType }} -AndroidMono + ${{ if eq(parameters.runtimeType, 'iosMono') }}: + extraSetupParameters: -Architecture ${{ parameters.archType }} -iOSMono -iOSLlvmBuild:$${{ parameters.iOSLlvmBuild }} variables: ${{ parameters.variables }} @@ -98,7 +104,7 @@ jobs: displayName: 'live-built libraries' # Download coreclr - - ${{ if ne(parameters.runtimeType, 'AndroidMono') }}: + - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono')) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(buildProductRootFolderPath) @@ -149,6 +155,16 @@ jobs: artifactFileName: 'AndroidMonoarm64.tar.gz' artifactName: 'AndroidMonoarm64' displayName: 'Mono Android runtime' + + # Download iOSMono tests + - ${{ if eq(parameters.runtimeType, 'iOSMono') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/iosHelloWorld/nollvm + cleanUnpackFolder: false + artifactFileName: 'iOSSampleAppNoLLVM.tar.gz' + artifactName: 'iOSSampleAppNoLLVM' + displayName: 'iOS Sample App NoLLVM' # Create Core_Root @@ -159,8 +175,8 @@ jobs: # Copy the runtime directory into the testhost folder to include OOBs. - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\runtime\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\shared\\Microsoft.NETCore.App\\6.0.0 /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\6.0.0\\corerun.exe" displayName: "Create mono dotnet (Windows)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), ne('${{ parameters.runtimeType }}', 'AndroidMono')) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono'))) - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/runtime/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/shared/Microsoft.NETCore.App/6.0.0 -rf;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun" displayName: "Create mono dotnet (Linux)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), ne('${{ parameters.runtimeType }}', 'AndroidMono')) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono'))) diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index de523bb..e031d0f 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -23,6 +23,7 @@ parameters: runKind: '' # required -- test category logicalMachine: '' # required -- Used to specify a which pool of machines the test should run against javascriptEngine: 'NoJS' + iosLlvmBuild: 'False' jobs: - template: xplat-pipeline-job.yml @@ -142,6 +143,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.runKind }}_${{ parameters.logicalMachine }}_${{ parameters.javascriptEngine }}_${{ parameters.pgoRunType }}' + artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}_${{ parameters.codeGenType }}_${{ parameters.runKind }}_${{ parameters.logicalMachine }}_${{ parameters.javascriptEngine }}_${{ parameters.pgoRunType }}_${{ parameters.iosLlvmBuild }}' continueOnError: true condition: always() diff --git a/eng/testing/performance/ios_scenarios.proj b/eng/testing/performance/ios_scenarios.proj new file mode 100644 index 0000000..c7fbf1e --- /dev/null +++ b/eng/testing/performance/ios_scenarios.proj @@ -0,0 +1,34 @@ + + + python3 + $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk + + + + + %(Identity) + + + + + nollvm + llvm + + + + %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + + + + $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + + + + + $(WorkItemDirectory) + cd $(ScenarioDirectory)helloios;xcopy %HELIX_CORRELATION_PAYLOAD%\iosHelloWorld\$(LlvmPath) .\app\/e;$(Python) pre.py + $(Python) test.py sod --scenario-name "%(Identity)" + $(Python) post.py + + + \ No newline at end of file diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 3fc5a44..03aff42 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -21,9 +21,11 @@ Param( [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind", [string] $LogicalMachine="", [switch] $AndroidMono, + [switch] $iOSMono, [switch] $NoPGO, [switch] $DynamicPGO, - [switch] $FullPGO + [switch] $FullPGO, + [switch] $iOSLlvmBuild ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") @@ -90,6 +92,10 @@ elseif($FullPGO) $Configurations += " PGOType=fullpgo" } +if ($iOSMono) { + $Configurations += " iOSLlvmBuild=$iOSLlvmBuild" +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel main --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" @@ -141,6 +147,15 @@ if ($AndroidMono) { $SetupArguments = $SetupArguments -replace $Architecture, 'arm64' } +if ($iOSMono) { + if(!(Test-Path $WorkItemDirectory)) + { + mkdir $WorkItemDirectory + } + Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse + $SetupArguments = $SetupArguments -replace $Architecture, 'arm64' +} + $DocsDir = (Join-Path $PerformanceDirectory "docs") robocopy $DocsDir $WorkItemDirectory @@ -168,6 +183,7 @@ Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'iOSLlvmBuild' -Value "$iOSLlvmBuild" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/src/mono/sample/iOS/Makefile b/src/mono/sample/iOS/Makefile index 0cea321..e3b7578 100644 --- a/src/mono/sample/iOS/Makefile +++ b/src/mono/sample/iOS/Makefile @@ -1,8 +1,9 @@ -MONO_CONFIG=Debug -MONO_ARCH=x64 +MONO_CONFIG?=Debug +MONO_ARCH?=x64 DOTNET := ../../../../dotnet.sh USE_LLVM=true -AOT=false +AOT?=false +TARGET?=iOSSimulator #If DIAGNOSTIC_PORTS is enabled, RUNTIME_COMPONENTS must also be enabled. #If RUNTIME_COMPONENTS is enabled, DIAGNOSTIC_PORTS is optional. @@ -20,7 +21,7 @@ appbuilder: $(DOTNET) build -c Debug $(TOOLS_DIR)/AppleAppBuilder/AppleAppBuilder.csproj runtimepack: - ../../../../build.sh Mono+Libs -os iOSSimulator -arch $(MONO_ARCH) -c $(MONO_CONFIG) + ../../../../build.sh Mono+Libs -os $(TARGET) -arch $(MONO_ARCH) -c $(MONO_CONFIG) run: clean appbuilder $(DOTNET) publish \ @@ -41,6 +42,10 @@ run-sim: clean appbuilder '/p:RuntimeComponents="$(RUNTIME_COMPONENTS)"' \ '/p:DiagnosticPorts="$(DIAGNOSTIC_PORTS)"' +build-appbundle: clean appbuilder + $(DOTNET) publish -c $(MONO_CONFIG) /p:TargetOS=$(TARGET) /p:TargetArchitecture=$(MONO_ARCH) \ + /p:UseLLVM=$(USE_LLVM) /p:ForceAOT=$(AOT) /bl \ + run-catalyst: $(DOTNET) publish \ -c $(MONO_CONFIG) \ -- 2.7.4