Build corelib as part of libraries build. (#34664)
authorManish Godse <61718172+mangod9@users.noreply.github.com>
Sat, 11 Apr 2020 03:26:51 +0000 (20:26 -0700)
committerGitHub <noreply@github.com>
Sat, 11 Apr 2020 03:26:51 +0000 (20:26 -0700)
* simplify libraries build dependencies.

* FIx depends for tests

* Fix artifact name for webAssembly

* try building corelib during libraries build.

* small typo

* Merge coreclr+libraries build scripts

* update scripts based on new refactor

* fix script error

* dont build corelib for mono or testbuilds

* Handle webassembly case

* fix allconfigurations build

* remove separate corelib job

* remove corelib jobs from pipelines

* delete build-corelib-job

* PR feedback

* remove special case for webassembly

now that its dependent on the mono runtimeFlavor

eng/pipelines/common/build-coreclr-and-libraries-job.yml
eng/pipelines/coreclr/ci.yml
eng/pipelines/coreclr/templates/build-corelib-job.yml [deleted file]
eng/pipelines/libraries/base-job.yml
eng/pipelines/libraries/build-job.yml
eng/pipelines/libraries/outerloop.yml
eng/pipelines/runtime-official.yml
eng/pipelines/runtime.yml

index 7ea5e7a..ddf5d81 100644 (file)
@@ -13,21 +13,6 @@ parameters:
   pool: ''
 
 jobs:
-- template: /eng/pipelines/coreclr/templates/build-corelib-job.yml
-  parameters:
-    buildConfig: ${{ parameters.buildConfig }}
-    archType: ${{ parameters.archType }}
-    osGroup: ${{ parameters.osGroup }}
-    osSubgroup: ${{ parameters.osSubgroup }}
-    container: ${{ parameters.container }}
-    testGroup: ${{ parameters.testGroup }}
-    crossrootfsDir: ${{ parameters.crossrootfsDir }}
-    timeoutInminutes: ${{ parameters.timeoutInMinutes }}
-    signBinaries: ${{ parameters.signBinaries }}
-    stagedBuild: ${{ parameters.stagedBuild }}
-    variables: ${{ parameters.variables }}
-    pool: ${{ parameters.pool }}
-    
 - template: /eng/pipelines/coreclr/templates/build-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
index 0848c4c..0249dc3 100644 (file)
@@ -32,17 +32,6 @@ jobs:
 - template: /eng/pipelines/common/checkout-job.yml
 
 #
-# Checked Corelib builds
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
-    buildConfig: checked
-    platformGroup: all
-    jobParameters:
-      testGroup: outerloop
-
-#
 # Debug builds
 #
 - template: /eng/pipelines/common/platform-matrix.yml
diff --git a/eng/pipelines/coreclr/templates/build-corelib-job.yml b/eng/pipelines/coreclr/templates/build-corelib-job.yml
deleted file mode 100644 (file)
index b2b2c9a..0000000
+++ /dev/null
@@ -1,107 +0,0 @@
-parameters:
-  archType: ''
-  buildConfig: ''
-  condition: true
-  container: ''
-  crossrootfsDir: ''
-  isOfficialBuild: false
-  osGroup: ''
-  osSubgroup: ''
-  platform: ''
-  pool: ''
-  signBinaries: false
-  stagedBuild: false
-  testGroup: ''
-  timeoutInMinutes: ''
-  variables: {}
-
-### Product build
-jobs:
-- template: xplat-pipeline-job.yml
-  parameters:
-    buildConfig: ${{ parameters.buildConfig }}
-    _BuildConfig: ${{ parameters.buildConfig }}
-    archType: ${{ parameters.archType }}
-    osGroup: ${{ parameters.osGroup }}
-    osSubgroup: ${{ parameters.osSubgroup }}
-    testGroup: ${{ parameters.testGroup }}
-    helixType: 'build/product/'
-    enableMicrobuild: true
-    stagedBuild: ${{ parameters.stagedBuild }}
-    pool: ${{ parameters.pool }}
-    condition: ${{ parameters.condition }}
-
-    name: ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-    displayName: ${{ format('CoreCLR CoreLib Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-    
-    # Run all steps in the container.
-    # Note that the containers are defined in platform-matrix.yml
-    container: ${{ parameters.container }}
-
-    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-
-    crossrootfsDir: ${{ parameters.crossrootfsDir }}
-
-    gatherAssetManifests: true
-    variables:
-    - name: osGroup
-      value: ${{ parameters.osGroup }}
-    - name: osSubgroup
-      value: ${{ parameters.osSubgroup }}
-    - name: compilerArg
-      value: ''
-    - name: publishLogsArtifactPrefix
-      value: 'BuildLogs_CoreLib'
-    - name: officialBuildIdArg
-      value: ''
-    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
-      - name: officialBuildIdArg
-        value: '/p:OfficialBuildId=$(Build.BuildNumber)'
-
-    - ${{ parameters.variables }}
-
-    steps:
-
-    # Install native dependencies
-    # Linux builds use docker images with dependencies preinstalled,
-    # and FreeBSD builds use a build agent with dependencies
-    # preinstalled, so we only need this step for OSX and Windows.
-    - ${{ if eq(parameters.osGroup, 'OSX') }}:
-      - script: sh $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup)
-        displayName: Install native dependencies
-    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      # Necessary to install python
-      - script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
-        displayName: Install native dependencies
-
-    # Install internal tools on official builds
-    # Since our internal tools are behind an authenticated feed,
-    # we need to use the DotNetCli AzDO task to restore from the feed using a service connection.
-    # We can't do this from within the build, so we need to do this as a separate step.
-    - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
-      - template: /eng/pipelines/common/restore-internal-tools.yml
-
-    # Build Private CoreLib
-    - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci
-      displayName: Build System.Private.CoreLib
-
-
-    # Publish corelib output directory for consumption by libraries.
-    - template: /eng/pipelines/common/upload-artifact-step.yml
-      parameters:
-        rootFolder: $(buildProductRootFolderPath)
-        includeRootFolder: false
-        archiveType: $(archiveType)
-        tarCompression: $(tarCompression)
-        archiveExtension: $(archiveExtension)
-        artifactName: $(corelibProductArtifactName)
-        displayName: Upload System.Private.CoreLib
-
-    # Publish Logs
-    - task: PublishPipelineArtifact@1
-      displayName: Publish Logs
-      inputs:
-        targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)'
-      continueOnError: true
-      condition: always()
index 0208840..df6f5e3 100644 (file)
@@ -96,16 +96,15 @@ jobs:
         - _runtimeArtifactName: ''
         - _runtimeDownloadPath: ''
         - _runtimeArtifactsPathArg: ''
+        - _runtimeConfigurationArg: ''
 
         - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
           - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}'
-          # Download corelib dependencies for coreclr libraries and not test
-          - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
-            - _runtimeArtifactName: 'CoreLib_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
+          - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }}
           # Download full product dependencies for mono or test
           - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
             - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}'
-          - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
+            - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)'
           - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }}
 
         # Windows variables
@@ -116,7 +115,7 @@ jobs:
         - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
           - _buildScript: ./$(_buildScriptFileName)$(scriptExt)
 
-        - _buildArguments: -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_runtimeOSArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg)
+        - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testScopeArg) $(_runtimeOSArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg)
         - ${{ parameters.variables }}
 
       dependsOn:
@@ -131,7 +130,7 @@ jobs:
       steps:
       - template: /eng/pipelines/common/clone-checkout-bundle-step.yml
 
-      - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
+      - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, ''))) }}:
         - template: /eng/pipelines/common/download-artifact-step.yml
           parameters:
             unpackFolder: $(_runtimeDownloadPath)
index 26ecca2..c541e68 100644 (file)
@@ -49,21 +49,23 @@ jobs:
 
       ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
         dependsOn:
-        # Use corelib dependencies for coreclr and non test builds
-        - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
-          - ${{ format('coreclr_corelib_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
+        # Use full product dependency for non-coreclr and test builds
         - ${{ if or(ne(parameters.runtimeFlavor, 'coreclr'), ne(parameters.testScope, '')) }}:
           - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }}
 
       variables:
         - _subset: libs
-        - _addtionalBuildArguments: ''
+        - _additionalBuildArguments: ''
         - ${{ parameters.variables }}
         - ${{ if eq(parameters.osGroup, 'WebAssembly') }}:
           - EMSDK_PATH: /usr/local/emscripten
+        # for coreclr library builds (when not testing) build corelib as well.   
+        - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.testScope, '')) }}:
+          - _subset: clr.corelib+libs
+        # Tests only run for 'allConfiguration' and 'net472' build-jobs
         - ${{ if eq(parameters.runTests, true) }}:
-          - _subset: libs+libs.tests
-          - _addtionalBuildArguments: /p:ArchiveTests=true
+          - _subset: clr.corelib+libs+libs.tests
+          - _additionalBuildArguments: /p:ArchiveTests=true
 
         - ${{ parameters.variables }}
 
@@ -83,7 +85,7 @@ jobs:
         - script: $(_buildScript)
                 -subset $(_subset)
                 $(_buildArguments)
-                $(_addtionalBuildArguments)
+                $(_additionalBuildArguments)
           displayName: Restore and Build Product
 
         - ${{ if eq(parameters.runTests, false) }}:
index eb3b2e9..9afbf20 100644 (file)
@@ -13,29 +13,6 @@ variables:
 
 jobs:
   - template: /eng/pipelines/common/checkout-job.yml
-  
-  #
-  # CoreLib Build
-  #
-  - template: /eng/pipelines/common/platform-matrix.yml
-    parameters:
-      jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
-      buildConfig: release
-      platforms:
-      - ${{ if eq(variables['includeWindowsOuterloop'], true) }}:
-        - Windows_NT_x64
-        - Windows_NT_x86
-      - ${{ if eq(variables['includeLinuxOuterloop'], true) }}:
-        - Linux_x64
-        - Linux_musl_x64
-        - ${{ if eq(variables['isFullMatrix'], true) }}:
-          - Linux_arm
-          - Linux_arm64
-          - Linux_musl_arm64
-      - ${{ if eq(variables['includeOsxOuterloop'], true) }}:        
-        - OSX_x64
-      jobParameters:
-        testGroup: innerloop
 
   #
   # CoreCLR Build
index 0ab8f97..e2cd3a2 100644 (file)
@@ -40,27 +40,6 @@ stages:
   - template: /eng/pipelines/common/checkout-job.yml
 
   #
-  # Build CoreLib
-  #
-  - template: /eng/pipelines/common/platform-matrix.yml
-    parameters:
-      jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
-      buildConfig: release
-      platforms:
-      - OSX_x64
-      - Linux_x64
-      - Linux_arm
-      - Linux_arm64
-      - Linux_musl_x64
-      - Linux_musl_arm64
-      - Windows_NT_x86
-      - Windows_NT_x64
-      - Windows_NT_arm
-      - Windows_NT_arm64
-      jobParameters:
-        isOfficialBuild: ${{ variables.isOfficialBuild }}
-
-  #
   # Build CoreCLR
   #
   - template: /eng/pipelines/common/platform-matrix.yml
index 38cdb04..69fcff0 100644 (file)
@@ -119,27 +119,6 @@ jobs:
       - eng/pipelines/libraries/*
 
 #
-# Build CoreLib release
-#
-- template: /eng/pipelines/common/platform-matrix.yml
-  parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-corelib-job.yml
-    buildConfig: release
-    platforms:
-    - OSX_x64
-    - Linux_x64
-    - Linux_arm
-    - Linux_arm64
-    - Linux_musl_x64
-    - Linux_musl_arm64
-    - Windows_NT_x64
-    - Windows_NT_x86
-    - Windows_NT_arm
-    - Windows_NT_arm64
-    jobParameters:
-      testGroup: innerloop
-
-#
 # Build CoreCLR checked
 # Only when CoreCLR is changed
 #