Correct platorm reach for the Official build (#21310)

* Correct platorm reach for the Official build

Does the following:

1. Sets up a ci rule for master and a pr rule for master
2. Corrects the OSX queues
3. Corrects ubuntu internal queue
4. Converts all linux jobs to build using containers
5. Only runs official builds on: Pri1, release, bringing us to parity with old process
6. Fixes centos build-test
7. Adds a super-annoying groupname tag that avoids name mangling.

* make sure we only add triggers for internal builds

* Fix syntax

* Missing :

* Remove pr/ci

* Fix internal queue name

* Correct Public/internal choices.

* Add crossgen as well

* Correct indentation

* Correct checked pri1 tests

* Address pr feedback

* Add todo comments

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index ae8191f..92de971 100644 (file)
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,6 +1,25 @@
 variables:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
 
+resources:
+  containers:
+  - container: ubuntu_1404_arm_cross_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420
+
+  - container: ubuntu_1604_arm64_cross_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921
+
+  - container: ubuntu_1604_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544
+
+  - container: musl_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:alpine-3.6-e2521f8-20180716231200
+
+  - container: centos7_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424
+
+  - container: centos6_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331
 
 jobs:
 
@@ -58,30 +77,30 @@ jobs:
 ##                                            |  (passed-in jobTemplate)  |                    (arcade)
 ##                                            \------> test-job.yml ------/
 
-
+# TODO: simplify logic surrounding official build/ci. See
+# https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information
 
 #
-# Debug build
+# Debug build (CI)
 #
-
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: debug
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: build-job.yml
+      buildConfig: debug
 
 #
-# Checked build
+# Checked build (CI)
 #
-
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: checked
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: build-job.yml
+      buildConfig: checked
 
 #
-# Release build
+# Release build (Official Build)
 #
-
 - template: eng/platform-matrix.yml
   parameters:
     jobTemplate: build-job.yml
@@ -91,53 +110,58 @@ jobs:
 # Checked test builds
 #
 
-# Pri0
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 0
-
-# Pri1
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 1
-      scenarios: 'normal;jitstress2'
-
-# Pri1 crossgen
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 1
-      crossgen: true
-      scenarios: 'normal;jitstress2'
+# Pri0 (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 0
+
+# Pri1 (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 1
+        scenarios: 'normal;jitstress2'
+
+# Pri1 crossgen (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 1
+        crossgen: true
+        scenarios: 'normal;jitstress2'
 
 #
-# Release test builds
+# Release test builds (Official Build)
 #
 
 # Pri1
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: release
-    jobParameters:
-      priority: 1
-
-# Pri1 crossgen
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: release
-    jobParameters:
-      priority: 1
-      crossgen: true
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: release
+      jobParameters:
+        priority: 1
+
+# Pri1 crossgen (Official Build)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: release
+      jobParameters:
+        priority: 1
+        crossgen: true
 
 
 # Publish build information to Build Assets Registry

diff --git a/build-test.sh b/build-test.sh
index 8a09aef..858085f 100755 (executable)
--- a/build-test.sh
+++ b/build-test.sh
@@ -11,12 +11,7 @@ initHostDistroRid()
 
     if [ "$__HostOS" == "Linux" ]; then
         if [ -e /etc/redhat-release ]; then
-            local redhatRelease=$(</etc/redhat-release)
-            if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
-                __HostDistroRid="rhel.6-$__HostArch"
-            else
-                __PortableBuild=1
-            fi
+            __PortableBuild=1
         elif [ -e /etc/os-release ]; then
             source /etc/os-release
             if [[ $ID == "alpine" ]]; then

diff --git a/eng/build-job.yml b/eng/build-job.yml
index 08f8ac0..e44df9c 100644 (file)
--- a/eng/build-job.yml
+++ b/eng/build-job.yml
@@ -2,6 +2,8 @@ parameters:
   buildConfig: ''
   archType: ''
   osGroup: ''
+  osGroupName: ''
+  containerName: ''
 
 ### Product build
 jobs:
@@ -10,15 +12,22 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
+    osGroupName: ${{ parameters.osGroupName }}
 
     # Compute job name from template parameters
-    name: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
-    displayName: ${{ format('Build {0} {1} {2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+    name: ${{ format('build_{0}_{1}_{2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+    displayName: ${{ format('Build {0} {1} {2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+
+    # Run all steps in the container.
+    # Note that the containers are resources defined in azure-pipelines.yml
+    containerName: ${{ parameters.containerName }}
 
     steps:
 
     # Install native dependencies
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    #
+    # This is only required for non-docker builds.
+    - ${{ if eq(parameters.osGroup, 'OSX') }}:
       - script: sh eng/install-native-dependencies.sh $(osGroup)
         displayName: Install native dependencies
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -26,19 +35,17 @@ jobs:
       - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
         displayName: Install native dependencies
 
-
     # Run init-tools (pre-arcade dependency bootstrapping)
     # TODO: replace this with an arcade equivalent
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - script: ./init-tools.sh
         displayName: Init tools
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: .\init-tools.cmd
         displayName: Init tools
 
-
     # Sync
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - script: ./Tools/dotnetcli/dotnet msbuild build.proj /p:RestoreDuringBuild=true /t:Sync
         displayName: Sync
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -47,9 +54,15 @@ jobs:
 
 
     # Build
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}:
       - script: ./build.sh $(buildConfig) $(archType) -skipnuget -skiprestore
         displayName: Build product
+    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}:
+      - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore
+        displayName: Build product
+    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}:
+      - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore
+        displayName: Build product
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?
       - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages -skiprestore
@@ -57,11 +70,11 @@ jobs:
 
 
     # Upload build as pipeline artifact
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - task: PublishPipelineArtifact@0
         displayName: Save product build as pipeline artifact
         inputs:
-          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
           targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - task: PublishPipelineArtifact@0

diff --git a/eng/platform-matrix.yml b/eng/platform-matrix.yml
index 7f1a463..fb1a3c6 100644 (file)
--- a/eng/platform-matrix.yml
+++ b/eng/platform-matrix.yml
@@ -5,6 +5,59 @@ parameters:
 
 jobs:
 
+# TODO: simplify osGroupName by adding osGroup and osSubGroup. See
+# https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information
+
+# Linux arm
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm
+    osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1404_arm_cross_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Linux arm64
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm64
+    osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1604_arm64_cross_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Linux musl
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_musl
+    containerName: musl_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# RHEL 6
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_rhel6
+    containerName: centos6_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# RHEL 7
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_rhel7
+    containerName: centos7_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
 # Linux x64
 
 - template: ${{ parameters.jobTemplate }}
@@ -12,6 +65,8 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1604_x64_build_image
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64
@@ -21,6 +76,7 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: OSX
+    osGroupName: OSX
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x64/x86/arm/arm64
@@ -30,6 +86,7 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
@@ -37,6 +94,7 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x86
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
@@ -44,6 +102,7 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: arm
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
@@ -51,11 +110,5 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: arm64
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
-
-# TODO for official build:
-# RedHat x64
-# Linux crossbuild arm
-# Linux crossbuild arm64
-# Linux musl x64
-

diff --git a/eng/test-job.yml b/eng/test-job.yml
index 0ec46d1..00982f7 100644 (file)
--- a/eng/test-job.yml
+++ b/eng/test-job.yml
@@ -2,6 +2,7 @@ parameters:
   buildConfig: ''
   archType: ''
   osGroup: ''
+  osGroupName: ''
   priority: 0
   crossgen: false
   scenarios: ''
@@ -17,14 +18,15 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
+    osGroupName: ${{ parameters.osGroupName }}
 
     # Compute job name from template parameters
     ${{ if eq(parameters.crossgen, 'false') }}:
-        name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }}
-        displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+        name: ${{ format('testbuild_pri{0}_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+        displayName: ${{ format('Test pri{0} {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
     ${{ if eq(parameters.crossgen, 'true') }}:
-        name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }}
-        displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+        name: ${{ format('testbuild_pri{0}_r2r_{1}_{2}_{3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+        displayName: ${{ format('Test Pri{0} R2R {1} {2} {3}', parameters.priority, parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
 
     variables:
       # Map template parameters to command line arguments
@@ -52,12 +54,16 @@ jobs:
       condition: false
 
     # Test job depends on the corresponding build job
-    dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+    dependsOn: ${{ format('build_{0}_{1}_{2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+
+    # Run all steps in the container.
+    # Note that the containers are resources defined in azure-pipelines.yml
+    containerName: ${{ parameters.containerName }}
 
     steps:
 
     # Install test build dependencies
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if eq(parameters.osGroup, 'OSX') }}:
       - script: sh eng/install-native-dependencies.sh $(osGroup)
         displayName: Install native dependencies
 
@@ -67,13 +73,13 @@ jobs:
       - task: DownloadPipelineArtifact@0
         displayName: Download product build pipeline artifact
         inputs:
-          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
           targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - task: DownloadPipelineArtifact@0
         displayName: Download product build pipeline artifact
         inputs:
-          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
           targetPath: $(Build.SourcesDirectory)\bin\Product\Windows_NT.$(archType).$(buildConfigUpper)
 
       
@@ -105,7 +111,7 @@ jobs:
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
             # Access token variable for internal project
-            HelixAccessToken: $(HelixApiAccessToken)
+            HelixAccessToken: $(DotNet-HelixApi-Access)
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)
@@ -115,7 +121,7 @@ jobs:
         env:
           ${{ if eq(variables['System.TeamProject'], 'internal') }}:
             # Access token variable for internal project
-            HelixAccessToken: $(HelixApiAccessToken)
+            HelixAccessToken: $(DotNet-HelixApi-Access)
           ${{ if eq(variables['System.TeamProject'], 'public') }}:
             # Access token variable for public project
             HelixAccessToken: $(BotAccount-dotnet-github-anon-kaonashi-bot-helix-token)

diff --git a/eng/xplat-job.yml b/eng/xplat-job.yml
index b2d04df..9677da4 100644 (file)
--- a/eng/xplat-job.yml
+++ b/eng/xplat-job.yml
@@ -2,10 +2,12 @@ parameters:
   buildConfig: ''
   archType: ''
   osGroup: ''
+  osGroupName: ''
   name: ''
   displayName: ''
   condition: ''
   dependsOn: ''
+  containerName: ''
   variables: {} ## any extra variables to add to the defaults defined below
 
 jobs:
@@ -20,11 +22,13 @@ jobs:
     dependsOn: ${{ parameters.dependsOn }}
 
     queue:
-      ${{ if eq(parameters.osGroup, 'Linux') }}:
+      ${{ if and(eq(parameters.osGroup, 'Linux'), eq(variables['System.TeamProject'], 'public')) }}:
         name: Hosted Ubuntu 1604
-      ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}:
+      ${{ if and(eq(parameters.osGroup, 'Linux'), ne(variables['System.TeamProject'], 'public')) }}:
+        name: dnceng-linux-internal-temp
+      ${{ if and(eq(parameters.osGroup, 'OSX'), ne(variables['System.TeamProject'], 'public')) }}:
         name: Hosted Mac Internal
-      ${{ if and(eq(parameters.osGroup, 'Windows_NT'), eq(variables['System.TeamProject'], 'public')) }}:
+      ${{ if and(eq(parameters.osGroup, 'OSX'), eq(variables['System.TeamProject'], 'public')) }}:
         name: Hosted MacOS
       ${{ if and(eq(parameters.osGroup, 'Windows_NT'), ne(variables['System.TeamProject'], 'public')) }}:
         name: dotnet-internal-temp
@@ -32,6 +36,9 @@ jobs:
         name: dotnet-external-temp
       timeoutInMinutes: 240
 
+      ${{ if ne(parameters.containerName, '') }}:
+        container: ${{ parameters.containerName }}
+
     ${{ if eq(parameters.osGroup, 'Linux') }}:
       agentOs: Ubuntu
     ${{ if eq(parameters.osGroup, 'OSX') }}:
@@ -49,6 +56,15 @@ jobs:
         buildConfigUpper: 'Release'
       archType: ${{ parameters.archType }}
       osGroup: ${{ parameters.osGroup }}
+      osGroupName: ${{ parameters.osGroupName }}
+      
+      # Crossbuild specific variables.
+      ${{ if eq(parameters.archType, 'arm') }}:
+        rootfsDir: /crossrootfs/arm
+        cacRootfsDir: /crossrootfs/x86
+      ${{ if eq(parameters.archType, 'arm64') }}:
+        rootfsDir: /crossrootfs/arm64
+
       ${{insert}}: ${{ parameters.variables }}
 
     steps: ${{ parameters.steps }}