eng/build-job.yml

   1 parameters:
   2   buildConfig: ''
   3   archType: ''
   4   osGroup: ''
   5   osIdentifier: ''
   6   container: ''
   7   crossrootfsDir: ''
   8   timeoutInMinutes: ''
   9
  10 ### Product build
  11 jobs:
  12 - template: xplat-job.yml
  13   parameters:
  14     buildConfig: ${{ parameters.buildConfig }}
  15     archType: ${{ parameters.archType }}
  16     osGroup: ${{ parameters.osGroup }}
  17     osIdentifier: ${{ parameters.osIdentifier }}
  18     helixType: 'build/product/'
  19     enableMicrobuild: true
  20
  21     # Compute job name from template parameters
  22     name: ${{ format('build_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
  23     displayName: ${{ format('{0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
  24
  25     # FreeBSD builds are disabled in the public project because we
  26     # don't have any FreeBSD agents in the public pool.
  27     ${{ if and(eq(parameters.osGroup, 'FreeBSD'), eq(variables['System.TeamProject'], 'public')) }}:
  28       condition: false
  29
  30     # Run all steps in the container.
  31     # Note that the containers are defined in platform-matrix.yml
  32     container: ${{ parameters.container }}
  33
  34     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
  35
  36     crossrootfsDir: ${{ parameters.crossrootfsDir }}
  37
  38     gatherAssetManifests: true
  39     variables:
  40     - name: osIdentifier
  41       value: ${{ parameters.osIdentifier }}
  42     - name: stripSymbolsArg
  43       value: ''
  44     # Strip symbols only on the release build
  45     - ${{ if eq(parameters.buildConfig, 'Release') }}:
  46       - name: stripSymbolsArg
  47         value: '-stripsymbols'
  48     - name: portableBuildArg
  49       value: ''
  50     # Ensure that we produce os-specific packages for the following distros:
  51     - ${{ if in(parameters.osIdentifier, 'Linux_rhel6') }}:
  52       - name: portableBuildArg
  53         value: '-portablebuild=false'
  54     - name: clangArg
  55       value: ''
  56     # Our FreeBSD doesn't yet detect available clang versions, so pass it explicitly.
  57     - ${{ if eq(parameters.osGroup, 'FreeBSD') }}:
  58       - name: clangArg
  59         value: '-clang6.0'
  60     - ${{ if eq(parameters.archType, 'arm64') }}:
  61       - name: clangArg
  62         value: '-clang5.0'
  63     - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
  64       # Variables used to publish packages to blob feed
  65       - name: dotnetfeedUrl
  66         value: https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json
  67       - name: dotnetfeedPAT
  68         value: $(dotnetfeed-storage-access-key-1)
  69       # Variables used by arcade to gather asset manifests
  70       - name: _DotNetPublishToBlobFeed
  71         value: true
  72     - name: officialBuildIdArg
  73       value: ''
  74     - name: ibcOptimizeArg
  75       value: ''
  76     - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
  77       - name: officialBuildIdArg
  78         value: '-officialbuildid=$(Build.BuildNumber)'
  79       # IBCMerge is currently Windows-only and x86/x64-only
  80       - ${{ if and(eq(parameters.osGroup, 'Windows_NT'), or(eq(parameters.archType, 'x64'), eq(parameters.archType, 'x86'))) }}:
  81         - name: ibcOptimizeArg
  82           value: '-ibcoptimize'
  83     - name: enforcePgoArg
  84       value: ''
  85     # The EnforcePGO script is only supported on Windows and is not supported on arm or arm64.
  86     - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'Windows_NT'), not(or(eq(parameters.archType, 'arm64'), eq(parameters.archType, 'arm'))))) }}:
  87       - name: enforcePgoArg
  88         value: '-enforcepgo'
  89
  90     steps:
  91
  92     # Install native dependencies
  93     # Linux builds use docker images with dependencies preinstalled,
  94     # and FreeBSD builds use a build agent with dependencies
  95     # preinstalled, so we only need this step for OSX and Windows.
  96     - ${{ if eq(parameters.osGroup, 'OSX') }}:
  97       - script: sh eng/install-native-dependencies.sh $(osGroup)
  98         displayName: Install native dependencies
  99     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
 100       # Necessary to install python
 101       - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
 102         displayName: Install native dependencies
 103
 104     # Install internal tools on official builds
 105     # Since our internal tools are behind an authenticated feed,
 106     # we need to use the DotNetCli AzDO task to restore from the feed using a service connection.
 107     # We can't do this from within the build, so we need to do this as a separate step.
 108     - ${{ if and(and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')), eq(parameters.osGroup, 'Windows_NT')) }}:
 109         - task: DotNetCoreInstaller@0
 110           inputs:
 111             packageType: 'sdk'
 112             version: '2.1.503'
 113         - task: DotNetCoreCLI@2
 114           displayName: Restore internal tools
 115           inputs:
 116             command: restore
 117             feedsToUse: config
 118             projects: '$(Build.SourcesDirectory)/eng/common/internal/Tools.csproj'
 119             nugetConfigPath: 'eng/internal/NuGet.config'
 120             restoreDirectory: '$(Build.SourcesDirectory)\.packages'
 121             verbosityRestore: 'normal'
 122             externalFeedCredentials: 'dotnet-core-internal-tooling'
 123
 124     # Build
 125     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
 126       - script: ./build.sh $(buildConfig) $(archType) $(crossArg) -skiptests -skipnuget $(clangArg) $(stripSymbolsArg) $(officialBuildIdArg) /p:ContinuousIntegrationBuild=true
 127         displayName: Build product
 128     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
 129       - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages $(officialBuildIdArg) $(ibcOptimizeArg) $(enforcePgoArg) /p:ContinuousIntegrationBuild=true
 130         displayName: Build product
 131
 132     # Sign on Windows
 133     - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.osGroup, 'Windows_NT')) }}:
 134       - powershell: eng\common\build.ps1 -ci -sign -restore -configuration:$(buildConfig) -warnaserror:0 /p:ArcadeBuild=true /p:OfficialBuild=true /p:BuildOS=$(osGroup) /p:BuildArch=$(archType) /p:BuildType=$(_BuildConfig) /p:DotNetSignType=$env:_SignType -projects $(Build.SourcesDirectory)\eng\empty.csproj
 135         displayName: Sign Binaries
 136
 137       - task: PublishBuildArtifacts@1
 138         displayName: Publish Signing Logs
 139         inputs:
 140           PathtoPublish: '$(Build.SourcesDirectory)/artifacts/'
 141           PublishLocation: Container
 142           ArtifactName: ${{ format('SignLogs_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 143         continueOnError: true
 144         condition: always()
 145
 146     # Publish product output directory for consumption by tests.
 147     - task: PublishBuildArtifacts@1
 148       displayName: Publish product build
 149       inputs:
 150         pathtoPublish: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(_BuildConfig)
 151         artifactName: ${{ format('BinDir_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 152
 153     # Get key vault secrets for publishing
 154     - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
 155       - task: AzureKeyVault@1
 156         inputs:
 157           azureSubscription: 'DotNet-Engineering-Services_KeyVault'
 158           KeyVaultName: EngKeyVault
 159           SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat'
 160
 161     # Build packages
 162     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
 163       - script: ./build-packages.sh -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(crossPackagesArg) $(officialBuildIdArg) $(portableBuildArg) -ci
 164         displayName: Build packages
 165     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
 166       - script: build-packages.cmd -BuildArch=$(archType) -BuildType=$(_BuildConfig) $(officialBuildIdArg) -ci
 167         displayName: Build packages
 168
 169     # Publish official build
 170     - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}:
 171       - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
 172         - script: ./eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osIdentifier) /bl:"$(Build.SourcesDirectory)/bin/Logs/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj
 173           displayName: Publish packages to blob feed
 174           env:
 175             # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed
 176             NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
 177             ${{ if eq(parameters.osGroup, 'FreeBSD') }}:
 178               # Arcade uses this SDK instead of trying to restore one.
 179               DotNetCoreSdkDir: /usr/local/dotnet
 180       - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
 181         # TODO: pass publish feed url and access token in from the internal pipeline
 182         - powershell: eng\common\build.ps1 -ci -restore -publish -configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:__BuildType=$(_BuildConfig) /p:__BuildArch=$(archType) /p:__BuildOS=$(osGroup) /p:OSIdentifier=$(osIdentifier) /bl:"$(Build.SourcesDirectory)\bin\Logs\publish-pkgs.binlog" -projects $(Build.SourcesDirectory)\eng\empty.csproj
 183           displayName: Publish packages to blob feed
 184           env:
 185             # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed
 186             NUGET_PACKAGES: $(Build.SourcesDirectory)\.packages
 187
 188     # Publish Logs
 189     - task: PublishBuildArtifacts@1
 190       displayName: Publish Logs
 191       inputs:
 192         pathtoPublish: $(Build.SourcesDirectory)/bin/Logs
 193         artifactName: ${{ format('BuildLogs_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
 194       continueOnError: true
 195       condition: always()